//! Official resource post-sync parse cache. //! //! The parser runs after official download verification has completed. It reads //! the immutable published resource tree, writes a small derived cache next to //! `official-download-manifest.json`, and leaves localized output generation to //! later patch/export stages. use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry}; use crate::path_security::{ ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, STATE_FILE_MODE, }; use bat_assetbundle::{ Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser, }; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; /// Parse-cache file name stored under a published official resource root. pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json"; /// Current parse-cache schema version. pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2; /// TextUnit detail index file name stored under a published official resource root. pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json"; /// Current TextUnit detail-index schema version. pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1; /// Configuration for one official resource parse-cache refresh. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialParseConfig { /// Published official resource root containing `official-download-manifest.json`. pub resource_root: PathBuf, /// `unzip` executable used to inspect zip archives without extracting them. pub unzip_command: PathBuf, } impl OfficialParseConfig { /// Creates parse-cache configuration for a published official resource root. pub fn new(resource_root: impl Into, unzip_command: impl Into) -> Self { Self { resource_root: resource_root.into(), unzip_command: unzip_command.into(), } } /// Returns the parse-cache path for this resource root. pub fn cache_path(&self) -> PathBuf { self.resource_root.join(OFFICIAL_PARSE_CACHE_FILE) } } /// Structured report returned by a parse-cache refresh. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialParseReport { /// Official resource root that was inspected. pub resource_root: PathBuf, /// Parse-cache path written by the refresh. pub cache_path: PathBuf, /// Aggregate parse-cache summary. pub summary: OfficialParseSummary, /// TextUnit detail-index path written by the refresh. pub textunit_index_path: PathBuf, /// Aggregate TextUnit detail-index summary. pub textunit_index_summary: OfficialTextUnitIndexSummary, } /// Aggregate counters for a parse-cache refresh. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialParseSummary { /// Number of entries in `official-download-manifest.json`. pub manifest_entry_count: usize, /// Number of entries stored in `official-parse-cache.json`. pub cache_entry_count: usize, /// Number of direct bundle files or zip inner files inspected as parser candidates. pub candidate_file_count: usize, /// Number of zip inner file entries inspected. pub zip_entry_count: usize, /// Number of cached entries reused because source URL/path/size/BLAKE3 did not change. pub skipped_unchanged_count: usize, /// Number of bundles parsed successfully. pub parsed_bundle_count: usize, /// Number of files intentionally skipped because they are not UnityFS bundles. pub unsupported_count: usize, /// Number of files or archives that failed parser/cache inspection. pub failed_count: usize, /// Total TextAsset objects found in parsed Unity serialized files. pub text_asset_count: usize, /// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields. #[serde(default)] pub text_unit_count: usize, /// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor. #[serde(default)] pub skipped_binary_text_asset_count: usize, /// Number of non-fatal TypeTree field extraction diagnostics. #[serde(default)] pub text_unit_error_count: usize, } /// Persistent parse cache for one official resource root. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialParseCache { /// Cache schema version. #[serde(default = "default_parse_cache_version")] pub version: u32, /// Cache generation time as Unix seconds. pub generated_unix_seconds: u64, /// Aggregate summary for this cache. pub summary: OfficialParseSummary, /// Cache entries keyed by source URL and optional archive entry. #[serde(default)] pub entries: BTreeMap, } /// One parsed or skipped official resource file. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialParseCacheEntry { /// Stable cache key. pub key: String, /// Official URL from the download manifest. pub source_url: String, /// Relative destination path under the official resource root. pub destination: String, /// Inner archive path when the source is a zip file. pub archive_entry: Option, /// Source classification used by the parser. pub source_kind: OfficialParseSourceKind, /// Fingerprint derived from the download manifest. pub fingerprint: OfficialParseSourceFingerprint, /// Parse status for this cache entry. pub status: OfficialParseStatus, /// Whether this entry was reused from the previous parse cache in this run. #[serde(default)] pub reused_from_previous_cache: bool, /// Unity editor version when a UnityFS bundle was parsed. pub unity_version: Option, /// Number of files extracted from the UnityFS directory table. pub file_count: usize, /// Number of Unity serialized files parsed from extracted files. pub serialized_file_count: usize, /// Number of TextAsset objects found. pub text_asset_count: usize, /// TextAsset names found in stable order. pub text_assets: Vec, /// Non-fatal serialized-file parse diagnostic count. pub serialized_parse_error_count: usize, /// Number of translation-ready TextUnit entries extracted from this bundle. #[serde(default)] pub text_unit_count: usize, /// Stable set of TextUnit payload formats such as json/csv/tsv/plain. #[serde(default)] pub text_unit_formats: Vec, /// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction. #[serde(default)] pub skipped_binary_text_asset_count: usize, /// Non-fatal TypeTree field extraction diagnostic count. #[serde(default)] pub text_unit_error_count: usize, /// Human-readable error or skip reason. pub error: Option, } /// Persistent TextUnit detail index for one official resource root. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialTextUnitIndex { /// Index schema version. #[serde(default = "default_textunit_index_version")] pub version: u32, /// Index generation time as Unix seconds. pub generated_unix_seconds: u64, /// Official resource root used to generate this index. pub resource_root: PathBuf, /// Aggregate counters for this index. pub summary: OfficialTextUnitIndexSummary, /// Extracted TextUnit details in deterministic order. #[serde(default)] pub units: Vec, /// Parse and extraction diagnostics in deterministic order. #[serde(default)] pub errors: Vec, } /// Aggregate counters for a TextUnit detail index. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialTextUnitIndexSummary { /// Number of TextUnit detail entries. pub unit_count: usize, /// Number of parse/extraction diagnostics. pub error_count: usize, /// Number of binary or invalid TextAsset payloads skipped by extraction. pub skipped_binary_text_asset_count: usize, } /// One TextUnit detail entry. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialTextUnitIndexUnit { /// Stable unit ID within the official release. pub id: String, /// Parse-cache entry key that produced this unit. pub parse_entry_key: String, /// Official URL from the download manifest. pub source_url: String, /// Relative destination path under the official resource root. pub destination: String, /// Inner archive path when the source is a zip file. #[serde(default, skip_serializing_if = "Option::is_none")] pub archive_entry: Option, /// Source classification used by the parser. pub source_kind: OfficialParseSourceKind, /// Unity editor version when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub unity_version: Option, /// Original source text. pub source_text: String, /// Unity serialized file path. #[serde(default, skip_serializing_if = "Option::is_none")] pub serialized_file: Option, /// Unity object path ID. #[serde(default, skip_serializing_if = "Option::is_none")] pub path_id: Option, /// Unity class ID. #[serde(default, skip_serializing_if = "Option::is_none")] pub class_id: Option, /// TypeTree field path. #[serde(default, skip_serializing_if = "Option::is_none")] pub field_path: Option, /// Byte offset relative to the beginning of the Unity object payload. #[serde(default, skip_serializing_if = "Option::is_none")] pub field_offset: Option, /// Number of bytes consumed by this field, including alignment padding. #[serde(default, skip_serializing_if = "Option::is_none")] pub field_byte_size: Option, /// TextUnit payload format such as json/csv/tsv/plain. #[serde(default, skip_serializing_if = "Option::is_none")] pub format: Option, /// Extraction source kind such as TextAsset or TypeTreeField. #[serde(default, skip_serializing_if = "Option::is_none")] pub text_source_kind: Option, /// TextAsset name when the unit came from a TextAsset payload. #[serde(default, skip_serializing_if = "Option::is_none")] pub asset_name: Option, /// Stable extraction context copied from the parser. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub context: BTreeMap, } /// One parse or extraction diagnostic in the TextUnit detail index. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialTextUnitIndexError { /// Stable diagnostic ID within the official release. pub id: String, /// Parse-cache entry key associated with this diagnostic. pub parse_entry_key: String, /// Official URL from the download manifest. pub source_url: String, /// Relative destination path under the official resource root. pub destination: String, /// Inner archive path when the source is a zip file. #[serde(default, skip_serializing_if = "Option::is_none")] pub archive_entry: Option, /// Source classification used by the parser. pub source_kind: OfficialParseSourceKind, /// Parse status associated with this diagnostic. pub status: OfficialParseStatus, /// Unity serialized file path when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub serialized_file: Option, /// Unity object path ID when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub path_id: Option, /// Unity class ID when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub class_id: Option, /// TypeTree field path when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub field_path: Option, /// Byte offset relative to the beginning of the Unity object payload. #[serde(default, skip_serializing_if = "Option::is_none")] pub offset: Option, /// Human-readable diagnostic. pub error: String, } /// Query filters for TextUnit detail entries. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialTextUnitQuery { /// Filter by destination path. pub destination: Option, /// Filter by destination glob pattern. pub path_pattern: Option, /// Filter by archive entry. pub archive_entry: Option, /// Filter by Unity object path ID. pub path_id: Option, /// Filter by Unity class ID. pub class_id: Option, /// Filter by field path. pub field_path: Option, /// Filter by TextUnit format. pub format: Option, } /// Source kind for a parse-cache entry. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OfficialParseSourceKind { /// A direct official resource file that should be a UnityFS bundle. DirectBundle, /// A file entry streamed from an official zip archive. ZipEntry, /// A manifest entry that is not expected to contain a UnityFS bundle. Unsupported, } /// Stable fingerprint copied from the official download manifest. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialParseSourceFingerprint { /// Official URL from the download manifest. pub source_url: String, /// Relative destination path under the official resource root. pub destination: String, /// File byte count from the verified download manifest. pub bytes: u64, /// BLAKE3 digest from the verified download manifest. pub blake3: String, } /// Parse status for one official resource cache entry. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OfficialParseStatus { /// UnityFS parsing succeeded. Parsed, /// The source was inspected and is not currently parsed as UnityFS. SkippedUnsupported, /// The source was expected to be parseable but failed inspection. Failed, } /// Service that refreshes official parse caches. #[derive(Debug, Default, Clone, Copy)] pub struct OfficialParseCacheService; impl OfficialParseCacheService { /// Creates a parse-cache service. pub fn new() -> Self { Self } /// Refreshes the parse cache for the configured official resource root. pub fn run(&self, config: &OfficialParseConfig) -> Result { let manifest = read_download_manifest_at(&config.resource_root)?.ok_or_else(|| { format!( "缺少官方下载 manifest，无法更新解析缓存：{}", config.resource_root.display() ) })?; let previous_cache = read_parse_cache_at(&config.resource_root)?; let previous_textunit_index = read_textunit_index_at(&config.resource_root)?; let mut summary = OfficialParseSummary { manifest_entry_count: manifest.entries.len(), ..OfficialParseSummary::default() }; let mut entries = BTreeMap::new(); let mut units = Vec::new(); let mut errors = Vec::new(); for manifest_entry in manifest.entries.values() { let produced = process_manifest_entry( config, manifest_entry, previous_cache.as_ref(), previous_textunit_index.as_ref(), ); for produced in produced { summary.record_entry(&produced.entry); units.extend(produced.units); errors.extend(produced.errors); entries.insert(produced.entry.key.clone(), produced.entry); } } summary.cache_entry_count = entries.len(); let textunit_index_summary = OfficialTextUnitIndexSummary { unit_count: units.len(), error_count: errors.len(), skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count, }; let cache = OfficialParseCache { version: OFFICIAL_PARSE_CACHE_VERSION, generated_unix_seconds: unix_seconds_now(), summary: summary.clone(), entries, }; write_parse_cache_at(&config.resource_root, &cache)?; let textunit_index = OfficialTextUnitIndex { version: OFFICIAL_TEXTUNIT_INDEX_VERSION, generated_unix_seconds: cache.generated_unix_seconds, resource_root: config.resource_root.clone(), summary: textunit_index_summary.clone(), units, errors, }; write_textunit_index_at(&config.resource_root, &textunit_index)?; Ok(OfficialParseReport { resource_root: config.resource_root.clone(), cache_path: config.cache_path(), summary, textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE), textunit_index_summary, }) } } struct ParseProduced { entry: OfficialParseCacheEntry, units: Vec, errors: Vec, } impl ParseProduced { fn from_entry(entry: OfficialParseCacheEntry) -> Self { let errors = error_from_status_entry(&entry).into_iter().collect(); Self { entry, units: Vec::new(), errors, } } fn from_cached( mut entry: OfficialParseCacheEntry, previous_index: Option<&OfficialTextUnitIndex>, ) -> Self { entry.reused_from_previous_cache = true; let units = previous_index .into_iter() .flat_map(|index| index.units.iter()) .filter(|unit| unit.parse_entry_key == entry.key) .cloned() .collect(); let errors = previous_index .into_iter() .flat_map(|index| index.errors.iter()) .filter(|error| error.parse_entry_key == entry.key) .cloned() .collect(); Self { entry, units, errors, } } } impl OfficialParseSummary { fn record_entry(&mut self, entry: &OfficialParseCacheEntry) { match entry.source_kind { OfficialParseSourceKind::DirectBundle => { self.candidate_file_count += 1; } OfficialParseSourceKind::ZipEntry => { self.candidate_file_count += 1; self.zip_entry_count += 1; } OfficialParseSourceKind::Unsupported => {} } match entry.status { OfficialParseStatus::Parsed => { self.parsed_bundle_count += 1; self.text_asset_count += entry.text_asset_count; self.text_unit_count += entry.text_unit_count; self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count; self.text_unit_error_count += entry.text_unit_error_count; } OfficialParseStatus::SkippedUnsupported => { self.unsupported_count += 1; } OfficialParseStatus::Failed => { self.failed_count += 1; } } if entry.reused_from_previous_cache { self.skipped_unchanged_count += 1; } } } /// Reads the parse cache under a published official resource root. /// /// Missing, corrupt, or unsupported-version caches return `Ok(None)` so a new /// cache can be regenerated without blocking official resource publication. pub fn read_parse_cache_at(resource_root: &Path) -> Result, String> { let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE); let Some(bytes) = read_file_no_symlink(&path, "官方解析缓存")? else { return Ok(None); }; let Ok(cache) = serde_json::from_slice::(&bytes) else { return Ok(None); }; if cache.version != OFFICIAL_PARSE_CACHE_VERSION { return Ok(None); } Ok(Some(cache)) } /// Writes the parse cache under a published official resource root. pub fn write_parse_cache_at( resource_root: &Path, cache: &OfficialParseCache, ) -> Result<(), String> { let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE); ensure_path_within_root(resource_root, &path)?; ensure_safe_file_target(resource_root, &path, "官方解析缓存")?; let bytes = serde_json::to_vec_pretty(cache) .map_err(|error| format!("序列化官方解析缓存失败：{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存") } /// Reads the TextUnit detail index under a published official resource root. pub fn read_textunit_index_at( resource_root: &Path, ) -> Result, String> { let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE); let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else { return Ok(None); }; let Ok(index) = serde_json::from_slice::(&bytes) else { return Ok(None); }; if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION { return Ok(None); } Ok(Some(index)) } /// Writes the TextUnit detail index under a published official resource root. pub fn write_textunit_index_at( resource_root: &Path, index: &OfficialTextUnitIndex, ) -> Result<(), String> { let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE); ensure_path_within_root(resource_root, &path)?; ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?; let bytes = serde_json::to_vec_pretty(index) .map_err(|error| format!("序列化官方 TextUnit 明细索引失败：{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引") } /// Returns TextUnit detail entries matching a query. pub fn query_textunit_index_units<'a>( index: &'a OfficialTextUnitIndex, query: &OfficialTextUnitQuery, ) -> Vec<&'a OfficialTextUnitIndexUnit> { index .units .iter() .filter(|unit| textunit_unit_matches(unit, query)) .collect() } /// Returns parse/extraction diagnostics matching a query. pub fn query_textunit_index_errors<'a>( index: &'a OfficialTextUnitIndex, query: &OfficialTextUnitQuery, ) -> Vec<&'a OfficialTextUnitIndexError> { index .errors .iter() .filter(|error| textunit_error_matches(error, query)) .collect() } fn process_manifest_entry( config: &OfficialParseConfig, manifest_entry: &OfficialDownloadManifestEntry, previous_cache: Option<&OfficialParseCache>, previous_index: Option<&OfficialTextUnitIndex>, ) -> Vec { let fingerprint = fingerprint_for(manifest_entry); if looks_like_zip_source(manifest_entry) { return process_zip_entry( config, manifest_entry, previous_cache, previous_index, fingerprint, ); } if looks_like_direct_bundle_source(manifest_entry) { let key = direct_key(&manifest_entry.url); if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) { return vec![ParseProduced::from_cached(cached, previous_index)]; } return vec![parse_direct_bundle( config, manifest_entry, key, fingerprint, )]; } let key = unsupported_key(&manifest_entry.url); if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) { return vec![ParseProduced::from_cached(cached, previous_index)]; } vec![ParseProduced::from_entry(unsupported_entry( manifest_entry, None, OfficialParseSourceKind::Unsupported, fingerprint, key, "非 UnityFS 候选资源", ))] } fn process_zip_entry( config: &OfficialParseConfig, manifest_entry: &OfficialDownloadManifestEntry, previous_cache: Option<&OfficialParseCache>, previous_index: Option<&OfficialTextUnitIndex>, fingerprint: OfficialParseSourceFingerprint, ) -> Vec { let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint); if !cached_entries.is_empty() { return cached_entries .into_iter() .map(|entry| ParseProduced::from_cached(entry, previous_index)) .collect(); } let archive_path = match resource_path_for(&config.resource_root, manifest_entry) { Ok(path) => path, Err(error) => { return vec![ParseProduced::from_entry(failed_entry( manifest_entry, None, OfficialParseSourceKind::ZipEntry, fingerprint, zip_list_key(&manifest_entry.url), error, ))] } }; let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) { Ok(entries) => entries, Err(error) => { return vec![ParseProduced::from_entry(failed_entry( manifest_entry, None, OfficialParseSourceKind::ZipEntry, fingerprint, zip_list_key(&manifest_entry.url), error, ))] } }; let mut produced = Vec::new(); for archive_entry in archive_entries { let key = zip_key(&manifest_entry.url, &archive_entry); let bytes = match read_zip_entry_bytes(&config.unzip_command, &archive_path, &archive_entry) { Ok(bytes) => bytes, Err(error) => { produced.push(ParseProduced::from_entry(failed_entry( manifest_entry, Some(archive_entry), OfficialParseSourceKind::ZipEntry, fingerprint.clone(), key, error, ))); continue; } }; produced.push(parse_zip_inner_file( manifest_entry, archive_entry, fingerprint.clone(), key, &bytes, )); } if produced.is_empty() { produced.push(ParseProduced::from_entry(unsupported_entry( manifest_entry, None, OfficialParseSourceKind::Unsupported, fingerprint, zip_list_key(&manifest_entry.url), "ZIP 内没有可检查文件条目", ))); } produced } fn parse_direct_bundle( config: &OfficialParseConfig, manifest_entry: &OfficialDownloadManifestEntry, key: String, fingerprint: OfficialParseSourceFingerprint, ) -> ParseProduced { let path = match resource_path_for(&config.resource_root, manifest_entry) { Ok(path) => path, Err(error) => { return ParseProduced::from_entry(failed_entry( manifest_entry, None, OfficialParseSourceKind::DirectBundle, fingerprint, key, error, )) } }; let bytes = match read_resource_file(&path) { Ok(bytes) => bytes, Err(error) => { return ParseProduced::from_entry(failed_entry( manifest_entry, None, OfficialParseSourceKind::DirectBundle, fingerprint, key, error, )) } }; if !UnityFsParser::has_unityfs_signature(&bytes) { return ParseProduced::from_entry(unsupported_entry( manifest_entry, None, OfficialParseSourceKind::DirectBundle, fingerprint, key, "文件不是 UnityFS bundle", )); } parsed_bundle_entry( manifest_entry, None, OfficialParseSourceKind::DirectBundle, fingerprint, key, &bytes, ) } fn parse_zip_inner_file( manifest_entry: &OfficialDownloadManifestEntry, archive_entry: String, fingerprint: OfficialParseSourceFingerprint, key: String, bytes: &[u8], ) -> ParseProduced { if !UnityFsParser::has_unityfs_signature(bytes) { return ParseProduced::from_entry(unsupported_entry( manifest_entry, Some(archive_entry), OfficialParseSourceKind::ZipEntry, fingerprint, key, "ZIP 条目不是 UnityFS bundle", )); } parsed_bundle_entry( manifest_entry, Some(archive_entry), OfficialParseSourceKind::ZipEntry, fingerprint, key, bytes, ) } fn parsed_bundle_entry( manifest_entry: &OfficialDownloadManifestEntry, archive_entry: Option, source_kind: OfficialParseSourceKind, fingerprint: OfficialParseSourceFingerprint, key: String, bytes: &[u8], ) -> ParseProduced { let parser = UnityFsParser::new(); match parser.parse(bytes) { Ok(parsed) => { let text_units = TextUnitExtractor::new().extract_bundle_with_context( &parsed, Some(&manifest_entry.destination), archive_entry.as_deref(), ); let text_unit_formats = text_units .units .iter() .filter_map(|unit| unit.context.get("format").cloned()) .collect::>() .into_iter() .collect(); let entry = OfficialParseCacheEntry { key, source_url: manifest_entry.url.clone(), destination: manifest_entry.destination.clone(), archive_entry, source_kind, fingerprint, status: OfficialParseStatus::Parsed, reused_from_previous_cache: false, unity_version: Some(parsed.unity_version), file_count: parsed.files.len(), serialized_file_count: parsed.serialized_files.len(), text_asset_count: parsed.text_assets.len(), text_assets: parsed .text_assets .iter() .map(|asset| asset.name.clone()) .collect(), serialized_parse_error_count: parsed.serialized_parse_errors.len(), text_unit_count: text_units.units.len(), text_unit_formats, skipped_binary_text_asset_count: text_units.skipped_binary_text_assets, text_unit_error_count: text_units.errors.len(), error: None, }; let units = indexed_text_units_for_entry(&entry, &text_units.units); let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors); ParseProduced { entry, units, errors, } } Err(error) => ParseProduced::from_entry(failed_entry( manifest_entry, archive_entry, source_kind, fingerprint, key, error.to_string(), )), } } fn indexed_text_units_for_entry( entry: &OfficialParseCacheEntry, units: &[TextUnit], ) -> Vec { units .iter() .enumerate() .map(|(index, unit)| OfficialTextUnitIndexUnit { id: format!("{}#unit:{index}", entry.key), parse_entry_key: entry.key.clone(), source_url: entry.source_url.clone(), destination: entry.destination.clone(), archive_entry: entry.archive_entry.clone(), source_kind: entry.source_kind, unity_version: entry.unity_version.clone(), source_text: unit.source_text.clone(), serialized_file: unit.serialized_file.clone(), path_id: unit.path_id, class_id: unit.class_id, field_path: unit.field_path.clone(), field_offset: unit.field_offset, field_byte_size: unit.field_byte_size, format: unit.context.get("format").cloned(), text_source_kind: unit.context.get("source_kind").cloned(), asset_name: unit.context.get("asset_name").cloned(), context: unit.context.clone(), }) .collect() } fn indexed_extraction_errors_for_entry( entry: &OfficialParseCacheEntry, errors: &[TextUnitExtractionError], ) -> Vec { errors .iter() .enumerate() .map(|(index, error)| OfficialTextUnitIndexError { id: format!("{}#extract-error:{index}", entry.key), parse_entry_key: entry.key.clone(), source_url: entry.source_url.clone(), destination: entry.destination.clone(), archive_entry: entry.archive_entry.clone(), source_kind: entry.source_kind, status: entry.status, serialized_file: error.serialized_file.clone(), path_id: error.path_id, class_id: error.class_id, field_path: error.field_path.clone(), offset: error.offset, error: error.error.clone(), }) .collect() } fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option { let error = entry.error.as_ref()?; Some(OfficialTextUnitIndexError { id: format!("{}#status", entry.key), parse_entry_key: entry.key.clone(), source_url: entry.source_url.clone(), destination: entry.destination.clone(), archive_entry: entry.archive_entry.clone(), source_kind: entry.source_kind, status: entry.status, serialized_file: None, path_id: None, class_id: None, field_path: None, offset: None, error: error.clone(), }) } fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool { if query .destination .as_ref() .is_some_and(|destination| &unit.destination != destination) { return false; } if query .path_pattern .as_ref() .is_some_and(|pattern| !glob_matches(pattern, &unit.destination)) { return false; } if query .archive_entry .as_ref() .is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry)) { return false; } if query .path_id .is_some_and(|path_id| unit.path_id != Some(path_id)) { return false; } if query .class_id .is_some_and(|class_id| unit.class_id != Some(class_id)) { return false; } if query .field_path .as_ref() .is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path)) { return false; } if query .format .as_ref() .is_some_and(|format| unit.format.as_ref() != Some(format)) { return false; } true } fn textunit_error_matches( error: &OfficialTextUnitIndexError, query: &OfficialTextUnitQuery, ) -> bool { if query .destination .as_ref() .is_some_and(|destination| &error.destination != destination) { return false; } if query .path_pattern .as_ref() .is_some_and(|pattern| !glob_matches(pattern, &error.destination)) { return false; } if query .archive_entry .as_ref() .is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry)) { return false; } if query .path_id .is_some_and(|path_id| error.path_id != Some(path_id)) { return false; } if query .class_id .is_some_and(|class_id| error.class_id != Some(class_id)) { return false; } if query .field_path .as_ref() .is_some_and(|field_path| error.field_path.as_ref() != Some(field_path)) { return false; } true } fn glob_matches(pattern: &str, value: &str) -> bool { glob_matches_bytes(pattern.as_bytes(), value.as_bytes()) } fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool { match pattern.split_first() { None => value.is_empty(), Some((&b'*', rest)) => { glob_matches_bytes(rest, value) || (!value.is_empty() && glob_matches_bytes(pattern, &value[1..])) } Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]), Some((&literal, rest)) => value .split_first() .is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)), } } fn failed_entry( manifest_entry: &OfficialDownloadManifestEntry, archive_entry: Option, source_kind: OfficialParseSourceKind, fingerprint: OfficialParseSourceFingerprint, key: String, error: impl Into, ) -> OfficialParseCacheEntry { status_entry( manifest_entry, archive_entry, source_kind, fingerprint, key, OfficialParseStatus::Failed, error.into(), ) } fn unsupported_entry( manifest_entry: &OfficialDownloadManifestEntry, archive_entry: Option, source_kind: OfficialParseSourceKind, fingerprint: OfficialParseSourceFingerprint, key: String, reason: impl Into, ) -> OfficialParseCacheEntry { status_entry( manifest_entry, archive_entry, source_kind, fingerprint, key, OfficialParseStatus::SkippedUnsupported, reason.into(), ) } fn status_entry( manifest_entry: &OfficialDownloadManifestEntry, archive_entry: Option, source_kind: OfficialParseSourceKind, fingerprint: OfficialParseSourceFingerprint, key: String, status: OfficialParseStatus, reason: String, ) -> OfficialParseCacheEntry { OfficialParseCacheEntry { key, source_url: manifest_entry.url.clone(), destination: manifest_entry.destination.clone(), archive_entry, source_kind, fingerprint, status, reused_from_previous_cache: false, unity_version: None, file_count: 0, serialized_file_count: 0, text_asset_count: 0, text_assets: Vec::new(), serialized_parse_error_count: 0, text_unit_count: 0, text_unit_formats: Vec::new(), skipped_binary_text_asset_count: 0, text_unit_error_count: 0, error: Some(reason), } } fn read_resource_file(path: &Path) -> Result, String> { read_file_no_symlink(path, "官方解析输入")? .ok_or_else(|| format!("官方解析输入不存在：{}", path.display())) } fn resource_path_for( resource_root: &Path, manifest_entry: &OfficialDownloadManifestEntry, ) -> Result { let path = resource_root.join(Path::new(&manifest_entry.destination)); ensure_path_within_root(resource_root, &path)?; ensure_safe_file_target(resource_root, &path, "官方解析输入")?; Ok(path) } fn list_zip_entries(unzip_command: &Path, archive_path: &Path) -> Result, String> { let output = Command::new(unzip_command) .arg("-Z1") .arg(archive_path) .output() .map_err(|error| { format!( "启动 unzip 列出 ZIP 条目失败 {}：{error}", archive_path.display() ) })?; if !output.status.success() { return Err(format!( "列出 ZIP 条目失败 {}：{}", archive_path.display(), String::from_utf8_lossy(&output.stderr).trim() )); } Ok(String::from_utf8_lossy(&output.stdout) .lines() .map(|line| line.trim_end_matches('\r').trim().to_string()) .filter(|line| !line.is_empty() && !line.ends_with('/')) .collect()) } fn read_zip_entry_bytes( unzip_command: &Path, archive_path: &Path, archive_entry: &str, ) -> Result, String> { let output = Command::new(unzip_command) .arg("-p") .arg(archive_path) .arg(archive_entry) .output() .map_err(|error| { format!( "启动 unzip 读取 ZIP 条目失败 {}!{}：{error}", archive_path.display(), archive_entry ) })?; if !output.status.success() { return Err(format!( "读取 ZIP 条目失败 {}!{}：{}", archive_path.display(), archive_entry, String::from_utf8_lossy(&output.stderr).trim() )); } Ok(output.stdout) } fn reusable_entry( previous_cache: Option<&OfficialParseCache>, key: &str, fingerprint: &OfficialParseSourceFingerprint, ) -> Option { previous_cache .and_then(|cache| cache.entries.get(key)) .filter(|entry| &entry.fingerprint == fingerprint) .cloned() } fn reusable_archive_entries( previous_cache: Option<&OfficialParseCache>, manifest_entry: &OfficialDownloadManifestEntry, fingerprint: &OfficialParseSourceFingerprint, ) -> Vec { previous_cache .into_iter() .flat_map(|cache| cache.entries.values()) .filter(|entry| { entry.source_url == manifest_entry.url && entry.destination == manifest_entry.destination && &entry.fingerprint == fingerprint }) .cloned() .collect() } fn fingerprint_for( manifest_entry: &OfficialDownloadManifestEntry, ) -> OfficialParseSourceFingerprint { OfficialParseSourceFingerprint { source_url: manifest_entry.url.clone(), destination: manifest_entry.destination.clone(), bytes: manifest_entry.bytes, blake3: manifest_entry.blake3.clone(), } } fn looks_like_zip_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool { has_case_insensitive_suffix(&manifest_entry.url, ".zip") || has_case_insensitive_suffix(&manifest_entry.destination, ".zip") } fn looks_like_direct_bundle_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool { [".bundle", ".unity3d"].iter().any(|suffix| { has_case_insensitive_suffix(&manifest_entry.url, suffix) || has_case_insensitive_suffix(&manifest_entry.destination, suffix) }) } fn has_case_insensitive_suffix(value: &str, suffix: &str) -> bool { value .rsplit(['/', '\\']) .next() .is_some_and(|name| name.to_ascii_lowercase().ends_with(suffix)) } fn direct_key(url: &str) -> String { format!("direct:{url}") } fn zip_key(url: &str, archive_entry: &str) -> String { format!("zip:{url}!{archive_entry}") } fn zip_list_key(url: &str) -> String { format!("zip-list:{url}") } fn unsupported_key(url: &str) -> String { format!("unsupported:{url}") } fn default_parse_cache_version() -> u32 { OFFICIAL_PARSE_CACHE_VERSION } fn default_textunit_index_version() -> u32 { OFFICIAL_TEXTUNIT_INDEX_VERSION } fn unix_seconds_now() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs() } #[cfg(test)] mod tests { use super::*; use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; fn push_c_string(data: &mut Vec, value: &str) { data.extend_from_slice(value.as_bytes()); data.push(0); } fn push_u16(data: &mut Vec, value: u16) { data.extend_from_slice(&value.to_be_bytes()); } fn push_u32(data: &mut Vec, value: u32) { data.extend_from_slice(&value.to_be_bytes()); } fn push_i32(data: &mut Vec, value: i32) { data.extend_from_slice(&value.to_be_bytes()); } fn push_u64(data: &mut Vec, value: u64) { data.extend_from_slice(&value.to_be_bytes()); } fn align(data: &mut Vec, alignment: usize) { let remainder = data.len() % alignment; if remainder != 0 { data.resize(data.len() + alignment - remainder, 0); } } fn synthetic_unityfs_bundle(directory_path: &str, payload: &[u8]) -> Vec { let mut blocks_info = Vec::new(); blocks_info.extend_from_slice(&[0xAB; 16]); push_i32(&mut blocks_info, 1); push_u32(&mut blocks_info, payload.len() as u32); push_u32(&mut blocks_info, payload.len() as u32); push_u16(&mut blocks_info, 0); push_i32(&mut blocks_info, 1); push_u64(&mut blocks_info, 0); push_u64(&mut blocks_info, payload.len() as u64); push_u32(&mut blocks_info, 0); push_c_string(&mut blocks_info, directory_path); let mut data = Vec::new(); push_c_string(&mut data, "UnityFS"); push_u32(&mut data, 8); push_c_string(&mut data, "5.x.x"); push_c_string(&mut data, "2021.3.56f2"); push_u64(&mut data, 0); push_u32(&mut data, blocks_info.len() as u32); push_u32(&mut data, blocks_info.len() as u32); push_u32(&mut data, 0); align(&mut data, 16); data.extend_from_slice(&blocks_info); data.extend_from_slice(payload); let total_size = data.len() as u64; let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); data } fn write_download_manifest( root: &Path, entries: &[(&str, &str, &[u8])], ) -> Vec { let mut manifest_entries = BTreeMap::new(); let mut written = Vec::new(); for (url, destination, bytes) in entries { let path = root.join(destination); fs::create_dir_all(path.parent().unwrap()).unwrap(); fs::write(&path, bytes).unwrap(); let entry = OfficialDownloadManifestEntry { url: (*url).to_string(), destination: (*destination).to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }; manifest_entries.insert((*url).to_string(), entry.clone()); written.push(entry); } let manifest = serde_json::json!({ "version": 1, "entries": manifest_entries, }); fs::write( root.join("official-download-manifest.json"), serde_json::to_vec_pretty(&manifest).unwrap(), ) .unwrap(); written } #[test] fn parses_direct_bundle_and_reuses_unchanged_cache() { let temp = tempfile::TempDir::new().unwrap(); let root = temp.path(); let bundle = synthetic_unityfs_bundle("CAB-fixture", b"data"); write_download_manifest( root, &[( "https://prod-clientpatch.bluearchiveyostar.com/r93/Bundle/test.bundle", "Bundle/test.bundle", &bundle, )], ); let config = OfficialParseConfig::new(root, "unzip"); let first = OfficialParseCacheService::new().run(&config).unwrap(); assert_eq!(first.summary.manifest_entry_count, 1); assert_eq!(first.summary.parsed_bundle_count, 1); assert_eq!(first.summary.skipped_unchanged_count, 0); assert!(first.cache_path.exists()); let second = OfficialParseCacheService::new().run(&config).unwrap(); assert_eq!(second.summary.parsed_bundle_count, 1); assert_eq!(second.summary.skipped_unchanged_count, 1); let cache = read_parse_cache_at(root).unwrap().unwrap(); let entry = cache.entries.values().next().unwrap(); assert_eq!(entry.status, OfficialParseStatus::Parsed); assert_eq!(entry.file_count, 1); } #[test] fn records_non_candidate_resources_as_unsupported() { let temp = tempfile::TempDir::new().unwrap(); let root = temp.path(); write_download_manifest( root, &[( "https://prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/TableCatalog.bytes", "TableBundles/TableCatalog.bytes", b"catalog", )], ); let config = OfficialParseConfig::new(root, "unzip"); let report = OfficialParseCacheService::new().run(&config).unwrap(); assert_eq!(report.summary.parsed_bundle_count, 0); assert_eq!(report.summary.unsupported_count, 1); assert_eq!(report.summary.failed_count, 0); } #[cfg(unix)] #[test] fn parses_zip_entries_without_extracting_archive_tree() { let temp = tempfile::TempDir::new().unwrap(); let root = temp.path(); let bundle = synthetic_unityfs_bundle("CAB-zipped", b"zipdata"); let bundle_path = temp.path().join("inner.bundle"); fs::write(&bundle_path, &bundle).unwrap(); let archive_bytes = b"zip-placeholder"; write_download_manifest( root, &[( "https://prod-clientpatch.bluearchiveyostar.com/r93/Windows_PatchPack/FullPatch_000.zip", "Windows_PatchPack/FullPatch_000.zip", archive_bytes, )], ); let unzip_script = temp.path().join("fake-unzip"); fs::write( &unzip_script, format!( r#"#!/usr/bin/env bash set -euo pipefail if [[ "${{1:-}}" == "-Z1" ]]; then printf '%s\n' 'assets/scenario.bundle' 'assets/readme.txt' exit 0 fi if [[ "${{1:-}}" == "-p" && "${{3:-}}" == "assets/scenario.bundle" ]]; then cat '{}' exit 0 fi if [[ "${{1:-}}" == "-p" ]]; then printf 'plain text' exit 0 fi exit 2 "#, bundle_path.display() ), ) .unwrap(); let mut permissions = fs::metadata(&unzip_script).unwrap().permissions(); permissions.set_mode(0o755); fs::set_permissions(&unzip_script, permissions).unwrap(); let config = OfficialParseConfig::new(root, unzip_script); let report = OfficialParseCacheService::new().run(&config).unwrap(); assert_eq!(report.summary.manifest_entry_count, 1); assert_eq!(report.summary.zip_entry_count, 2); assert_eq!(report.summary.parsed_bundle_count, 1); assert_eq!(report.summary.unsupported_count, 1); assert_eq!(report.summary.failed_count, 0); assert!(!root.join("assets").exists()); } }