Files
BlueArchiveToolkit/infrastructure/src/official_parse.rs
T
nyaKazuha 2079c6a307 feat(sync): 接入解析缓存与汉化发布前置
补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
2026-07-31 00:38:45 +08:00

1473 lines
50 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Official resource post-sync parse cache.
//!
//! The parser runs after official download verification has completed. It reads
//! the immutable published resource tree, writes a small derived cache next to
//! `official-download-manifest.json`, and leaves localized output generation to
//! later patch/export stages.
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use bat_assetbundle::{
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
/// Parse-cache file name stored under a published official resource root.
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
/// Current parse-cache schema version.
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
/// TextUnit detail index file name stored under a published official resource root.
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
/// Current TextUnit detail-index schema version.
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
/// Configuration for one official resource parse-cache refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialParseConfig {
/// Published official resource root containing `official-download-manifest.json`.
pub resource_root: PathBuf,
/// `unzip` executable used to inspect zip archives without extracting them.
pub unzip_command: PathBuf,
}
impl OfficialParseConfig {
/// Creates parse-cache configuration for a published official resource root.
pub fn new(resource_root: impl Into<PathBuf>, unzip_command: impl Into<PathBuf>) -> Self {
Self {
resource_root: resource_root.into(),
unzip_command: unzip_command.into(),
}
}
/// Returns the parse-cache path for this resource root.
pub fn cache_path(&self) -> PathBuf {
self.resource_root.join(OFFICIAL_PARSE_CACHE_FILE)
}
}
/// Structured report returned by a parse-cache refresh.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialParseReport {
/// Official resource root that was inspected.
pub resource_root: PathBuf,
/// Parse-cache path written by the refresh.
pub cache_path: PathBuf,
/// Aggregate parse-cache summary.
pub summary: OfficialParseSummary,
/// TextUnit detail-index path written by the refresh.
pub textunit_index_path: PathBuf,
/// Aggregate TextUnit detail-index summary.
pub textunit_index_summary: OfficialTextUnitIndexSummary,
}
/// Aggregate counters for a parse-cache refresh.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialParseSummary {
/// Number of entries in `official-download-manifest.json`.
pub manifest_entry_count: usize,
/// Number of entries stored in `official-parse-cache.json`.
pub cache_entry_count: usize,
/// Number of direct bundle files or zip inner files inspected as parser candidates.
pub candidate_file_count: usize,
/// Number of zip inner file entries inspected.
pub zip_entry_count: usize,
/// Number of cached entries reused because source URL/path/size/BLAKE3 did not change.
pub skipped_unchanged_count: usize,
/// Number of bundles parsed successfully.
pub parsed_bundle_count: usize,
/// Number of files intentionally skipped because they are not UnityFS bundles.
pub unsupported_count: usize,
/// Number of files or archives that failed parser/cache inspection.
pub failed_count: usize,
/// Total TextAsset objects found in parsed Unity serialized files.
pub text_asset_count: usize,
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
#[serde(default)]
pub text_unit_count: usize,
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Number of non-fatal TypeTree field extraction diagnostics.
#[serde(default)]
pub text_unit_error_count: usize,
}
/// Persistent parse cache for one official resource root.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialParseCache {
/// Cache schema version.
#[serde(default = "default_parse_cache_version")]
pub version: u32,
/// Cache generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Aggregate summary for this cache.
pub summary: OfficialParseSummary,
/// Cache entries keyed by source URL and optional archive entry.
#[serde(default)]
pub entries: BTreeMap<String, OfficialParseCacheEntry>,
}
/// One parsed or skipped official resource file.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialParseCacheEntry {
/// Stable cache key.
pub key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Fingerprint derived from the download manifest.
pub fingerprint: OfficialParseSourceFingerprint,
/// Parse status for this cache entry.
pub status: OfficialParseStatus,
/// Whether this entry was reused from the previous parse cache in this run.
#[serde(default)]
pub reused_from_previous_cache: bool,
/// Unity editor version when a UnityFS bundle was parsed.
pub unity_version: Option<String>,
/// Number of files extracted from the UnityFS directory table.
pub file_count: usize,
/// Number of Unity serialized files parsed from extracted files.
pub serialized_file_count: usize,
/// Number of TextAsset objects found.
pub text_asset_count: usize,
/// TextAsset names found in stable order.
pub text_assets: Vec<String>,
/// Non-fatal serialized-file parse diagnostic count.
pub serialized_parse_error_count: usize,
/// Number of translation-ready TextUnit entries extracted from this bundle.
#[serde(default)]
pub text_unit_count: usize,
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
#[serde(default)]
pub text_unit_formats: Vec<String>,
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Non-fatal TypeTree field extraction diagnostic count.
#[serde(default)]
pub text_unit_error_count: usize,
/// Human-readable error or skip reason.
pub error: Option<String>,
}
/// Persistent TextUnit detail index for one official resource root.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndex {
/// Index schema version.
#[serde(default = "default_textunit_index_version")]
pub version: u32,
/// Index generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Official resource root used to generate this index.
pub resource_root: PathBuf,
/// Aggregate counters for this index.
pub summary: OfficialTextUnitIndexSummary,
/// Extracted TextUnit details in deterministic order.
#[serde(default)]
pub units: Vec<OfficialTextUnitIndexUnit>,
/// Parse and extraction diagnostics in deterministic order.
#[serde(default)]
pub errors: Vec<OfficialTextUnitIndexError>,
}
/// Aggregate counters for a TextUnit detail index.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexSummary {
/// Number of TextUnit detail entries.
pub unit_count: usize,
/// Number of parse/extraction diagnostics.
pub error_count: usize,
/// Number of binary or invalid TextAsset payloads skipped by extraction.
pub skipped_binary_text_asset_count: usize,
}
/// One TextUnit detail entry.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexUnit {
/// Stable unit ID within the official release.
pub id: String,
/// Parse-cache entry key that produced this unit.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Unity editor version when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unity_version: Option<String>,
/// Original source text.
pub source_text: String,
/// Unity serialized file path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// TextUnit payload format such as json/csv/tsv/plain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
/// Extraction source kind such as TextAsset or TypeTreeField.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_source_kind: Option<String>,
/// TextAsset name when the unit came from a TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset_name: Option<String>,
/// Stable extraction context copied from the parser.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub context: BTreeMap<String, String>,
}
/// One parse or extraction diagnostic in the TextUnit detail index.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexError {
/// Stable diagnostic ID within the official release.
pub id: String,
/// Parse-cache entry key associated with this diagnostic.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Parse status associated with this diagnostic.
pub status: OfficialParseStatus,
/// Unity serialized file path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable diagnostic.
pub error: String,
}
/// Query filters for TextUnit detail entries.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialTextUnitQuery {
/// Filter by destination path.
pub destination: Option<String>,
/// Filter by destination glob pattern.
pub path_pattern: Option<String>,
/// Filter by archive entry.
pub archive_entry: Option<String>,
/// Filter by Unity object path ID.
pub path_id: Option<i64>,
/// Filter by Unity class ID.
pub class_id: Option<i32>,
/// Filter by field path.
pub field_path: Option<String>,
/// Filter by TextUnit format.
pub format: Option<String>,
}
/// Source kind for a parse-cache entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialParseSourceKind {
/// A direct official resource file that should be a UnityFS bundle.
DirectBundle,
/// A file entry streamed from an official zip archive.
ZipEntry,
/// A manifest entry that is not expected to contain a UnityFS bundle.
Unsupported,
}
/// Stable fingerprint copied from the official download manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialParseSourceFingerprint {
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// File byte count from the verified download manifest.
pub bytes: u64,
/// BLAKE3 digest from the verified download manifest.
pub blake3: String,
}
/// Parse status for one official resource cache entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialParseStatus {
/// UnityFS parsing succeeded.
Parsed,
/// The source was inspected and is not currently parsed as UnityFS.
SkippedUnsupported,
/// The source was expected to be parseable but failed inspection.
Failed,
}
/// Service that refreshes official parse caches.
#[derive(Debug, Default, Clone, Copy)]
pub struct OfficialParseCacheService;
impl OfficialParseCacheService {
/// Creates a parse-cache service.
pub fn new() -> Self {
Self
}
/// Refreshes the parse cache for the configured official resource root.
pub fn run(&self, config: &OfficialParseConfig) -> Result<OfficialParseReport, String> {
let manifest = read_download_manifest_at(&config.resource_root)?.ok_or_else(|| {
format!(
"缺少官方下载 manifest,无法更新解析缓存:{}",
config.resource_root.display()
)
})?;
let previous_cache = read_parse_cache_at(&config.resource_root)?;
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
let mut summary = OfficialParseSummary {
manifest_entry_count: manifest.entries.len(),
..OfficialParseSummary::default()
};
let mut entries = BTreeMap::new();
let mut units = Vec::new();
let mut errors = Vec::new();
for manifest_entry in manifest.entries.values() {
let produced = process_manifest_entry(
config,
manifest_entry,
previous_cache.as_ref(),
previous_textunit_index.as_ref(),
);
for produced in produced {
summary.record_entry(&produced.entry);
units.extend(produced.units);
errors.extend(produced.errors);
entries.insert(produced.entry.key.clone(), produced.entry);
}
}
summary.cache_entry_count = entries.len();
let textunit_index_summary = OfficialTextUnitIndexSummary {
unit_count: units.len(),
error_count: errors.len(),
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: unix_seconds_now(),
summary: summary.clone(),
entries,
};
write_parse_cache_at(&config.resource_root, &cache)?;
let textunit_index = OfficialTextUnitIndex {
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
generated_unix_seconds: cache.generated_unix_seconds,
resource_root: config.resource_root.clone(),
summary: textunit_index_summary.clone(),
units,
errors,
};
write_textunit_index_at(&config.resource_root, &textunit_index)?;
Ok(OfficialParseReport {
resource_root: config.resource_root.clone(),
cache_path: config.cache_path(),
summary,
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
textunit_index_summary,
})
}
}
struct ParseProduced {
entry: OfficialParseCacheEntry,
units: Vec<OfficialTextUnitIndexUnit>,
errors: Vec<OfficialTextUnitIndexError>,
}
impl ParseProduced {
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
let errors = error_from_status_entry(&entry).into_iter().collect();
Self {
entry,
units: Vec::new(),
errors,
}
}
fn from_cached(
mut entry: OfficialParseCacheEntry,
previous_index: Option<&OfficialTextUnitIndex>,
) -> Self {
entry.reused_from_previous_cache = true;
let units = previous_index
.into_iter()
.flat_map(|index| index.units.iter())
.filter(|unit| unit.parse_entry_key == entry.key)
.cloned()
.collect();
let errors = previous_index
.into_iter()
.flat_map(|index| index.errors.iter())
.filter(|error| error.parse_entry_key == entry.key)
.cloned()
.collect();
Self {
entry,
units,
errors,
}
}
}
impl OfficialParseSummary {
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
match entry.source_kind {
OfficialParseSourceKind::DirectBundle => {
self.candidate_file_count += 1;
}
OfficialParseSourceKind::ZipEntry => {
self.candidate_file_count += 1;
self.zip_entry_count += 1;
}
OfficialParseSourceKind::Unsupported => {}
}
match entry.status {
OfficialParseStatus::Parsed => {
self.parsed_bundle_count += 1;
self.text_asset_count += entry.text_asset_count;
self.text_unit_count += entry.text_unit_count;
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
self.text_unit_error_count += entry.text_unit_error_count;
}
OfficialParseStatus::SkippedUnsupported => {
self.unsupported_count += 1;
}
OfficialParseStatus::Failed => {
self.failed_count += 1;
}
}
if entry.reused_from_previous_cache {
self.skipped_unchanged_count += 1;
}
}
}
/// Reads the parse cache under a published official resource root.
///
/// Missing, corrupt, or unsupported-version caches return `Ok(None)` so a new
/// cache can be regenerated without blocking official resource publication.
pub fn read_parse_cache_at(resource_root: &Path) -> Result<Option<OfficialParseCache>, String> {
let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方解析缓存")? else {
return Ok(None);
};
let Ok(cache) = serde_json::from_slice::<OfficialParseCache>(&bytes) else {
return Ok(None);
};
if cache.version != OFFICIAL_PARSE_CACHE_VERSION {
return Ok(None);
}
Ok(Some(cache))
}
/// Writes the parse cache under a published official resource root.
pub fn write_parse_cache_at(
resource_root: &Path,
cache: &OfficialParseCache,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方解析缓存")?;
let bytes = serde_json::to_vec_pretty(cache)
.map_err(|error| format!("序列化官方解析缓存失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
}
/// Reads the TextUnit detail index under a published official resource root.
pub fn read_textunit_index_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitIndex>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
return Ok(None);
};
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
return Ok(None);
};
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
return Ok(None);
}
Ok(Some(index))
}
/// Writes the TextUnit detail index under a published official resource root.
pub fn write_textunit_index_at(
resource_root: &Path,
index: &OfficialTextUnitIndex,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
let bytes = serde_json::to_vec_pretty(index)
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
}
/// Returns TextUnit detail entries matching a query.
pub fn query_textunit_index_units<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexUnit> {
index
.units
.iter()
.filter(|unit| textunit_unit_matches(unit, query))
.collect()
}
/// Returns parse/extraction diagnostics matching a query.
pub fn query_textunit_index_errors<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexError> {
index
.errors
.iter()
.filter(|error| textunit_error_matches(error, query))
.collect()
}
fn process_manifest_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
previous_index: Option<&OfficialTextUnitIndex>,
) -> Vec<ParseProduced> {
let fingerprint = fingerprint_for(manifest_entry);
if looks_like_zip_source(manifest_entry) {
return process_zip_entry(
config,
manifest_entry,
previous_cache,
previous_index,
fingerprint,
);
}
if looks_like_direct_bundle_source(manifest_entry) {
let key = direct_key(&manifest_entry.url);
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
return vec![parse_direct_bundle(
config,
manifest_entry,
key,
fingerprint,
)];
}
let key = unsupported_key(&manifest_entry.url);
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
vec![ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
key,
"非 UnityFS 候选资源",
))]
}
fn process_zip_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
previous_index: Option<&OfficialTextUnitIndex>,
fingerprint: OfficialParseSourceFingerprint,
) -> Vec<ParseProduced> {
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
if !cached_entries.is_empty() {
return cached_entries
.into_iter()
.map(|entry| ParseProduced::from_cached(entry, previous_index))
.collect();
}
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
))]
}
};
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
Ok(entries) => entries,
Err(error) => {
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
))]
}
};
let mut produced = Vec::new();
for archive_entry in archive_entries {
let key = zip_key(&manifest_entry.url, &archive_entry);
let bytes = match read_zip_entry_bytes(&config.unzip_command, &archive_path, &archive_entry)
{
Ok(bytes) => bytes,
Err(error) => {
produced.push(ParseProduced::from_entry(failed_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint.clone(),
key,
error,
)));
continue;
}
};
produced.push(parse_zip_inner_file(
manifest_entry,
archive_entry,
fingerprint.clone(),
key,
&bytes,
));
}
if produced.is_empty() {
produced.push(ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
zip_list_key(&manifest_entry.url),
"ZIP 内没有可检查文件条目",
)));
}
produced
}
fn parse_direct_bundle(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
key: String,
fingerprint: OfficialParseSourceFingerprint,
) -> ParseProduced {
let path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
))
}
};
let bytes = match read_resource_file(&path) {
Ok(bytes) => bytes,
Err(error) => {
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
))
}
};
if !UnityFsParser::has_unityfs_signature(&bytes) {
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
"文件不是 UnityFS bundle",
));
}
parsed_bundle_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
&bytes,
)
}
fn parse_zip_inner_file(
manifest_entry: &OfficialDownloadManifestEntry,
archive_entry: String,
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> ParseProduced {
if !UnityFsParser::has_unityfs_signature(bytes) {
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint,
key,
"ZIP 条目不是 UnityFS bundle",
));
}
parsed_bundle_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint,
key,
bytes,
)
}
fn parsed_bundle_entry(
manifest_entry: &OfficialDownloadManifestEntry,
archive_entry: Option<String>,
source_kind: OfficialParseSourceKind,
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> ParseProduced {
let parser = UnityFsParser::new();
match parser.parse(bytes) {
Ok(parsed) => {
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
&parsed,
Some(&manifest_entry.destination),
archive_entry.as_deref(),
);
let text_unit_formats = text_units
.units
.iter()
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let entry = OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some(parsed.unity_version),
file_count: parsed.files.len(),
serialized_file_count: parsed.serialized_files.len(),
text_asset_count: parsed.text_assets.len(),
text_assets: parsed
.text_assets
.iter()
.map(|asset| asset.name.clone())
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
text_unit_error_count: text_units.errors.len(),
error: None,
};
let units = indexed_text_units_for_entry(&entry, &text_units.units);
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
ParseProduced {
entry,
units,
errors,
}
}
Err(error) => ParseProduced::from_entry(failed_entry(
manifest_entry,
archive_entry,
source_kind,
fingerprint,
key,
error.to_string(),
)),
}
}
fn indexed_text_units_for_entry(
entry: &OfficialParseCacheEntry,
units: &[TextUnit],
) -> Vec<OfficialTextUnitIndexUnit> {
units
.iter()
.enumerate()
.map(|(index, unit)| OfficialTextUnitIndexUnit {
id: format!("{}#unit:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
unity_version: entry.unity_version.clone(),
source_text: unit.source_text.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
field_offset: unit.field_offset,
field_byte_size: unit.field_byte_size,
format: unit.context.get("format").cloned(),
text_source_kind: unit.context.get("source_kind").cloned(),
asset_name: unit.context.get("asset_name").cloned(),
context: unit.context.clone(),
})
.collect()
}
fn indexed_extraction_errors_for_entry(
entry: &OfficialParseCacheEntry,
errors: &[TextUnitExtractionError],
) -> Vec<OfficialTextUnitIndexError> {
errors
.iter()
.enumerate()
.map(|(index, error)| OfficialTextUnitIndexError {
id: format!("{}#extract-error:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: error.serialized_file.clone(),
path_id: error.path_id,
class_id: error.class_id,
field_path: error.field_path.clone(),
offset: error.offset,
error: error.error.clone(),
})
.collect()
}
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
let error = entry.error.as_ref()?;
Some(OfficialTextUnitIndexError {
id: format!("{}#status", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: None,
path_id: None,
class_id: None,
field_path: None,
offset: None,
error: error.clone(),
})
}
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &unit.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| unit.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| unit.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
{
return false;
}
if query
.format
.as_ref()
.is_some_and(|format| unit.format.as_ref() != Some(format))
{
return false;
}
true
}
fn textunit_error_matches(
error: &OfficialTextUnitIndexError,
query: &OfficialTextUnitQuery,
) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &error.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| error.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| error.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
{
return false;
}
true
}
fn glob_matches(pattern: &str, value: &str) -> bool {
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
}
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
match pattern.split_first() {
None => value.is_empty(),
Some((&b'*', rest)) => {
glob_matches_bytes(rest, value)
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
}
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
Some((&literal, rest)) => value
.split_first()
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
}
}
fn failed_entry(
manifest_entry: &OfficialDownloadManifestEntry,
archive_entry: Option<String>,
source_kind: OfficialParseSourceKind,
fingerprint: OfficialParseSourceFingerprint,
key: String,
error: impl Into<String>,
) -> OfficialParseCacheEntry {
status_entry(
manifest_entry,
archive_entry,
source_kind,
fingerprint,
key,
OfficialParseStatus::Failed,
error.into(),
)
}
fn unsupported_entry(
manifest_entry: &OfficialDownloadManifestEntry,
archive_entry: Option<String>,
source_kind: OfficialParseSourceKind,
fingerprint: OfficialParseSourceFingerprint,
key: String,
reason: impl Into<String>,
) -> OfficialParseCacheEntry {
status_entry(
manifest_entry,
archive_entry,
source_kind,
fingerprint,
key,
OfficialParseStatus::SkippedUnsupported,
reason.into(),
)
}
fn status_entry(
manifest_entry: &OfficialDownloadManifestEntry,
archive_entry: Option<String>,
source_kind: OfficialParseSourceKind,
fingerprint: OfficialParseSourceFingerprint,
key: String,
status: OfficialParseStatus,
reason: String,
) -> OfficialParseCacheEntry {
OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status,
reused_from_previous_cache: false,
unity_version: None,
file_count: 0,
serialized_file_count: 0,
text_asset_count: 0,
text_assets: Vec::new(),
serialized_parse_error_count: 0,
text_unit_count: 0,
text_unit_formats: Vec::new(),
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: Some(reason),
}
}
fn read_resource_file(path: &Path) -> Result<Vec<u8>, String> {
read_file_no_symlink(path, "官方解析输入")?
.ok_or_else(|| format!("官方解析输入不存在:{}", path.display()))
}
fn resource_path_for(
resource_root: &Path,
manifest_entry: &OfficialDownloadManifestEntry,
) -> Result<PathBuf, String> {
let path = resource_root.join(Path::new(&manifest_entry.destination));
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方解析输入")?;
Ok(path)
}
fn list_zip_entries(unzip_command: &Path, archive_path: &Path) -> Result<Vec<String>, String> {
let output = Command::new(unzip_command)
.arg("-Z1")
.arg(archive_path)
.output()
.map_err(|error| {
format!(
"启动 unzip 列出 ZIP 条目失败 {}{error}",
archive_path.display()
)
})?;
if !output.status.success() {
return Err(format!(
"列出 ZIP 条目失败 {}{}",
archive_path.display(),
String::from_utf8_lossy(&output.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&output.stdout)
.lines()
.map(|line| line.trim_end_matches('\r').trim().to_string())
.filter(|line| !line.is_empty() && !line.ends_with('/'))
.collect())
}
fn read_zip_entry_bytes(
unzip_command: &Path,
archive_path: &Path,
archive_entry: &str,
) -> Result<Vec<u8>, String> {
let output = Command::new(unzip_command)
.arg("-p")
.arg(archive_path)
.arg(archive_entry)
.output()
.map_err(|error| {
format!(
"启动 unzip 读取 ZIP 条目失败 {}!{}{error}",
archive_path.display(),
archive_entry
)
})?;
if !output.status.success() {
return Err(format!(
"读取 ZIP 条目失败 {}!{}{}",
archive_path.display(),
archive_entry,
String::from_utf8_lossy(&output.stderr).trim()
));
}
Ok(output.stdout)
}
fn reusable_entry(
previous_cache: Option<&OfficialParseCache>,
key: &str,
fingerprint: &OfficialParseSourceFingerprint,
) -> Option<OfficialParseCacheEntry> {
previous_cache
.and_then(|cache| cache.entries.get(key))
.filter(|entry| &entry.fingerprint == fingerprint)
.cloned()
}
fn reusable_archive_entries(
previous_cache: Option<&OfficialParseCache>,
manifest_entry: &OfficialDownloadManifestEntry,
fingerprint: &OfficialParseSourceFingerprint,
) -> Vec<OfficialParseCacheEntry> {
previous_cache
.into_iter()
.flat_map(|cache| cache.entries.values())
.filter(|entry| {
entry.source_url == manifest_entry.url
&& entry.destination == manifest_entry.destination
&& &entry.fingerprint == fingerprint
})
.cloned()
.collect()
}
fn fingerprint_for(
manifest_entry: &OfficialDownloadManifestEntry,
) -> OfficialParseSourceFingerprint {
OfficialParseSourceFingerprint {
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
bytes: manifest_entry.bytes,
blake3: manifest_entry.blake3.clone(),
}
}
fn looks_like_zip_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool {
has_case_insensitive_suffix(&manifest_entry.url, ".zip")
|| has_case_insensitive_suffix(&manifest_entry.destination, ".zip")
}
fn looks_like_direct_bundle_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool {
[".bundle", ".unity3d"].iter().any(|suffix| {
has_case_insensitive_suffix(&manifest_entry.url, suffix)
|| has_case_insensitive_suffix(&manifest_entry.destination, suffix)
})
}
fn has_case_insensitive_suffix(value: &str, suffix: &str) -> bool {
value
.rsplit(['/', '\\'])
.next()
.is_some_and(|name| name.to_ascii_lowercase().ends_with(suffix))
}
fn direct_key(url: &str) -> String {
format!("direct:{url}")
}
fn zip_key(url: &str, archive_entry: &str) -> String {
format!("zip:{url}!{archive_entry}")
}
fn zip_list_key(url: &str) -> String {
format!("zip-list:{url}")
}
fn unsupported_key(url: &str) -> String {
format!("unsupported:{url}")
}
fn default_parse_cache_version() -> u32 {
OFFICIAL_PARSE_CACHE_VERSION
}
fn default_textunit_index_version() -> u32 {
OFFICIAL_TEXTUNIT_INDEX_VERSION
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
fn push_c_string(data: &mut Vec<u8>, value: &str) {
data.extend_from_slice(value.as_bytes());
data.push(0);
}
fn push_u16(data: &mut Vec<u8>, value: u16) {
data.extend_from_slice(&value.to_be_bytes());
}
fn push_u32(data: &mut Vec<u8>, value: u32) {
data.extend_from_slice(&value.to_be_bytes());
}
fn push_i32(data: &mut Vec<u8>, value: i32) {
data.extend_from_slice(&value.to_be_bytes());
}
fn push_u64(data: &mut Vec<u8>, value: u64) {
data.extend_from_slice(&value.to_be_bytes());
}
fn align(data: &mut Vec<u8>, alignment: usize) {
let remainder = data.len() % alignment;
if remainder != 0 {
data.resize(data.len() + alignment - remainder, 0);
}
}
fn synthetic_unityfs_bundle(directory_path: &str, payload: &[u8]) -> Vec<u8> {
let mut blocks_info = Vec::new();
blocks_info.extend_from_slice(&[0xAB; 16]);
push_i32(&mut blocks_info, 1);
push_u32(&mut blocks_info, payload.len() as u32);
push_u32(&mut blocks_info, payload.len() as u32);
push_u16(&mut blocks_info, 0);
push_i32(&mut blocks_info, 1);
push_u64(&mut blocks_info, 0);
push_u64(&mut blocks_info, payload.len() as u64);
push_u32(&mut blocks_info, 0);
push_c_string(&mut blocks_info, directory_path);
let mut data = Vec::new();
push_c_string(&mut data, "UnityFS");
push_u32(&mut data, 8);
push_c_string(&mut data, "5.x.x");
push_c_string(&mut data, "2021.3.56f2");
push_u64(&mut data, 0);
push_u32(&mut data, blocks_info.len() as u32);
push_u32(&mut data, blocks_info.len() as u32);
push_u32(&mut data, 0);
align(&mut data, 16);
data.extend_from_slice(&blocks_info);
data.extend_from_slice(payload);
let total_size = data.len() as u64;
let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len();
data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes());
data
}
fn write_download_manifest(
root: &Path,
entries: &[(&str, &str, &[u8])],
) -> Vec<OfficialDownloadManifestEntry> {
let mut manifest_entries = BTreeMap::new();
let mut written = Vec::new();
for (url, destination, bytes) in entries {
let path = root.join(destination);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(&path, bytes).unwrap();
let entry = OfficialDownloadManifestEntry {
url: (*url).to_string(),
destination: (*destination).to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
};
manifest_entries.insert((*url).to_string(), entry.clone());
written.push(entry);
}
let manifest = serde_json::json!({
"version": 1,
"entries": manifest_entries,
});
fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec_pretty(&manifest).unwrap(),
)
.unwrap();
written
}
#[test]
fn parses_direct_bundle_and_reuses_unchanged_cache() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
let bundle = synthetic_unityfs_bundle("CAB-fixture", b"data");
write_download_manifest(
root,
&[(
"https://prod-clientpatch.bluearchiveyostar.com/r93/Bundle/test.bundle",
"Bundle/test.bundle",
&bundle,
)],
);
let config = OfficialParseConfig::new(root, "unzip");
let first = OfficialParseCacheService::new().run(&config).unwrap();
assert_eq!(first.summary.manifest_entry_count, 1);
assert_eq!(first.summary.parsed_bundle_count, 1);
assert_eq!(first.summary.skipped_unchanged_count, 0);
assert!(first.cache_path.exists());
let second = OfficialParseCacheService::new().run(&config).unwrap();
assert_eq!(second.summary.parsed_bundle_count, 1);
assert_eq!(second.summary.skipped_unchanged_count, 1);
let cache = read_parse_cache_at(root).unwrap().unwrap();
let entry = cache.entries.values().next().unwrap();
assert_eq!(entry.status, OfficialParseStatus::Parsed);
assert_eq!(entry.file_count, 1);
}
#[test]
fn records_non_candidate_resources_as_unsupported() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
write_download_manifest(
root,
&[(
"https://prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/TableCatalog.bytes",
"TableBundles/TableCatalog.bytes",
b"catalog",
)],
);
let config = OfficialParseConfig::new(root, "unzip");
let report = OfficialParseCacheService::new().run(&config).unwrap();
assert_eq!(report.summary.parsed_bundle_count, 0);
assert_eq!(report.summary.unsupported_count, 1);
assert_eq!(report.summary.failed_count, 0);
}
#[cfg(unix)]
#[test]
fn parses_zip_entries_without_extracting_archive_tree() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
let bundle = synthetic_unityfs_bundle("CAB-zipped", b"zipdata");
let bundle_path = temp.path().join("inner.bundle");
fs::write(&bundle_path, &bundle).unwrap();
let archive_bytes = b"zip-placeholder";
write_download_manifest(
root,
&[(
"https://prod-clientpatch.bluearchiveyostar.com/r93/Windows_PatchPack/FullPatch_000.zip",
"Windows_PatchPack/FullPatch_000.zip",
archive_bytes,
)],
);
let unzip_script = temp.path().join("fake-unzip");
fs::write(
&unzip_script,
format!(
r#"#!/usr/bin/env bash
set -euo pipefail
if [[ "${{1:-}}" == "-Z1" ]]; then
printf '%s\n' 'assets/scenario.bundle' 'assets/readme.txt'
exit 0
fi
if [[ "${{1:-}}" == "-p" && "${{3:-}}" == "assets/scenario.bundle" ]]; then
cat '{}'
exit 0
fi
if [[ "${{1:-}}" == "-p" ]]; then
printf 'plain text'
exit 0
fi
exit 2
"#,
bundle_path.display()
),
)
.unwrap();
let mut permissions = fs::metadata(&unzip_script).unwrap().permissions();
permissions.set_mode(0o755);
fs::set_permissions(&unzip_script, permissions).unwrap();
let config = OfficialParseConfig::new(root, unzip_script);
let report = OfficialParseCacheService::new().run(&config).unwrap();
assert_eq!(report.summary.manifest_entry_count, 1);
assert_eq!(report.summary.zip_entry_count, 2);
assert_eq!(report.summary.parsed_bundle_count, 1);
assert_eq!(report.summary.unsupported_count, 1);
assert_eq!(report.summary.failed_count, 0);
assert!(!root.join("assets").exists());
}
}