mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 10:54:55 +08:00
补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。 同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。 验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
1473 lines
50 KiB
Rust
1473 lines
50 KiB
Rust
//! Official resource post-sync parse cache.
|
||
//!
|
||
//! The parser runs after official download verification has completed. It reads
|
||
//! the immutable published resource tree, writes a small derived cache next to
|
||
//! `official-download-manifest.json`, and leaves localized output generation to
|
||
//! later patch/export stages.
|
||
|
||
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
|
||
use crate::path_security::{
|
||
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
||
STATE_FILE_MODE,
|
||
};
|
||
use bat_assetbundle::{
|
||
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
|
||
};
|
||
use serde::{Deserialize, Serialize};
|
||
use std::collections::{BTreeMap, BTreeSet};
|
||
use std::path::{Path, PathBuf};
|
||
use std::process::Command;
|
||
use std::time::{SystemTime, UNIX_EPOCH};
|
||
|
||
/// Parse-cache file name stored under a published official resource root.
|
||
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
|
||
/// Current parse-cache schema version.
|
||
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
|
||
/// TextUnit detail index file name stored under a published official resource root.
|
||
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
|
||
/// Current TextUnit detail-index schema version.
|
||
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
|
||
|
||
/// Configuration for one official resource parse-cache refresh.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct OfficialParseConfig {
|
||
/// Published official resource root containing `official-download-manifest.json`.
|
||
pub resource_root: PathBuf,
|
||
/// `unzip` executable used to inspect zip archives without extracting them.
|
||
pub unzip_command: PathBuf,
|
||
}
|
||
|
||
impl OfficialParseConfig {
|
||
/// Creates parse-cache configuration for a published official resource root.
|
||
pub fn new(resource_root: impl Into<PathBuf>, unzip_command: impl Into<PathBuf>) -> Self {
|
||
Self {
|
||
resource_root: resource_root.into(),
|
||
unzip_command: unzip_command.into(),
|
||
}
|
||
}
|
||
|
||
/// Returns the parse-cache path for this resource root.
|
||
pub fn cache_path(&self) -> PathBuf {
|
||
self.resource_root.join(OFFICIAL_PARSE_CACHE_FILE)
|
||
}
|
||
}
|
||
|
||
/// Structured report returned by a parse-cache refresh.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialParseReport {
|
||
/// Official resource root that was inspected.
|
||
pub resource_root: PathBuf,
|
||
/// Parse-cache path written by the refresh.
|
||
pub cache_path: PathBuf,
|
||
/// Aggregate parse-cache summary.
|
||
pub summary: OfficialParseSummary,
|
||
/// TextUnit detail-index path written by the refresh.
|
||
pub textunit_index_path: PathBuf,
|
||
/// Aggregate TextUnit detail-index summary.
|
||
pub textunit_index_summary: OfficialTextUnitIndexSummary,
|
||
}
|
||
|
||
/// Aggregate counters for a parse-cache refresh.
|
||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialParseSummary {
|
||
/// Number of entries in `official-download-manifest.json`.
|
||
pub manifest_entry_count: usize,
|
||
/// Number of entries stored in `official-parse-cache.json`.
|
||
pub cache_entry_count: usize,
|
||
/// Number of direct bundle files or zip inner files inspected as parser candidates.
|
||
pub candidate_file_count: usize,
|
||
/// Number of zip inner file entries inspected.
|
||
pub zip_entry_count: usize,
|
||
/// Number of cached entries reused because source URL/path/size/BLAKE3 did not change.
|
||
pub skipped_unchanged_count: usize,
|
||
/// Number of bundles parsed successfully.
|
||
pub parsed_bundle_count: usize,
|
||
/// Number of files intentionally skipped because they are not UnityFS bundles.
|
||
pub unsupported_count: usize,
|
||
/// Number of files or archives that failed parser/cache inspection.
|
||
pub failed_count: usize,
|
||
/// Total TextAsset objects found in parsed Unity serialized files.
|
||
pub text_asset_count: usize,
|
||
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
|
||
#[serde(default)]
|
||
pub text_unit_count: usize,
|
||
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
|
||
#[serde(default)]
|
||
pub skipped_binary_text_asset_count: usize,
|
||
/// Number of non-fatal TypeTree field extraction diagnostics.
|
||
#[serde(default)]
|
||
pub text_unit_error_count: usize,
|
||
}
|
||
|
||
/// Persistent parse cache for one official resource root.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialParseCache {
|
||
/// Cache schema version.
|
||
#[serde(default = "default_parse_cache_version")]
|
||
pub version: u32,
|
||
/// Cache generation time as Unix seconds.
|
||
pub generated_unix_seconds: u64,
|
||
/// Aggregate summary for this cache.
|
||
pub summary: OfficialParseSummary,
|
||
/// Cache entries keyed by source URL and optional archive entry.
|
||
#[serde(default)]
|
||
pub entries: BTreeMap<String, OfficialParseCacheEntry>,
|
||
}
|
||
|
||
/// One parsed or skipped official resource file.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialParseCacheEntry {
|
||
/// Stable cache key.
|
||
pub key: String,
|
||
/// Official URL from the download manifest.
|
||
pub source_url: String,
|
||
/// Relative destination path under the official resource root.
|
||
pub destination: String,
|
||
/// Inner archive path when the source is a zip file.
|
||
pub archive_entry: Option<String>,
|
||
/// Source classification used by the parser.
|
||
pub source_kind: OfficialParseSourceKind,
|
||
/// Fingerprint derived from the download manifest.
|
||
pub fingerprint: OfficialParseSourceFingerprint,
|
||
/// Parse status for this cache entry.
|
||
pub status: OfficialParseStatus,
|
||
/// Whether this entry was reused from the previous parse cache in this run.
|
||
#[serde(default)]
|
||
pub reused_from_previous_cache: bool,
|
||
/// Unity editor version when a UnityFS bundle was parsed.
|
||
pub unity_version: Option<String>,
|
||
/// Number of files extracted from the UnityFS directory table.
|
||
pub file_count: usize,
|
||
/// Number of Unity serialized files parsed from extracted files.
|
||
pub serialized_file_count: usize,
|
||
/// Number of TextAsset objects found.
|
||
pub text_asset_count: usize,
|
||
/// TextAsset names found in stable order.
|
||
pub text_assets: Vec<String>,
|
||
/// Non-fatal serialized-file parse diagnostic count.
|
||
pub serialized_parse_error_count: usize,
|
||
/// Number of translation-ready TextUnit entries extracted from this bundle.
|
||
#[serde(default)]
|
||
pub text_unit_count: usize,
|
||
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
|
||
#[serde(default)]
|
||
pub text_unit_formats: Vec<String>,
|
||
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
|
||
#[serde(default)]
|
||
pub skipped_binary_text_asset_count: usize,
|
||
/// Non-fatal TypeTree field extraction diagnostic count.
|
||
#[serde(default)]
|
||
pub text_unit_error_count: usize,
|
||
/// Human-readable error or skip reason.
|
||
pub error: Option<String>,
|
||
}
|
||
|
||
/// Persistent TextUnit detail index for one official resource root.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialTextUnitIndex {
|
||
/// Index schema version.
|
||
#[serde(default = "default_textunit_index_version")]
|
||
pub version: u32,
|
||
/// Index generation time as Unix seconds.
|
||
pub generated_unix_seconds: u64,
|
||
/// Official resource root used to generate this index.
|
||
pub resource_root: PathBuf,
|
||
/// Aggregate counters for this index.
|
||
pub summary: OfficialTextUnitIndexSummary,
|
||
/// Extracted TextUnit details in deterministic order.
|
||
#[serde(default)]
|
||
pub units: Vec<OfficialTextUnitIndexUnit>,
|
||
/// Parse and extraction diagnostics in deterministic order.
|
||
#[serde(default)]
|
||
pub errors: Vec<OfficialTextUnitIndexError>,
|
||
}
|
||
|
||
/// Aggregate counters for a TextUnit detail index.
|
||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialTextUnitIndexSummary {
|
||
/// Number of TextUnit detail entries.
|
||
pub unit_count: usize,
|
||
/// Number of parse/extraction diagnostics.
|
||
pub error_count: usize,
|
||
/// Number of binary or invalid TextAsset payloads skipped by extraction.
|
||
pub skipped_binary_text_asset_count: usize,
|
||
}
|
||
|
||
/// One TextUnit detail entry.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialTextUnitIndexUnit {
|
||
/// Stable unit ID within the official release.
|
||
pub id: String,
|
||
/// Parse-cache entry key that produced this unit.
|
||
pub parse_entry_key: String,
|
||
/// Official URL from the download manifest.
|
||
pub source_url: String,
|
||
/// Relative destination path under the official resource root.
|
||
pub destination: String,
|
||
/// Inner archive path when the source is a zip file.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub archive_entry: Option<String>,
|
||
/// Source classification used by the parser.
|
||
pub source_kind: OfficialParseSourceKind,
|
||
/// Unity editor version when known.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub unity_version: Option<String>,
|
||
/// Original source text.
|
||
pub source_text: String,
|
||
/// Unity serialized file path.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub serialized_file: Option<String>,
|
||
/// Unity object path ID.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub path_id: Option<i64>,
|
||
/// Unity class ID.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub class_id: Option<i32>,
|
||
/// TypeTree field path.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub field_path: Option<String>,
|
||
/// Byte offset relative to the beginning of the Unity object payload.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub field_offset: Option<usize>,
|
||
/// Number of bytes consumed by this field, including alignment padding.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub field_byte_size: Option<usize>,
|
||
/// TextUnit payload format such as json/csv/tsv/plain.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub format: Option<String>,
|
||
/// Extraction source kind such as TextAsset or TypeTreeField.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub text_source_kind: Option<String>,
|
||
/// TextAsset name when the unit came from a TextAsset payload.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub asset_name: Option<String>,
|
||
/// Stable extraction context copied from the parser.
|
||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||
pub context: BTreeMap<String, String>,
|
||
}
|
||
|
||
/// One parse or extraction diagnostic in the TextUnit detail index.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialTextUnitIndexError {
|
||
/// Stable diagnostic ID within the official release.
|
||
pub id: String,
|
||
/// Parse-cache entry key associated with this diagnostic.
|
||
pub parse_entry_key: String,
|
||
/// Official URL from the download manifest.
|
||
pub source_url: String,
|
||
/// Relative destination path under the official resource root.
|
||
pub destination: String,
|
||
/// Inner archive path when the source is a zip file.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub archive_entry: Option<String>,
|
||
/// Source classification used by the parser.
|
||
pub source_kind: OfficialParseSourceKind,
|
||
/// Parse status associated with this diagnostic.
|
||
pub status: OfficialParseStatus,
|
||
/// Unity serialized file path when known.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub serialized_file: Option<String>,
|
||
/// Unity object path ID when known.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub path_id: Option<i64>,
|
||
/// Unity class ID when known.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub class_id: Option<i32>,
|
||
/// TypeTree field path when known.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub field_path: Option<String>,
|
||
/// Byte offset relative to the beginning of the Unity object payload.
|
||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||
pub offset: Option<usize>,
|
||
/// Human-readable diagnostic.
|
||
pub error: String,
|
||
}
|
||
|
||
/// Query filters for TextUnit detail entries.
|
||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||
pub struct OfficialTextUnitQuery {
|
||
/// Filter by destination path.
|
||
pub destination: Option<String>,
|
||
/// Filter by destination glob pattern.
|
||
pub path_pattern: Option<String>,
|
||
/// Filter by archive entry.
|
||
pub archive_entry: Option<String>,
|
||
/// Filter by Unity object path ID.
|
||
pub path_id: Option<i64>,
|
||
/// Filter by Unity class ID.
|
||
pub class_id: Option<i32>,
|
||
/// Filter by field path.
|
||
pub field_path: Option<String>,
|
||
/// Filter by TextUnit format.
|
||
pub format: Option<String>,
|
||
}
|
||
|
||
/// Source kind for a parse-cache entry.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum OfficialParseSourceKind {
|
||
/// A direct official resource file that should be a UnityFS bundle.
|
||
DirectBundle,
|
||
/// A file entry streamed from an official zip archive.
|
||
ZipEntry,
|
||
/// A manifest entry that is not expected to contain a UnityFS bundle.
|
||
Unsupported,
|
||
}
|
||
|
||
/// Stable fingerprint copied from the official download manifest.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct OfficialParseSourceFingerprint {
|
||
/// Official URL from the download manifest.
|
||
pub source_url: String,
|
||
/// Relative destination path under the official resource root.
|
||
pub destination: String,
|
||
/// File byte count from the verified download manifest.
|
||
pub bytes: u64,
|
||
/// BLAKE3 digest from the verified download manifest.
|
||
pub blake3: String,
|
||
}
|
||
|
||
/// Parse status for one official resource cache entry.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum OfficialParseStatus {
|
||
/// UnityFS parsing succeeded.
|
||
Parsed,
|
||
/// The source was inspected and is not currently parsed as UnityFS.
|
||
SkippedUnsupported,
|
||
/// The source was expected to be parseable but failed inspection.
|
||
Failed,
|
||
}
|
||
|
||
/// Service that refreshes official parse caches.
|
||
#[derive(Debug, Default, Clone, Copy)]
|
||
pub struct OfficialParseCacheService;
|
||
|
||
impl OfficialParseCacheService {
|
||
/// Creates a parse-cache service.
|
||
pub fn new() -> Self {
|
||
Self
|
||
}
|
||
|
||
/// Refreshes the parse cache for the configured official resource root.
|
||
pub fn run(&self, config: &OfficialParseConfig) -> Result<OfficialParseReport, String> {
|
||
let manifest = read_download_manifest_at(&config.resource_root)?.ok_or_else(|| {
|
||
format!(
|
||
"缺少官方下载 manifest,无法更新解析缓存:{}",
|
||
config.resource_root.display()
|
||
)
|
||
})?;
|
||
let previous_cache = read_parse_cache_at(&config.resource_root)?;
|
||
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
|
||
let mut summary = OfficialParseSummary {
|
||
manifest_entry_count: manifest.entries.len(),
|
||
..OfficialParseSummary::default()
|
||
};
|
||
let mut entries = BTreeMap::new();
|
||
let mut units = Vec::new();
|
||
let mut errors = Vec::new();
|
||
|
||
for manifest_entry in manifest.entries.values() {
|
||
let produced = process_manifest_entry(
|
||
config,
|
||
manifest_entry,
|
||
previous_cache.as_ref(),
|
||
previous_textunit_index.as_ref(),
|
||
);
|
||
for produced in produced {
|
||
summary.record_entry(&produced.entry);
|
||
units.extend(produced.units);
|
||
errors.extend(produced.errors);
|
||
entries.insert(produced.entry.key.clone(), produced.entry);
|
||
}
|
||
}
|
||
|
||
summary.cache_entry_count = entries.len();
|
||
let textunit_index_summary = OfficialTextUnitIndexSummary {
|
||
unit_count: units.len(),
|
||
error_count: errors.len(),
|
||
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
|
||
};
|
||
let cache = OfficialParseCache {
|
||
version: OFFICIAL_PARSE_CACHE_VERSION,
|
||
generated_unix_seconds: unix_seconds_now(),
|
||
summary: summary.clone(),
|
||
entries,
|
||
};
|
||
write_parse_cache_at(&config.resource_root, &cache)?;
|
||
let textunit_index = OfficialTextUnitIndex {
|
||
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
|
||
generated_unix_seconds: cache.generated_unix_seconds,
|
||
resource_root: config.resource_root.clone(),
|
||
summary: textunit_index_summary.clone(),
|
||
units,
|
||
errors,
|
||
};
|
||
write_textunit_index_at(&config.resource_root, &textunit_index)?;
|
||
|
||
Ok(OfficialParseReport {
|
||
resource_root: config.resource_root.clone(),
|
||
cache_path: config.cache_path(),
|
||
summary,
|
||
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
|
||
textunit_index_summary,
|
||
})
|
||
}
|
||
}
|
||
|
||
struct ParseProduced {
|
||
entry: OfficialParseCacheEntry,
|
||
units: Vec<OfficialTextUnitIndexUnit>,
|
||
errors: Vec<OfficialTextUnitIndexError>,
|
||
}
|
||
|
||
impl ParseProduced {
|
||
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
|
||
let errors = error_from_status_entry(&entry).into_iter().collect();
|
||
Self {
|
||
entry,
|
||
units: Vec::new(),
|
||
errors,
|
||
}
|
||
}
|
||
|
||
fn from_cached(
|
||
mut entry: OfficialParseCacheEntry,
|
||
previous_index: Option<&OfficialTextUnitIndex>,
|
||
) -> Self {
|
||
entry.reused_from_previous_cache = true;
|
||
let units = previous_index
|
||
.into_iter()
|
||
.flat_map(|index| index.units.iter())
|
||
.filter(|unit| unit.parse_entry_key == entry.key)
|
||
.cloned()
|
||
.collect();
|
||
let errors = previous_index
|
||
.into_iter()
|
||
.flat_map(|index| index.errors.iter())
|
||
.filter(|error| error.parse_entry_key == entry.key)
|
||
.cloned()
|
||
.collect();
|
||
Self {
|
||
entry,
|
||
units,
|
||
errors,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl OfficialParseSummary {
|
||
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
|
||
match entry.source_kind {
|
||
OfficialParseSourceKind::DirectBundle => {
|
||
self.candidate_file_count += 1;
|
||
}
|
||
OfficialParseSourceKind::ZipEntry => {
|
||
self.candidate_file_count += 1;
|
||
self.zip_entry_count += 1;
|
||
}
|
||
OfficialParseSourceKind::Unsupported => {}
|
||
}
|
||
match entry.status {
|
||
OfficialParseStatus::Parsed => {
|
||
self.parsed_bundle_count += 1;
|
||
self.text_asset_count += entry.text_asset_count;
|
||
self.text_unit_count += entry.text_unit_count;
|
||
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
|
||
self.text_unit_error_count += entry.text_unit_error_count;
|
||
}
|
||
OfficialParseStatus::SkippedUnsupported => {
|
||
self.unsupported_count += 1;
|
||
}
|
||
OfficialParseStatus::Failed => {
|
||
self.failed_count += 1;
|
||
}
|
||
}
|
||
if entry.reused_from_previous_cache {
|
||
self.skipped_unchanged_count += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Reads the parse cache under a published official resource root.
|
||
///
|
||
/// Missing, corrupt, or unsupported-version caches return `Ok(None)` so a new
|
||
/// cache can be regenerated without blocking official resource publication.
|
||
pub fn read_parse_cache_at(resource_root: &Path) -> Result<Option<OfficialParseCache>, String> {
|
||
let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE);
|
||
let Some(bytes) = read_file_no_symlink(&path, "官方解析缓存")? else {
|
||
return Ok(None);
|
||
};
|
||
let Ok(cache) = serde_json::from_slice::<OfficialParseCache>(&bytes) else {
|
||
return Ok(None);
|
||
};
|
||
if cache.version != OFFICIAL_PARSE_CACHE_VERSION {
|
||
return Ok(None);
|
||
}
|
||
Ok(Some(cache))
|
||
}
|
||
|
||
/// Writes the parse cache under a published official resource root.
|
||
pub fn write_parse_cache_at(
|
||
resource_root: &Path,
|
||
cache: &OfficialParseCache,
|
||
) -> Result<(), String> {
|
||
let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE);
|
||
ensure_path_within_root(resource_root, &path)?;
|
||
ensure_safe_file_target(resource_root, &path, "官方解析缓存")?;
|
||
let bytes = serde_json::to_vec_pretty(cache)
|
||
.map_err(|error| format!("序列化官方解析缓存失败:{error}"))?;
|
||
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
|
||
}
|
||
|
||
/// Reads the TextUnit detail index under a published official resource root.
|
||
pub fn read_textunit_index_at(
|
||
resource_root: &Path,
|
||
) -> Result<Option<OfficialTextUnitIndex>, String> {
|
||
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
|
||
return Ok(None);
|
||
};
|
||
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
|
||
return Ok(None);
|
||
};
|
||
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
|
||
return Ok(None);
|
||
}
|
||
Ok(Some(index))
|
||
}
|
||
|
||
/// Writes the TextUnit detail index under a published official resource root.
|
||
pub fn write_textunit_index_at(
|
||
resource_root: &Path,
|
||
index: &OfficialTextUnitIndex,
|
||
) -> Result<(), String> {
|
||
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||
ensure_path_within_root(resource_root, &path)?;
|
||
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
|
||
let bytes = serde_json::to_vec_pretty(index)
|
||
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
|
||
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
|
||
}
|
||
|
||
/// Returns TextUnit detail entries matching a query.
|
||
pub fn query_textunit_index_units<'a>(
|
||
index: &'a OfficialTextUnitIndex,
|
||
query: &OfficialTextUnitQuery,
|
||
) -> Vec<&'a OfficialTextUnitIndexUnit> {
|
||
index
|
||
.units
|
||
.iter()
|
||
.filter(|unit| textunit_unit_matches(unit, query))
|
||
.collect()
|
||
}
|
||
|
||
/// Returns parse/extraction diagnostics matching a query.
|
||
pub fn query_textunit_index_errors<'a>(
|
||
index: &'a OfficialTextUnitIndex,
|
||
query: &OfficialTextUnitQuery,
|
||
) -> Vec<&'a OfficialTextUnitIndexError> {
|
||
index
|
||
.errors
|
||
.iter()
|
||
.filter(|error| textunit_error_matches(error, query))
|
||
.collect()
|
||
}
|
||
|
||
fn process_manifest_entry(
|
||
config: &OfficialParseConfig,
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
previous_cache: Option<&OfficialParseCache>,
|
||
previous_index: Option<&OfficialTextUnitIndex>,
|
||
) -> Vec<ParseProduced> {
|
||
let fingerprint = fingerprint_for(manifest_entry);
|
||
if looks_like_zip_source(manifest_entry) {
|
||
return process_zip_entry(
|
||
config,
|
||
manifest_entry,
|
||
previous_cache,
|
||
previous_index,
|
||
fingerprint,
|
||
);
|
||
}
|
||
|
||
if looks_like_direct_bundle_source(manifest_entry) {
|
||
let key = direct_key(&manifest_entry.url);
|
||
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||
}
|
||
return vec![parse_direct_bundle(
|
||
config,
|
||
manifest_entry,
|
||
key,
|
||
fingerprint,
|
||
)];
|
||
}
|
||
|
||
let key = unsupported_key(&manifest_entry.url);
|
||
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||
}
|
||
vec![ParseProduced::from_entry(unsupported_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::Unsupported,
|
||
fingerprint,
|
||
key,
|
||
"非 UnityFS 候选资源",
|
||
))]
|
||
}
|
||
|
||
fn process_zip_entry(
|
||
config: &OfficialParseConfig,
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
previous_cache: Option<&OfficialParseCache>,
|
||
previous_index: Option<&OfficialTextUnitIndex>,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
) -> Vec<ParseProduced> {
|
||
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
|
||
if !cached_entries.is_empty() {
|
||
return cached_entries
|
||
.into_iter()
|
||
.map(|entry| ParseProduced::from_cached(entry, previous_index))
|
||
.collect();
|
||
}
|
||
|
||
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||
Ok(path) => path,
|
||
Err(error) => {
|
||
return vec![ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::ZipEntry,
|
||
fingerprint,
|
||
zip_list_key(&manifest_entry.url),
|
||
error,
|
||
))]
|
||
}
|
||
};
|
||
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
|
||
Ok(entries) => entries,
|
||
Err(error) => {
|
||
return vec![ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::ZipEntry,
|
||
fingerprint,
|
||
zip_list_key(&manifest_entry.url),
|
||
error,
|
||
))]
|
||
}
|
||
};
|
||
|
||
let mut produced = Vec::new();
|
||
for archive_entry in archive_entries {
|
||
let key = zip_key(&manifest_entry.url, &archive_entry);
|
||
let bytes = match read_zip_entry_bytes(&config.unzip_command, &archive_path, &archive_entry)
|
||
{
|
||
Ok(bytes) => bytes,
|
||
Err(error) => {
|
||
produced.push(ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
Some(archive_entry),
|
||
OfficialParseSourceKind::ZipEntry,
|
||
fingerprint.clone(),
|
||
key,
|
||
error,
|
||
)));
|
||
continue;
|
||
}
|
||
};
|
||
produced.push(parse_zip_inner_file(
|
||
manifest_entry,
|
||
archive_entry,
|
||
fingerprint.clone(),
|
||
key,
|
||
&bytes,
|
||
));
|
||
}
|
||
|
||
if produced.is_empty() {
|
||
produced.push(ParseProduced::from_entry(unsupported_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::Unsupported,
|
||
fingerprint,
|
||
zip_list_key(&manifest_entry.url),
|
||
"ZIP 内没有可检查文件条目",
|
||
)));
|
||
}
|
||
produced
|
||
}
|
||
|
||
fn parse_direct_bundle(
|
||
config: &OfficialParseConfig,
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
key: String,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
) -> ParseProduced {
|
||
let path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||
Ok(path) => path,
|
||
Err(error) => {
|
||
return ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::DirectBundle,
|
||
fingerprint,
|
||
key,
|
||
error,
|
||
))
|
||
}
|
||
};
|
||
let bytes = match read_resource_file(&path) {
|
||
Ok(bytes) => bytes,
|
||
Err(error) => {
|
||
return ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::DirectBundle,
|
||
fingerprint,
|
||
key,
|
||
error,
|
||
))
|
||
}
|
||
};
|
||
if !UnityFsParser::has_unityfs_signature(&bytes) {
|
||
return ParseProduced::from_entry(unsupported_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::DirectBundle,
|
||
fingerprint,
|
||
key,
|
||
"文件不是 UnityFS bundle",
|
||
));
|
||
}
|
||
parsed_bundle_entry(
|
||
manifest_entry,
|
||
None,
|
||
OfficialParseSourceKind::DirectBundle,
|
||
fingerprint,
|
||
key,
|
||
&bytes,
|
||
)
|
||
}
|
||
|
||
fn parse_zip_inner_file(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
archive_entry: String,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
key: String,
|
||
bytes: &[u8],
|
||
) -> ParseProduced {
|
||
if !UnityFsParser::has_unityfs_signature(bytes) {
|
||
return ParseProduced::from_entry(unsupported_entry(
|
||
manifest_entry,
|
||
Some(archive_entry),
|
||
OfficialParseSourceKind::ZipEntry,
|
||
fingerprint,
|
||
key,
|
||
"ZIP 条目不是 UnityFS bundle",
|
||
));
|
||
}
|
||
parsed_bundle_entry(
|
||
manifest_entry,
|
||
Some(archive_entry),
|
||
OfficialParseSourceKind::ZipEntry,
|
||
fingerprint,
|
||
key,
|
||
bytes,
|
||
)
|
||
}
|
||
|
||
fn parsed_bundle_entry(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
archive_entry: Option<String>,
|
||
source_kind: OfficialParseSourceKind,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
key: String,
|
||
bytes: &[u8],
|
||
) -> ParseProduced {
|
||
let parser = UnityFsParser::new();
|
||
match parser.parse(bytes) {
|
||
Ok(parsed) => {
|
||
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
|
||
&parsed,
|
||
Some(&manifest_entry.destination),
|
||
archive_entry.as_deref(),
|
||
);
|
||
let text_unit_formats = text_units
|
||
.units
|
||
.iter()
|
||
.filter_map(|unit| unit.context.get("format").cloned())
|
||
.collect::<BTreeSet<_>>()
|
||
.into_iter()
|
||
.collect();
|
||
|
||
let entry = OfficialParseCacheEntry {
|
||
key,
|
||
source_url: manifest_entry.url.clone(),
|
||
destination: manifest_entry.destination.clone(),
|
||
archive_entry,
|
||
source_kind,
|
||
fingerprint,
|
||
status: OfficialParseStatus::Parsed,
|
||
reused_from_previous_cache: false,
|
||
unity_version: Some(parsed.unity_version),
|
||
file_count: parsed.files.len(),
|
||
serialized_file_count: parsed.serialized_files.len(),
|
||
text_asset_count: parsed.text_assets.len(),
|
||
text_assets: parsed
|
||
.text_assets
|
||
.iter()
|
||
.map(|asset| asset.name.clone())
|
||
.collect(),
|
||
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
||
text_unit_count: text_units.units.len(),
|
||
text_unit_formats,
|
||
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
|
||
text_unit_error_count: text_units.errors.len(),
|
||
error: None,
|
||
};
|
||
let units = indexed_text_units_for_entry(&entry, &text_units.units);
|
||
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
|
||
ParseProduced {
|
||
entry,
|
||
units,
|
||
errors,
|
||
}
|
||
}
|
||
Err(error) => ParseProduced::from_entry(failed_entry(
|
||
manifest_entry,
|
||
archive_entry,
|
||
source_kind,
|
||
fingerprint,
|
||
key,
|
||
error.to_string(),
|
||
)),
|
||
}
|
||
}
|
||
|
||
fn indexed_text_units_for_entry(
|
||
entry: &OfficialParseCacheEntry,
|
||
units: &[TextUnit],
|
||
) -> Vec<OfficialTextUnitIndexUnit> {
|
||
units
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(index, unit)| OfficialTextUnitIndexUnit {
|
||
id: format!("{}#unit:{index}", entry.key),
|
||
parse_entry_key: entry.key.clone(),
|
||
source_url: entry.source_url.clone(),
|
||
destination: entry.destination.clone(),
|
||
archive_entry: entry.archive_entry.clone(),
|
||
source_kind: entry.source_kind,
|
||
unity_version: entry.unity_version.clone(),
|
||
source_text: unit.source_text.clone(),
|
||
serialized_file: unit.serialized_file.clone(),
|
||
path_id: unit.path_id,
|
||
class_id: unit.class_id,
|
||
field_path: unit.field_path.clone(),
|
||
field_offset: unit.field_offset,
|
||
field_byte_size: unit.field_byte_size,
|
||
format: unit.context.get("format").cloned(),
|
||
text_source_kind: unit.context.get("source_kind").cloned(),
|
||
asset_name: unit.context.get("asset_name").cloned(),
|
||
context: unit.context.clone(),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
fn indexed_extraction_errors_for_entry(
|
||
entry: &OfficialParseCacheEntry,
|
||
errors: &[TextUnitExtractionError],
|
||
) -> Vec<OfficialTextUnitIndexError> {
|
||
errors
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(index, error)| OfficialTextUnitIndexError {
|
||
id: format!("{}#extract-error:{index}", entry.key),
|
||
parse_entry_key: entry.key.clone(),
|
||
source_url: entry.source_url.clone(),
|
||
destination: entry.destination.clone(),
|
||
archive_entry: entry.archive_entry.clone(),
|
||
source_kind: entry.source_kind,
|
||
status: entry.status,
|
||
serialized_file: error.serialized_file.clone(),
|
||
path_id: error.path_id,
|
||
class_id: error.class_id,
|
||
field_path: error.field_path.clone(),
|
||
offset: error.offset,
|
||
error: error.error.clone(),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
|
||
let error = entry.error.as_ref()?;
|
||
Some(OfficialTextUnitIndexError {
|
||
id: format!("{}#status", entry.key),
|
||
parse_entry_key: entry.key.clone(),
|
||
source_url: entry.source_url.clone(),
|
||
destination: entry.destination.clone(),
|
||
archive_entry: entry.archive_entry.clone(),
|
||
source_kind: entry.source_kind,
|
||
status: entry.status,
|
||
serialized_file: None,
|
||
path_id: None,
|
||
class_id: None,
|
||
field_path: None,
|
||
offset: None,
|
||
error: error.clone(),
|
||
})
|
||
}
|
||
|
||
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
|
||
if query
|
||
.destination
|
||
.as_ref()
|
||
.is_some_and(|destination| &unit.destination != destination)
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.path_pattern
|
||
.as_ref()
|
||
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.archive_entry
|
||
.as_ref()
|
||
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.path_id
|
||
.is_some_and(|path_id| unit.path_id != Some(path_id))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.class_id
|
||
.is_some_and(|class_id| unit.class_id != Some(class_id))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.field_path
|
||
.as_ref()
|
||
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.format
|
||
.as_ref()
|
||
.is_some_and(|format| unit.format.as_ref() != Some(format))
|
||
{
|
||
return false;
|
||
}
|
||
true
|
||
}
|
||
|
||
fn textunit_error_matches(
|
||
error: &OfficialTextUnitIndexError,
|
||
query: &OfficialTextUnitQuery,
|
||
) -> bool {
|
||
if query
|
||
.destination
|
||
.as_ref()
|
||
.is_some_and(|destination| &error.destination != destination)
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.path_pattern
|
||
.as_ref()
|
||
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.archive_entry
|
||
.as_ref()
|
||
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.path_id
|
||
.is_some_and(|path_id| error.path_id != Some(path_id))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.class_id
|
||
.is_some_and(|class_id| error.class_id != Some(class_id))
|
||
{
|
||
return false;
|
||
}
|
||
if query
|
||
.field_path
|
||
.as_ref()
|
||
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
|
||
{
|
||
return false;
|
||
}
|
||
true
|
||
}
|
||
|
||
fn glob_matches(pattern: &str, value: &str) -> bool {
|
||
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
|
||
}
|
||
|
||
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
|
||
match pattern.split_first() {
|
||
None => value.is_empty(),
|
||
Some((&b'*', rest)) => {
|
||
glob_matches_bytes(rest, value)
|
||
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
|
||
}
|
||
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
|
||
Some((&literal, rest)) => value
|
||
.split_first()
|
||
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
|
||
}
|
||
}
|
||
|
||
fn failed_entry(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
archive_entry: Option<String>,
|
||
source_kind: OfficialParseSourceKind,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
key: String,
|
||
error: impl Into<String>,
|
||
) -> OfficialParseCacheEntry {
|
||
status_entry(
|
||
manifest_entry,
|
||
archive_entry,
|
||
source_kind,
|
||
fingerprint,
|
||
key,
|
||
OfficialParseStatus::Failed,
|
||
error.into(),
|
||
)
|
||
}
|
||
|
||
fn unsupported_entry(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
archive_entry: Option<String>,
|
||
source_kind: OfficialParseSourceKind,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
key: String,
|
||
reason: impl Into<String>,
|
||
) -> OfficialParseCacheEntry {
|
||
status_entry(
|
||
manifest_entry,
|
||
archive_entry,
|
||
source_kind,
|
||
fingerprint,
|
||
key,
|
||
OfficialParseStatus::SkippedUnsupported,
|
||
reason.into(),
|
||
)
|
||
}
|
||
|
||
fn status_entry(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
archive_entry: Option<String>,
|
||
source_kind: OfficialParseSourceKind,
|
||
fingerprint: OfficialParseSourceFingerprint,
|
||
key: String,
|
||
status: OfficialParseStatus,
|
||
reason: String,
|
||
) -> OfficialParseCacheEntry {
|
||
OfficialParseCacheEntry {
|
||
key,
|
||
source_url: manifest_entry.url.clone(),
|
||
destination: manifest_entry.destination.clone(),
|
||
archive_entry,
|
||
source_kind,
|
||
fingerprint,
|
||
status,
|
||
reused_from_previous_cache: false,
|
||
unity_version: None,
|
||
file_count: 0,
|
||
serialized_file_count: 0,
|
||
text_asset_count: 0,
|
||
text_assets: Vec::new(),
|
||
serialized_parse_error_count: 0,
|
||
text_unit_count: 0,
|
||
text_unit_formats: Vec::new(),
|
||
skipped_binary_text_asset_count: 0,
|
||
text_unit_error_count: 0,
|
||
error: Some(reason),
|
||
}
|
||
}
|
||
|
||
fn read_resource_file(path: &Path) -> Result<Vec<u8>, String> {
|
||
read_file_no_symlink(path, "官方解析输入")?
|
||
.ok_or_else(|| format!("官方解析输入不存在:{}", path.display()))
|
||
}
|
||
|
||
fn resource_path_for(
|
||
resource_root: &Path,
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
) -> Result<PathBuf, String> {
|
||
let path = resource_root.join(Path::new(&manifest_entry.destination));
|
||
ensure_path_within_root(resource_root, &path)?;
|
||
ensure_safe_file_target(resource_root, &path, "官方解析输入")?;
|
||
Ok(path)
|
||
}
|
||
|
||
fn list_zip_entries(unzip_command: &Path, archive_path: &Path) -> Result<Vec<String>, String> {
|
||
let output = Command::new(unzip_command)
|
||
.arg("-Z1")
|
||
.arg(archive_path)
|
||
.output()
|
||
.map_err(|error| {
|
||
format!(
|
||
"启动 unzip 列出 ZIP 条目失败 {}:{error}",
|
||
archive_path.display()
|
||
)
|
||
})?;
|
||
if !output.status.success() {
|
||
return Err(format!(
|
||
"列出 ZIP 条目失败 {}:{}",
|
||
archive_path.display(),
|
||
String::from_utf8_lossy(&output.stderr).trim()
|
||
));
|
||
}
|
||
Ok(String::from_utf8_lossy(&output.stdout)
|
||
.lines()
|
||
.map(|line| line.trim_end_matches('\r').trim().to_string())
|
||
.filter(|line| !line.is_empty() && !line.ends_with('/'))
|
||
.collect())
|
||
}
|
||
|
||
fn read_zip_entry_bytes(
|
||
unzip_command: &Path,
|
||
archive_path: &Path,
|
||
archive_entry: &str,
|
||
) -> Result<Vec<u8>, String> {
|
||
let output = Command::new(unzip_command)
|
||
.arg("-p")
|
||
.arg(archive_path)
|
||
.arg(archive_entry)
|
||
.output()
|
||
.map_err(|error| {
|
||
format!(
|
||
"启动 unzip 读取 ZIP 条目失败 {}!{}:{error}",
|
||
archive_path.display(),
|
||
archive_entry
|
||
)
|
||
})?;
|
||
if !output.status.success() {
|
||
return Err(format!(
|
||
"读取 ZIP 条目失败 {}!{}:{}",
|
||
archive_path.display(),
|
||
archive_entry,
|
||
String::from_utf8_lossy(&output.stderr).trim()
|
||
));
|
||
}
|
||
Ok(output.stdout)
|
||
}
|
||
|
||
fn reusable_entry(
|
||
previous_cache: Option<&OfficialParseCache>,
|
||
key: &str,
|
||
fingerprint: &OfficialParseSourceFingerprint,
|
||
) -> Option<OfficialParseCacheEntry> {
|
||
previous_cache
|
||
.and_then(|cache| cache.entries.get(key))
|
||
.filter(|entry| &entry.fingerprint == fingerprint)
|
||
.cloned()
|
||
}
|
||
|
||
fn reusable_archive_entries(
|
||
previous_cache: Option<&OfficialParseCache>,
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
fingerprint: &OfficialParseSourceFingerprint,
|
||
) -> Vec<OfficialParseCacheEntry> {
|
||
previous_cache
|
||
.into_iter()
|
||
.flat_map(|cache| cache.entries.values())
|
||
.filter(|entry| {
|
||
entry.source_url == manifest_entry.url
|
||
&& entry.destination == manifest_entry.destination
|
||
&& &entry.fingerprint == fingerprint
|
||
})
|
||
.cloned()
|
||
.collect()
|
||
}
|
||
|
||
fn fingerprint_for(
|
||
manifest_entry: &OfficialDownloadManifestEntry,
|
||
) -> OfficialParseSourceFingerprint {
|
||
OfficialParseSourceFingerprint {
|
||
source_url: manifest_entry.url.clone(),
|
||
destination: manifest_entry.destination.clone(),
|
||
bytes: manifest_entry.bytes,
|
||
blake3: manifest_entry.blake3.clone(),
|
||
}
|
||
}
|
||
|
||
fn looks_like_zip_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool {
|
||
has_case_insensitive_suffix(&manifest_entry.url, ".zip")
|
||
|| has_case_insensitive_suffix(&manifest_entry.destination, ".zip")
|
||
}
|
||
|
||
fn looks_like_direct_bundle_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool {
|
||
[".bundle", ".unity3d"].iter().any(|suffix| {
|
||
has_case_insensitive_suffix(&manifest_entry.url, suffix)
|
||
|| has_case_insensitive_suffix(&manifest_entry.destination, suffix)
|
||
})
|
||
}
|
||
|
||
fn has_case_insensitive_suffix(value: &str, suffix: &str) -> bool {
|
||
value
|
||
.rsplit(['/', '\\'])
|
||
.next()
|
||
.is_some_and(|name| name.to_ascii_lowercase().ends_with(suffix))
|
||
}
|
||
|
||
fn direct_key(url: &str) -> String {
|
||
format!("direct:{url}")
|
||
}
|
||
|
||
fn zip_key(url: &str, archive_entry: &str) -> String {
|
||
format!("zip:{url}!{archive_entry}")
|
||
}
|
||
|
||
fn zip_list_key(url: &str) -> String {
|
||
format!("zip-list:{url}")
|
||
}
|
||
|
||
fn unsupported_key(url: &str) -> String {
|
||
format!("unsupported:{url}")
|
||
}
|
||
|
||
fn default_parse_cache_version() -> u32 {
|
||
OFFICIAL_PARSE_CACHE_VERSION
|
||
}
|
||
|
||
fn default_textunit_index_version() -> u32 {
|
||
OFFICIAL_TEXTUNIT_INDEX_VERSION
|
||
}
|
||
|
||
fn unix_seconds_now() -> u64 {
|
||
SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_secs()
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use std::fs;
|
||
|
||
#[cfg(unix)]
|
||
use std::os::unix::fs::PermissionsExt;
|
||
|
||
fn push_c_string(data: &mut Vec<u8>, value: &str) {
|
||
data.extend_from_slice(value.as_bytes());
|
||
data.push(0);
|
||
}
|
||
|
||
fn push_u16(data: &mut Vec<u8>, value: u16) {
|
||
data.extend_from_slice(&value.to_be_bytes());
|
||
}
|
||
|
||
fn push_u32(data: &mut Vec<u8>, value: u32) {
|
||
data.extend_from_slice(&value.to_be_bytes());
|
||
}
|
||
|
||
fn push_i32(data: &mut Vec<u8>, value: i32) {
|
||
data.extend_from_slice(&value.to_be_bytes());
|
||
}
|
||
|
||
fn push_u64(data: &mut Vec<u8>, value: u64) {
|
||
data.extend_from_slice(&value.to_be_bytes());
|
||
}
|
||
|
||
fn align(data: &mut Vec<u8>, alignment: usize) {
|
||
let remainder = data.len() % alignment;
|
||
if remainder != 0 {
|
||
data.resize(data.len() + alignment - remainder, 0);
|
||
}
|
||
}
|
||
|
||
fn synthetic_unityfs_bundle(directory_path: &str, payload: &[u8]) -> Vec<u8> {
|
||
let mut blocks_info = Vec::new();
|
||
blocks_info.extend_from_slice(&[0xAB; 16]);
|
||
push_i32(&mut blocks_info, 1);
|
||
push_u32(&mut blocks_info, payload.len() as u32);
|
||
push_u32(&mut blocks_info, payload.len() as u32);
|
||
push_u16(&mut blocks_info, 0);
|
||
push_i32(&mut blocks_info, 1);
|
||
push_u64(&mut blocks_info, 0);
|
||
push_u64(&mut blocks_info, payload.len() as u64);
|
||
push_u32(&mut blocks_info, 0);
|
||
push_c_string(&mut blocks_info, directory_path);
|
||
|
||
let mut data = Vec::new();
|
||
push_c_string(&mut data, "UnityFS");
|
||
push_u32(&mut data, 8);
|
||
push_c_string(&mut data, "5.x.x");
|
||
push_c_string(&mut data, "2021.3.56f2");
|
||
push_u64(&mut data, 0);
|
||
push_u32(&mut data, blocks_info.len() as u32);
|
||
push_u32(&mut data, blocks_info.len() as u32);
|
||
push_u32(&mut data, 0);
|
||
align(&mut data, 16);
|
||
data.extend_from_slice(&blocks_info);
|
||
data.extend_from_slice(payload);
|
||
|
||
let total_size = data.len() as u64;
|
||
let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len();
|
||
data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes());
|
||
data
|
||
}
|
||
|
||
fn write_download_manifest(
|
||
root: &Path,
|
||
entries: &[(&str, &str, &[u8])],
|
||
) -> Vec<OfficialDownloadManifestEntry> {
|
||
let mut manifest_entries = BTreeMap::new();
|
||
let mut written = Vec::new();
|
||
for (url, destination, bytes) in entries {
|
||
let path = root.join(destination);
|
||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||
fs::write(&path, bytes).unwrap();
|
||
let entry = OfficialDownloadManifestEntry {
|
||
url: (*url).to_string(),
|
||
destination: (*destination).to_string(),
|
||
bytes: bytes.len() as u64,
|
||
blake3: blake3::hash(bytes).to_hex().to_string(),
|
||
};
|
||
manifest_entries.insert((*url).to_string(), entry.clone());
|
||
written.push(entry);
|
||
}
|
||
let manifest = serde_json::json!({
|
||
"version": 1,
|
||
"entries": manifest_entries,
|
||
});
|
||
fs::write(
|
||
root.join("official-download-manifest.json"),
|
||
serde_json::to_vec_pretty(&manifest).unwrap(),
|
||
)
|
||
.unwrap();
|
||
written
|
||
}
|
||
|
||
#[test]
|
||
fn parses_direct_bundle_and_reuses_unchanged_cache() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let root = temp.path();
|
||
let bundle = synthetic_unityfs_bundle("CAB-fixture", b"data");
|
||
write_download_manifest(
|
||
root,
|
||
&[(
|
||
"https://prod-clientpatch.bluearchiveyostar.com/r93/Bundle/test.bundle",
|
||
"Bundle/test.bundle",
|
||
&bundle,
|
||
)],
|
||
);
|
||
let config = OfficialParseConfig::new(root, "unzip");
|
||
|
||
let first = OfficialParseCacheService::new().run(&config).unwrap();
|
||
assert_eq!(first.summary.manifest_entry_count, 1);
|
||
assert_eq!(first.summary.parsed_bundle_count, 1);
|
||
assert_eq!(first.summary.skipped_unchanged_count, 0);
|
||
assert!(first.cache_path.exists());
|
||
|
||
let second = OfficialParseCacheService::new().run(&config).unwrap();
|
||
assert_eq!(second.summary.parsed_bundle_count, 1);
|
||
assert_eq!(second.summary.skipped_unchanged_count, 1);
|
||
let cache = read_parse_cache_at(root).unwrap().unwrap();
|
||
let entry = cache.entries.values().next().unwrap();
|
||
assert_eq!(entry.status, OfficialParseStatus::Parsed);
|
||
assert_eq!(entry.file_count, 1);
|
||
}
|
||
|
||
#[test]
|
||
fn records_non_candidate_resources_as_unsupported() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let root = temp.path();
|
||
write_download_manifest(
|
||
root,
|
||
&[(
|
||
"https://prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/TableCatalog.bytes",
|
||
"TableBundles/TableCatalog.bytes",
|
||
b"catalog",
|
||
)],
|
||
);
|
||
let config = OfficialParseConfig::new(root, "unzip");
|
||
|
||
let report = OfficialParseCacheService::new().run(&config).unwrap();
|
||
|
||
assert_eq!(report.summary.parsed_bundle_count, 0);
|
||
assert_eq!(report.summary.unsupported_count, 1);
|
||
assert_eq!(report.summary.failed_count, 0);
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn parses_zip_entries_without_extracting_archive_tree() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let root = temp.path();
|
||
let bundle = synthetic_unityfs_bundle("CAB-zipped", b"zipdata");
|
||
let bundle_path = temp.path().join("inner.bundle");
|
||
fs::write(&bundle_path, &bundle).unwrap();
|
||
let archive_bytes = b"zip-placeholder";
|
||
write_download_manifest(
|
||
root,
|
||
&[(
|
||
"https://prod-clientpatch.bluearchiveyostar.com/r93/Windows_PatchPack/FullPatch_000.zip",
|
||
"Windows_PatchPack/FullPatch_000.zip",
|
||
archive_bytes,
|
||
)],
|
||
);
|
||
let unzip_script = temp.path().join("fake-unzip");
|
||
fs::write(
|
||
&unzip_script,
|
||
format!(
|
||
r#"#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
if [[ "${{1:-}}" == "-Z1" ]]; then
|
||
printf '%s\n' 'assets/scenario.bundle' 'assets/readme.txt'
|
||
exit 0
|
||
fi
|
||
if [[ "${{1:-}}" == "-p" && "${{3:-}}" == "assets/scenario.bundle" ]]; then
|
||
cat '{}'
|
||
exit 0
|
||
fi
|
||
if [[ "${{1:-}}" == "-p" ]]; then
|
||
printf 'plain text'
|
||
exit 0
|
||
fi
|
||
exit 2
|
||
"#,
|
||
bundle_path.display()
|
||
),
|
||
)
|
||
.unwrap();
|
||
let mut permissions = fs::metadata(&unzip_script).unwrap().permissions();
|
||
permissions.set_mode(0o755);
|
||
fs::set_permissions(&unzip_script, permissions).unwrap();
|
||
let config = OfficialParseConfig::new(root, unzip_script);
|
||
|
||
let report = OfficialParseCacheService::new().run(&config).unwrap();
|
||
|
||
assert_eq!(report.summary.manifest_entry_count, 1);
|
||
assert_eq!(report.summary.zip_entry_count, 2);
|
||
assert_eq!(report.summary.parsed_bundle_count, 1);
|
||
assert_eq!(report.summary.unsupported_count, 1);
|
||
assert_eq!(report.summary.failed_count, 0);
|
||
assert!(!root.join("assets").exists());
|
||
}
|
||
}
|