mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 13:34:53 +08:00
feat(sync): 接入解析缓存与汉化发布前置
补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。 同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。 验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
@@ -10,9 +10,11 @@ use crate::path_security::{
|
||||
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
||||
STATE_FILE_MODE,
|
||||
};
|
||||
use bat_assetbundle::{Parser, UnityFsParser};
|
||||
use bat_assetbundle::{
|
||||
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -20,7 +22,11 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
/// Parse-cache file name stored under a published official resource root.
|
||||
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
|
||||
/// Current parse-cache schema version.
|
||||
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1;
|
||||
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
|
||||
/// TextUnit detail index file name stored under a published official resource root.
|
||||
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
|
||||
/// Current TextUnit detail-index schema version.
|
||||
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
|
||||
|
||||
/// Configuration for one official resource parse-cache refresh.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -55,6 +61,10 @@ pub struct OfficialParseReport {
|
||||
pub cache_path: PathBuf,
|
||||
/// Aggregate parse-cache summary.
|
||||
pub summary: OfficialParseSummary,
|
||||
/// TextUnit detail-index path written by the refresh.
|
||||
pub textunit_index_path: PathBuf,
|
||||
/// Aggregate TextUnit detail-index summary.
|
||||
pub textunit_index_summary: OfficialTextUnitIndexSummary,
|
||||
}
|
||||
|
||||
/// Aggregate counters for a parse-cache refresh.
|
||||
@@ -78,6 +88,15 @@ pub struct OfficialParseSummary {
|
||||
pub failed_count: usize,
|
||||
/// Total TextAsset objects found in parsed Unity serialized files.
|
||||
pub text_asset_count: usize,
|
||||
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
|
||||
#[serde(default)]
|
||||
pub text_unit_count: usize,
|
||||
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
|
||||
#[serde(default)]
|
||||
pub skipped_binary_text_asset_count: usize,
|
||||
/// Number of non-fatal TypeTree field extraction diagnostics.
|
||||
#[serde(default)]
|
||||
pub text_unit_error_count: usize,
|
||||
}
|
||||
|
||||
/// Persistent parse cache for one official resource root.
|
||||
@@ -127,10 +146,162 @@ pub struct OfficialParseCacheEntry {
|
||||
pub text_assets: Vec<String>,
|
||||
/// Non-fatal serialized-file parse diagnostic count.
|
||||
pub serialized_parse_error_count: usize,
|
||||
/// Number of translation-ready TextUnit entries extracted from this bundle.
|
||||
#[serde(default)]
|
||||
pub text_unit_count: usize,
|
||||
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
|
||||
#[serde(default)]
|
||||
pub text_unit_formats: Vec<String>,
|
||||
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
|
||||
#[serde(default)]
|
||||
pub skipped_binary_text_asset_count: usize,
|
||||
/// Non-fatal TypeTree field extraction diagnostic count.
|
||||
#[serde(default)]
|
||||
pub text_unit_error_count: usize,
|
||||
/// Human-readable error or skip reason.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Persistent TextUnit detail index for one official resource root.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OfficialTextUnitIndex {
|
||||
/// Index schema version.
|
||||
#[serde(default = "default_textunit_index_version")]
|
||||
pub version: u32,
|
||||
/// Index generation time as Unix seconds.
|
||||
pub generated_unix_seconds: u64,
|
||||
/// Official resource root used to generate this index.
|
||||
pub resource_root: PathBuf,
|
||||
/// Aggregate counters for this index.
|
||||
pub summary: OfficialTextUnitIndexSummary,
|
||||
/// Extracted TextUnit details in deterministic order.
|
||||
#[serde(default)]
|
||||
pub units: Vec<OfficialTextUnitIndexUnit>,
|
||||
/// Parse and extraction diagnostics in deterministic order.
|
||||
#[serde(default)]
|
||||
pub errors: Vec<OfficialTextUnitIndexError>,
|
||||
}
|
||||
|
||||
/// Aggregate counters for a TextUnit detail index.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OfficialTextUnitIndexSummary {
|
||||
/// Number of TextUnit detail entries.
|
||||
pub unit_count: usize,
|
||||
/// Number of parse/extraction diagnostics.
|
||||
pub error_count: usize,
|
||||
/// Number of binary or invalid TextAsset payloads skipped by extraction.
|
||||
pub skipped_binary_text_asset_count: usize,
|
||||
}
|
||||
|
||||
/// One TextUnit detail entry.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OfficialTextUnitIndexUnit {
|
||||
/// Stable unit ID within the official release.
|
||||
pub id: String,
|
||||
/// Parse-cache entry key that produced this unit.
|
||||
pub parse_entry_key: String,
|
||||
/// Official URL from the download manifest.
|
||||
pub source_url: String,
|
||||
/// Relative destination path under the official resource root.
|
||||
pub destination: String,
|
||||
/// Inner archive path when the source is a zip file.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub archive_entry: Option<String>,
|
||||
/// Source classification used by the parser.
|
||||
pub source_kind: OfficialParseSourceKind,
|
||||
/// Unity editor version when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unity_version: Option<String>,
|
||||
/// Original source text.
|
||||
pub source_text: String,
|
||||
/// Unity serialized file path.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialized_file: Option<String>,
|
||||
/// Unity object path ID.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path_id: Option<i64>,
|
||||
/// Unity class ID.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub class_id: Option<i32>,
|
||||
/// TypeTree field path.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub field_path: Option<String>,
|
||||
/// Byte offset relative to the beginning of the Unity object payload.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub field_offset: Option<usize>,
|
||||
/// Number of bytes consumed by this field, including alignment padding.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub field_byte_size: Option<usize>,
|
||||
/// TextUnit payload format such as json/csv/tsv/plain.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub format: Option<String>,
|
||||
/// Extraction source kind such as TextAsset or TypeTreeField.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_source_kind: Option<String>,
|
||||
/// TextAsset name when the unit came from a TextAsset payload.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub asset_name: Option<String>,
|
||||
/// Stable extraction context copied from the parser.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub context: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
/// One parse or extraction diagnostic in the TextUnit detail index.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OfficialTextUnitIndexError {
|
||||
/// Stable diagnostic ID within the official release.
|
||||
pub id: String,
|
||||
/// Parse-cache entry key associated with this diagnostic.
|
||||
pub parse_entry_key: String,
|
||||
/// Official URL from the download manifest.
|
||||
pub source_url: String,
|
||||
/// Relative destination path under the official resource root.
|
||||
pub destination: String,
|
||||
/// Inner archive path when the source is a zip file.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub archive_entry: Option<String>,
|
||||
/// Source classification used by the parser.
|
||||
pub source_kind: OfficialParseSourceKind,
|
||||
/// Parse status associated with this diagnostic.
|
||||
pub status: OfficialParseStatus,
|
||||
/// Unity serialized file path when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialized_file: Option<String>,
|
||||
/// Unity object path ID when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path_id: Option<i64>,
|
||||
/// Unity class ID when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub class_id: Option<i32>,
|
||||
/// TypeTree field path when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub field_path: Option<String>,
|
||||
/// Byte offset relative to the beginning of the Unity object payload.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub offset: Option<usize>,
|
||||
/// Human-readable diagnostic.
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
/// Query filters for TextUnit detail entries.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct OfficialTextUnitQuery {
|
||||
/// Filter by destination path.
|
||||
pub destination: Option<String>,
|
||||
/// Filter by destination glob pattern.
|
||||
pub path_pattern: Option<String>,
|
||||
/// Filter by archive entry.
|
||||
pub archive_entry: Option<String>,
|
||||
/// Filter by Unity object path ID.
|
||||
pub path_id: Option<i64>,
|
||||
/// Filter by Unity class ID.
|
||||
pub class_id: Option<i32>,
|
||||
/// Filter by field path.
|
||||
pub field_path: Option<String>,
|
||||
/// Filter by TextUnit format.
|
||||
pub format: Option<String>,
|
||||
}
|
||||
|
||||
/// Source kind for a parse-cache entry.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -187,21 +358,36 @@ impl OfficialParseCacheService {
|
||||
)
|
||||
})?;
|
||||
let previous_cache = read_parse_cache_at(&config.resource_root)?;
|
||||
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
|
||||
let mut summary = OfficialParseSummary {
|
||||
manifest_entry_count: manifest.entries.len(),
|
||||
..OfficialParseSummary::default()
|
||||
};
|
||||
let mut entries = BTreeMap::new();
|
||||
let mut units = Vec::new();
|
||||
let mut errors = Vec::new();
|
||||
|
||||
for manifest_entry in manifest.entries.values() {
|
||||
let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref());
|
||||
for entry in produced {
|
||||
summary.record_entry(&entry);
|
||||
entries.insert(entry.key.clone(), entry);
|
||||
let produced = process_manifest_entry(
|
||||
config,
|
||||
manifest_entry,
|
||||
previous_cache.as_ref(),
|
||||
previous_textunit_index.as_ref(),
|
||||
);
|
||||
for produced in produced {
|
||||
summary.record_entry(&produced.entry);
|
||||
units.extend(produced.units);
|
||||
errors.extend(produced.errors);
|
||||
entries.insert(produced.entry.key.clone(), produced.entry);
|
||||
}
|
||||
}
|
||||
|
||||
summary.cache_entry_count = entries.len();
|
||||
let textunit_index_summary = OfficialTextUnitIndexSummary {
|
||||
unit_count: units.len(),
|
||||
error_count: errors.len(),
|
||||
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
|
||||
};
|
||||
let cache = OfficialParseCache {
|
||||
version: OFFICIAL_PARSE_CACHE_VERSION,
|
||||
generated_unix_seconds: unix_seconds_now(),
|
||||
@@ -209,15 +395,67 @@ impl OfficialParseCacheService {
|
||||
entries,
|
||||
};
|
||||
write_parse_cache_at(&config.resource_root, &cache)?;
|
||||
let textunit_index = OfficialTextUnitIndex {
|
||||
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
|
||||
generated_unix_seconds: cache.generated_unix_seconds,
|
||||
resource_root: config.resource_root.clone(),
|
||||
summary: textunit_index_summary.clone(),
|
||||
units,
|
||||
errors,
|
||||
};
|
||||
write_textunit_index_at(&config.resource_root, &textunit_index)?;
|
||||
|
||||
Ok(OfficialParseReport {
|
||||
resource_root: config.resource_root.clone(),
|
||||
cache_path: config.cache_path(),
|
||||
summary,
|
||||
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
|
||||
textunit_index_summary,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ParseProduced {
|
||||
entry: OfficialParseCacheEntry,
|
||||
units: Vec<OfficialTextUnitIndexUnit>,
|
||||
errors: Vec<OfficialTextUnitIndexError>,
|
||||
}
|
||||
|
||||
impl ParseProduced {
|
||||
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
|
||||
let errors = error_from_status_entry(&entry).into_iter().collect();
|
||||
Self {
|
||||
entry,
|
||||
units: Vec::new(),
|
||||
errors,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_cached(
|
||||
mut entry: OfficialParseCacheEntry,
|
||||
previous_index: Option<&OfficialTextUnitIndex>,
|
||||
) -> Self {
|
||||
entry.reused_from_previous_cache = true;
|
||||
let units = previous_index
|
||||
.into_iter()
|
||||
.flat_map(|index| index.units.iter())
|
||||
.filter(|unit| unit.parse_entry_key == entry.key)
|
||||
.cloned()
|
||||
.collect();
|
||||
let errors = previous_index
|
||||
.into_iter()
|
||||
.flat_map(|index| index.errors.iter())
|
||||
.filter(|error| error.parse_entry_key == entry.key)
|
||||
.cloned()
|
||||
.collect();
|
||||
Self {
|
||||
entry,
|
||||
units,
|
||||
errors,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl OfficialParseSummary {
|
||||
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
|
||||
match entry.source_kind {
|
||||
@@ -234,6 +472,9 @@ impl OfficialParseSummary {
|
||||
OfficialParseStatus::Parsed => {
|
||||
self.parsed_bundle_count += 1;
|
||||
self.text_asset_count += entry.text_asset_count;
|
||||
self.text_unit_count += entry.text_unit_count;
|
||||
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
|
||||
self.text_unit_error_count += entry.text_unit_error_count;
|
||||
}
|
||||
OfficialParseStatus::SkippedUnsupported => {
|
||||
self.unsupported_count += 1;
|
||||
@@ -279,21 +520,81 @@ pub fn write_parse_cache_at(
|
||||
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
|
||||
}
|
||||
|
||||
/// Reads the TextUnit detail index under a published official resource root.
|
||||
pub fn read_textunit_index_at(
|
||||
resource_root: &Path,
|
||||
) -> Result<Option<OfficialTextUnitIndex>, String> {
|
||||
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||||
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(index))
|
||||
}
|
||||
|
||||
/// Writes the TextUnit detail index under a published official resource root.
|
||||
pub fn write_textunit_index_at(
|
||||
resource_root: &Path,
|
||||
index: &OfficialTextUnitIndex,
|
||||
) -> Result<(), String> {
|
||||
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||||
ensure_path_within_root(resource_root, &path)?;
|
||||
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
|
||||
let bytes = serde_json::to_vec_pretty(index)
|
||||
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
|
||||
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
|
||||
}
|
||||
|
||||
/// Returns TextUnit detail entries matching a query.
|
||||
pub fn query_textunit_index_units<'a>(
|
||||
index: &'a OfficialTextUnitIndex,
|
||||
query: &OfficialTextUnitQuery,
|
||||
) -> Vec<&'a OfficialTextUnitIndexUnit> {
|
||||
index
|
||||
.units
|
||||
.iter()
|
||||
.filter(|unit| textunit_unit_matches(unit, query))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns parse/extraction diagnostics matching a query.
|
||||
pub fn query_textunit_index_errors<'a>(
|
||||
index: &'a OfficialTextUnitIndex,
|
||||
query: &OfficialTextUnitQuery,
|
||||
) -> Vec<&'a OfficialTextUnitIndexError> {
|
||||
index
|
||||
.errors
|
||||
.iter()
|
||||
.filter(|error| textunit_error_matches(error, query))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn process_manifest_entry(
|
||||
config: &OfficialParseConfig,
|
||||
manifest_entry: &OfficialDownloadManifestEntry,
|
||||
previous_cache: Option<&OfficialParseCache>,
|
||||
) -> Vec<OfficialParseCacheEntry> {
|
||||
previous_index: Option<&OfficialTextUnitIndex>,
|
||||
) -> Vec<ParseProduced> {
|
||||
let fingerprint = fingerprint_for(manifest_entry);
|
||||
if looks_like_zip_source(manifest_entry) {
|
||||
return process_zip_entry(config, manifest_entry, previous_cache, fingerprint);
|
||||
return process_zip_entry(
|
||||
config,
|
||||
manifest_entry,
|
||||
previous_cache,
|
||||
previous_index,
|
||||
fingerprint,
|
||||
);
|
||||
}
|
||||
|
||||
if looks_like_direct_bundle_source(manifest_entry) {
|
||||
let key = direct_key(&manifest_entry.url);
|
||||
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||
cached.reused_from_previous_cache = true;
|
||||
return vec![cached];
|
||||
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||||
}
|
||||
return vec![parse_direct_bundle(
|
||||
config,
|
||||
@@ -304,61 +605,58 @@ fn process_manifest_entry(
|
||||
}
|
||||
|
||||
let key = unsupported_key(&manifest_entry.url);
|
||||
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||
cached.reused_from_previous_cache = true;
|
||||
return vec![cached];
|
||||
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||||
}
|
||||
vec![unsupported_entry(
|
||||
vec![ParseProduced::from_entry(unsupported_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::Unsupported,
|
||||
fingerprint,
|
||||
key,
|
||||
"非 UnityFS 候选资源",
|
||||
)]
|
||||
))]
|
||||
}
|
||||
|
||||
fn process_zip_entry(
|
||||
config: &OfficialParseConfig,
|
||||
manifest_entry: &OfficialDownloadManifestEntry,
|
||||
previous_cache: Option<&OfficialParseCache>,
|
||||
previous_index: Option<&OfficialTextUnitIndex>,
|
||||
fingerprint: OfficialParseSourceFingerprint,
|
||||
) -> Vec<OfficialParseCacheEntry> {
|
||||
) -> Vec<ParseProduced> {
|
||||
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
|
||||
if !cached_entries.is_empty() {
|
||||
return cached_entries
|
||||
.into_iter()
|
||||
.map(|mut entry| {
|
||||
entry.reused_from_previous_cache = true;
|
||||
entry
|
||||
})
|
||||
.map(|entry| ParseProduced::from_cached(entry, previous_index))
|
||||
.collect();
|
||||
}
|
||||
|
||||
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
return vec![failed_entry(
|
||||
return vec![ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::ZipEntry,
|
||||
fingerprint,
|
||||
zip_list_key(&manifest_entry.url),
|
||||
error,
|
||||
)]
|
||||
))]
|
||||
}
|
||||
};
|
||||
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
|
||||
Ok(entries) => entries,
|
||||
Err(error) => {
|
||||
return vec![failed_entry(
|
||||
return vec![ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::ZipEntry,
|
||||
fingerprint,
|
||||
zip_list_key(&manifest_entry.url),
|
||||
error,
|
||||
)]
|
||||
))]
|
||||
}
|
||||
};
|
||||
|
||||
@@ -369,14 +667,14 @@ fn process_zip_entry(
|
||||
{
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
produced.push(failed_entry(
|
||||
produced.push(ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
Some(archive_entry),
|
||||
OfficialParseSourceKind::ZipEntry,
|
||||
fingerprint.clone(),
|
||||
key,
|
||||
error,
|
||||
));
|
||||
)));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -390,14 +688,14 @@ fn process_zip_entry(
|
||||
}
|
||||
|
||||
if produced.is_empty() {
|
||||
produced.push(unsupported_entry(
|
||||
produced.push(ParseProduced::from_entry(unsupported_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::Unsupported,
|
||||
fingerprint,
|
||||
zip_list_key(&manifest_entry.url),
|
||||
"ZIP 内没有可检查文件条目",
|
||||
));
|
||||
)));
|
||||
}
|
||||
produced
|
||||
}
|
||||
@@ -407,42 +705,42 @@ fn parse_direct_bundle(
|
||||
manifest_entry: &OfficialDownloadManifestEntry,
|
||||
key: String,
|
||||
fingerprint: OfficialParseSourceFingerprint,
|
||||
) -> OfficialParseCacheEntry {
|
||||
) -> ParseProduced {
|
||||
let path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
return failed_entry(
|
||||
return ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::DirectBundle,
|
||||
fingerprint,
|
||||
key,
|
||||
error,
|
||||
)
|
||||
))
|
||||
}
|
||||
};
|
||||
let bytes = match read_resource_file(&path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
return failed_entry(
|
||||
return ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::DirectBundle,
|
||||
fingerprint,
|
||||
key,
|
||||
error,
|
||||
)
|
||||
))
|
||||
}
|
||||
};
|
||||
if !UnityFsParser::has_unityfs_signature(&bytes) {
|
||||
return unsupported_entry(
|
||||
return ParseProduced::from_entry(unsupported_entry(
|
||||
manifest_entry,
|
||||
None,
|
||||
OfficialParseSourceKind::DirectBundle,
|
||||
fingerprint,
|
||||
key,
|
||||
"文件不是 UnityFS bundle",
|
||||
);
|
||||
));
|
||||
}
|
||||
parsed_bundle_entry(
|
||||
manifest_entry,
|
||||
@@ -460,16 +758,16 @@ fn parse_zip_inner_file(
|
||||
fingerprint: OfficialParseSourceFingerprint,
|
||||
key: String,
|
||||
bytes: &[u8],
|
||||
) -> OfficialParseCacheEntry {
|
||||
) -> ParseProduced {
|
||||
if !UnityFsParser::has_unityfs_signature(bytes) {
|
||||
return unsupported_entry(
|
||||
return ParseProduced::from_entry(unsupported_entry(
|
||||
manifest_entry,
|
||||
Some(archive_entry),
|
||||
OfficialParseSourceKind::ZipEntry,
|
||||
fingerprint,
|
||||
key,
|
||||
"ZIP 条目不是 UnityFS bundle",
|
||||
);
|
||||
));
|
||||
}
|
||||
parsed_bundle_entry(
|
||||
manifest_entry,
|
||||
@@ -488,38 +786,254 @@ fn parsed_bundle_entry(
|
||||
fingerprint: OfficialParseSourceFingerprint,
|
||||
key: String,
|
||||
bytes: &[u8],
|
||||
) -> OfficialParseCacheEntry {
|
||||
) -> ParseProduced {
|
||||
let parser = UnityFsParser::new();
|
||||
match parser.parse(bytes) {
|
||||
Ok(parsed) => OfficialParseCacheEntry {
|
||||
key,
|
||||
source_url: manifest_entry.url.clone(),
|
||||
destination: manifest_entry.destination.clone(),
|
||||
archive_entry,
|
||||
source_kind,
|
||||
fingerprint,
|
||||
status: OfficialParseStatus::Parsed,
|
||||
reused_from_previous_cache: false,
|
||||
unity_version: Some(parsed.unity_version),
|
||||
file_count: parsed.files.len(),
|
||||
serialized_file_count: parsed.serialized_files.len(),
|
||||
text_asset_count: parsed.text_assets.len(),
|
||||
text_assets: parsed
|
||||
.text_assets
|
||||
Ok(parsed) => {
|
||||
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
|
||||
&parsed,
|
||||
Some(&manifest_entry.destination),
|
||||
archive_entry.as_deref(),
|
||||
);
|
||||
let text_unit_formats = text_units
|
||||
.units
|
||||
.iter()
|
||||
.map(|asset| asset.name.clone())
|
||||
.collect(),
|
||||
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
||||
error: None,
|
||||
},
|
||||
Err(error) => failed_entry(
|
||||
.filter_map(|unit| unit.context.get("format").cloned())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let entry = OfficialParseCacheEntry {
|
||||
key,
|
||||
source_url: manifest_entry.url.clone(),
|
||||
destination: manifest_entry.destination.clone(),
|
||||
archive_entry,
|
||||
source_kind,
|
||||
fingerprint,
|
||||
status: OfficialParseStatus::Parsed,
|
||||
reused_from_previous_cache: false,
|
||||
unity_version: Some(parsed.unity_version),
|
||||
file_count: parsed.files.len(),
|
||||
serialized_file_count: parsed.serialized_files.len(),
|
||||
text_asset_count: parsed.text_assets.len(),
|
||||
text_assets: parsed
|
||||
.text_assets
|
||||
.iter()
|
||||
.map(|asset| asset.name.clone())
|
||||
.collect(),
|
||||
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
||||
text_unit_count: text_units.units.len(),
|
||||
text_unit_formats,
|
||||
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
|
||||
text_unit_error_count: text_units.errors.len(),
|
||||
error: None,
|
||||
};
|
||||
let units = indexed_text_units_for_entry(&entry, &text_units.units);
|
||||
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
|
||||
ParseProduced {
|
||||
entry,
|
||||
units,
|
||||
errors,
|
||||
}
|
||||
}
|
||||
Err(error) => ParseProduced::from_entry(failed_entry(
|
||||
manifest_entry,
|
||||
archive_entry,
|
||||
source_kind,
|
||||
fingerprint,
|
||||
key,
|
||||
error.to_string(),
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn indexed_text_units_for_entry(
|
||||
entry: &OfficialParseCacheEntry,
|
||||
units: &[TextUnit],
|
||||
) -> Vec<OfficialTextUnitIndexUnit> {
|
||||
units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, unit)| OfficialTextUnitIndexUnit {
|
||||
id: format!("{}#unit:{index}", entry.key),
|
||||
parse_entry_key: entry.key.clone(),
|
||||
source_url: entry.source_url.clone(),
|
||||
destination: entry.destination.clone(),
|
||||
archive_entry: entry.archive_entry.clone(),
|
||||
source_kind: entry.source_kind,
|
||||
unity_version: entry.unity_version.clone(),
|
||||
source_text: unit.source_text.clone(),
|
||||
serialized_file: unit.serialized_file.clone(),
|
||||
path_id: unit.path_id,
|
||||
class_id: unit.class_id,
|
||||
field_path: unit.field_path.clone(),
|
||||
field_offset: unit.field_offset,
|
||||
field_byte_size: unit.field_byte_size,
|
||||
format: unit.context.get("format").cloned(),
|
||||
text_source_kind: unit.context.get("source_kind").cloned(),
|
||||
asset_name: unit.context.get("asset_name").cloned(),
|
||||
context: unit.context.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn indexed_extraction_errors_for_entry(
|
||||
entry: &OfficialParseCacheEntry,
|
||||
errors: &[TextUnitExtractionError],
|
||||
) -> Vec<OfficialTextUnitIndexError> {
|
||||
errors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, error)| OfficialTextUnitIndexError {
|
||||
id: format!("{}#extract-error:{index}", entry.key),
|
||||
parse_entry_key: entry.key.clone(),
|
||||
source_url: entry.source_url.clone(),
|
||||
destination: entry.destination.clone(),
|
||||
archive_entry: entry.archive_entry.clone(),
|
||||
source_kind: entry.source_kind,
|
||||
status: entry.status,
|
||||
serialized_file: error.serialized_file.clone(),
|
||||
path_id: error.path_id,
|
||||
class_id: error.class_id,
|
||||
field_path: error.field_path.clone(),
|
||||
offset: error.offset,
|
||||
error: error.error.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
|
||||
let error = entry.error.as_ref()?;
|
||||
Some(OfficialTextUnitIndexError {
|
||||
id: format!("{}#status", entry.key),
|
||||
parse_entry_key: entry.key.clone(),
|
||||
source_url: entry.source_url.clone(),
|
||||
destination: entry.destination.clone(),
|
||||
archive_entry: entry.archive_entry.clone(),
|
||||
source_kind: entry.source_kind,
|
||||
status: entry.status,
|
||||
serialized_file: None,
|
||||
path_id: None,
|
||||
class_id: None,
|
||||
field_path: None,
|
||||
offset: None,
|
||||
error: error.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
|
||||
if query
|
||||
.destination
|
||||
.as_ref()
|
||||
.is_some_and(|destination| &unit.destination != destination)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.path_pattern
|
||||
.as_ref()
|
||||
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.archive_entry
|
||||
.as_ref()
|
||||
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.path_id
|
||||
.is_some_and(|path_id| unit.path_id != Some(path_id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.class_id
|
||||
.is_some_and(|class_id| unit.class_id != Some(class_id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.field_path
|
||||
.as_ref()
|
||||
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.format
|
||||
.as_ref()
|
||||
.is_some_and(|format| unit.format.as_ref() != Some(format))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn textunit_error_matches(
|
||||
error: &OfficialTextUnitIndexError,
|
||||
query: &OfficialTextUnitQuery,
|
||||
) -> bool {
|
||||
if query
|
||||
.destination
|
||||
.as_ref()
|
||||
.is_some_and(|destination| &error.destination != destination)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.path_pattern
|
||||
.as_ref()
|
||||
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.archive_entry
|
||||
.as_ref()
|
||||
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.path_id
|
||||
.is_some_and(|path_id| error.path_id != Some(path_id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.class_id
|
||||
.is_some_and(|class_id| error.class_id != Some(class_id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if query
|
||||
.field_path
|
||||
.as_ref()
|
||||
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn glob_matches(pattern: &str, value: &str) -> bool {
|
||||
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
|
||||
}
|
||||
|
||||
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
|
||||
match pattern.split_first() {
|
||||
None => value.is_empty(),
|
||||
Some((&b'*', rest)) => {
|
||||
glob_matches_bytes(rest, value)
|
||||
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
|
||||
}
|
||||
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
|
||||
Some((&literal, rest)) => value
|
||||
.split_first()
|
||||
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,6 +1099,10 @@ fn status_entry(
|
||||
text_asset_count: 0,
|
||||
text_assets: Vec::new(),
|
||||
serialized_parse_error_count: 0,
|
||||
text_unit_count: 0,
|
||||
text_unit_formats: Vec::new(),
|
||||
skipped_binary_text_asset_count: 0,
|
||||
text_unit_error_count: 0,
|
||||
error: Some(reason),
|
||||
}
|
||||
}
|
||||
@@ -735,6 +1253,10 @@ fn default_parse_cache_version() -> u32 {
|
||||
OFFICIAL_PARSE_CACHE_VERSION
|
||||
}
|
||||
|
||||
fn default_textunit_index_version() -> u32 {
|
||||
OFFICIAL_TEXTUNIT_INDEX_VERSION
|
||||
}
|
||||
|
||||
fn unix_seconds_now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
|
||||
Reference in New Issue
Block a user