feat(sync): 接入解析缓存与汉化发布前置

补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
2026-07-31 00:38:45 +08:00
parent f4880a71bd
commit 2079c6a307
25 changed files with 17287 additions and 225 deletions
Generated
+2
View File
@@ -88,6 +88,7 @@ dependencies = [
"hex", "hex",
"lz4", "lz4",
"lzma-rs", "lzma-rs",
"md-5",
"serde", "serde",
"serde_json", "serde_json",
"thiserror", "thiserror",
@@ -145,6 +146,7 @@ dependencies = [
"bat-assetbundle", "bat-assetbundle",
"bat-cas-engine", "bat-cas-engine",
"bat-core", "bat-core",
"bat-patch",
"blake3", "blake3",
"hex", "hex",
"libc", "libc",
+2 -2
View File
@@ -13,7 +13,7 @@ pub use adapter::{
}; };
pub use registry::UnityAdapterRegistry; pub use registry::UnityAdapterRegistry;
pub use serialized_file::{ pub use serialized_file::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
UnityTypeTreeNode, UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
}; };
pub use unity_2021_3::Unity2021_3Adapter; pub use unity_2021_3::Unity2021_3Adapter;
+2 -2
View File
@@ -4,6 +4,6 @@
//! existing call sites can continue to import through `bat_adapters::unity`. //! existing call sites can continue to import through `bat_adapters::unity`.
pub use bat_assetbundle::{ pub use bat_assetbundle::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
UnityTypeTreeNode, UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
}; };
+3 -1
View File
@@ -7,7 +7,9 @@ pub mod translation;
pub use game_client::{ClientStatus, GameClient, GameRegion}; pub use game_client::{ClientStatus, GameClient, GameRegion};
pub use game_version::{GameVersion, UnityVersion}; pub use game_version::{GameVersion, UnityVersion};
pub use resource::{crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceType}; pub use resource::{
crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceMetadata, ResourceType,
};
pub use translation::{ pub use translation::{
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText, ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
TranslationStatus, TranslationStatus,
+55
View File
@@ -43,6 +43,58 @@ pub struct ResourceEntry {
pub crc: Option<u32>, pub crc: Option<u32>,
} }
/// 资源解析与发布侧元数据。
///
/// 该结构默认全空,保证旧索引和只保存基础 manifest 信息的资源仍可反序列化。
/// 官方资源导入会按 release manifest 和 parse cache 填充这些字段,供
/// `resource.index` 等只读接口暴露版本、平台、bundle、TextAsset 和 TextUnit 摘要。
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ResourceMetadata {
/// 资源所属的官方 release ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub official_release_id: Option<String>,
/// 从官方相对路径推断的平台标签,例如 `windows` 或 `android`。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub platform: Option<String>,
/// 资源本身或所在 bundle 的官方相对路径。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bundle_path: Option<String>,
/// ZIP 内被解析到的 bundle entry;直接 bundle 为空。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub archive_entries: Vec<String>,
/// parse cache 中出现过的解析状态标签。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub parse_statuses: Vec<String>,
/// 解析到的 Unity 版本集合。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unity_versions: Vec<String>,
/// UnityFS directory file 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub unityfs_file_count: u64,
/// Unity serialized file 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub serialized_file_count: u64,
/// TextAsset 对象总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_asset_count: u64,
/// TextAsset 名称集合。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub text_assets: Vec<String>,
/// TextUnit 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_unit_count: u64,
/// TextUnit 格式标签集合,例如 `json`、`csv`、`tsv`、`plain`。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub text_unit_formats: Vec<String>,
/// TextUnit 提取阶段的非致命诊断数量。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_unit_error_count: u64,
}
fn is_zero(value: &u64) -> bool {
*value == 0
}
/// 已下载字节与 catalog 声明的可校验字段不一致。 /// 已下载字节与 catalog 声明的可校验字段不一致。
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum IntegrityMismatch { pub enum IntegrityMismatch {
@@ -133,6 +185,9 @@ pub struct Resource {
pub local_path: PathBuf, pub local_path: PathBuf,
/// 资源条目 /// 资源条目
pub entry: ResourceEntry, pub entry: ResourceEntry,
/// 解析、发布和索引侧扩展元数据。
#[serde(default)]
pub metadata: ResourceMetadata,
} }
#[cfg(test)] #[cfg(test)]
+1
View File
@@ -13,6 +13,7 @@ serde_json.workspace = true
tracing.workspace = true tracing.workspace = true
lz4 = "1.28" lz4 = "1.28"
lzma-rs = "0.3" lzma-rs = "0.3"
md-5 = "0.10"
[dev-dependencies] [dev-dependencies]
hex = "0.4" hex = "0.4"
+14 -2
View File
@@ -9,14 +9,26 @@
pub mod error; pub mod error;
pub mod parser; pub mod parser;
pub mod patch;
pub mod serialized; pub mod serialized;
pub mod text;
pub mod types; pub mod types;
pub use error::{AssetBundleError, Result}; pub use error::{AssetBundleError, Result};
pub use parser::{compression_from_flags, Parser, UnityFsParser}; pub use parser::{compression_from_flags, Parser, UnityFsParser};
pub use patch::{
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
StringFieldPatch, TextAssetPatch,
};
pub use serialized::{ pub use serialized::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, UnityManagedReferenceMetadata, UnityManagedReferenceRecord, UnitySerializedField,
UnityTypeTreeNode, UnitySerializedFieldReplacement, UnitySerializedFile, UnitySerializedObject,
UnitySerializedReplacementValue, UnitySerializedTextAsset, UnitySerializedType,
UnitySerializedValue, UnityTypeTreeNode,
};
pub use text::{
text_units_to_jsonl, TextUnit, TextUnitExtractionError, TextUnitExtractionReport,
TextUnitExtractor,
}; };
pub use types::{ pub use types::{
AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle, AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+949
View File
@@ -0,0 +1,949 @@
//! Text extraction from parsed Unity serialized objects.
use crate::serialized::{
managed_reference_metadata_from_fields, UnityManagedReferenceMetadata,
UnityManagedReferenceRecord, UnitySerializedField, UnitySerializedValue,
};
use crate::types::ParsedAssetBundle;
use std::collections::BTreeMap;
/// One text unit used by translation, glossary and patch pipelines.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnit {
/// Original source text.
pub source_text: String,
/// Logical AssetBundle path, when provided by the caller.
pub bundle_path: Option<String>,
/// ZIP/archive entry containing the bundle, when known.
pub archive_entry: Option<String>,
/// Unity serialized file path.
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID, for example `49` for `TextAsset`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path. `TextAsset` is used for a whole TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// Unity version associated with the source.
pub version: String,
/// Stable context for format, asset name and extraction details.
pub context: BTreeMap<String, String>,
}
/// Non-fatal diagnostic generated while extracting text units.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnitExtractionError {
/// Serialized file containing the failed object.
pub serialized_file: Option<String>,
/// Object path ID, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable error.
pub error: String,
}
/// Result of extracting text units from one parsed bundle.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnitExtractionReport {
/// Extracted text units in deterministic traversal order.
pub units: Vec<TextUnit>,
/// Non-fatal object-level errors.
pub errors: Vec<TextUnitExtractionError>,
/// TextAsset payloads that were binary or invalid UTF-8.
pub skipped_binary_text_assets: usize,
}
/// Extracts translation-ready text from Unity bundle data.
#[derive(Debug, Default, Clone, Copy)]
pub struct TextUnitExtractor;
impl TextUnitExtractor {
/// Creates an extractor.
pub fn new() -> Self {
Self
}
/// Extracts TextAsset and TypeTree string fields from a parsed bundle.
pub fn extract_bundle(
&self,
bundle: &ParsedAssetBundle,
bundle_path: Option<&str>,
) -> TextUnitExtractionReport {
self.extract_bundle_with_context(bundle, bundle_path, None)
}
/// Extracts text with both logical bundle and archive-entry context.
pub fn extract_bundle_with_context(
&self,
bundle: &ParsedAssetBundle,
bundle_path: Option<&str>,
archive_entry: Option<&str>,
) -> TextUnitExtractionReport {
let mut report = TextUnitExtractionReport {
units: Vec::new(),
errors: Vec::new(),
skipped_binary_text_assets: 0,
};
for asset in &bundle.text_assets {
if let Some((format, text)) = decode_text_payload(&asset.bytes) {
let mut context = BTreeMap::new();
context.insert("format".to_string(), format.to_string());
context.insert("asset_name".to_string(), asset.name.clone());
context.insert("source_kind".to_string(), "TextAsset".to_string());
report.units.push(TextUnit {
source_text: text,
bundle_path: bundle_path.map(ToOwned::to_owned),
archive_entry: archive_entry.map(ToOwned::to_owned),
serialized_file: asset.source_path.clone(),
path_id: Some(asset.path_id),
class_id: Some(49),
field_path: Some("TextAsset".to_string()),
field_offset: None,
field_byte_size: None,
version: bundle.unity_version.clone(),
context,
});
} else {
report.skipped_binary_text_assets += 1;
}
}
for serialized_file in &bundle.serialized_files {
for object in &serialized_file.objects {
if object.class_id == 49 || !serialized_file.object_has_type_tree(object) {
continue;
}
let fields = match serialized_file.fields_for_object_entry(object) {
Ok(fields) => fields,
Err(error) => {
report.errors.push(TextUnitExtractionError {
serialized_file: serialized_file.source_path.clone(),
path_id: Some(object.path_id),
class_id: Some(object.class_id),
field_path: None,
offset: None,
error: error.to_string(),
});
continue;
}
};
let context = FieldTextContext {
serialized_file_path: serialized_file.source_path.as_deref(),
path_id: object.path_id,
class_id: object.class_id,
version: &bundle.unity_version,
bundle_path,
archive_entry,
managed_reference: None,
};
for field in fields {
collect_field_text(&mut report.units, &context, &field);
}
}
}
report
}
}
/// Serializes text units as one stable JSON object per line.
pub fn text_units_to_jsonl(units: &[TextUnit]) -> Result<String, serde_json::Error> {
let mut output = String::new();
for unit in units {
output.push_str(&serde_json::to_string(unit)?);
output.push('\n');
}
Ok(output)
}
#[derive(Clone)]
struct FieldTextContext<'a> {
serialized_file_path: Option<&'a str>,
path_id: i64,
class_id: i32,
version: &'a str,
bundle_path: Option<&'a str>,
archive_entry: Option<&'a str>,
managed_reference: Option<UnityManagedReferenceMetadata>,
}
impl<'a> FieldTextContext<'a> {
fn with_managed_reference(&self, metadata: Option<&UnityManagedReferenceMetadata>) -> Self {
Self {
serialized_file_path: self.serialized_file_path,
path_id: self.path_id,
class_id: self.class_id,
version: self.version,
bundle_path: self.bundle_path,
archive_entry: self.archive_entry,
managed_reference: metadata.cloned().or_else(|| self.managed_reference.clone()),
}
}
}
fn collect_field_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
field: &'a UnitySerializedField,
) {
match &field.value {
UnitySerializedValue::String(text) if !text.is_empty() => {
let mut unit_context = BTreeMap::new();
unit_context.insert("format".to_string(), "plain".to_string());
unit_context.insert(
"source_kind".to_string(),
if context.managed_reference.is_some() {
"ManagedReferenceField"
} else {
"TypeTreeField"
}
.to_string(),
);
unit_context.insert("type_name".to_string(), field.type_name.clone());
if let Some(metadata) = &context.managed_reference {
insert_managed_reference_context(&mut unit_context, metadata);
}
units.push(TextUnit {
source_text: text.clone(),
bundle_path: context.bundle_path.map(ToOwned::to_owned),
archive_entry: context.archive_entry.map(ToOwned::to_owned),
serialized_file: context.serialized_file_path.map(ToOwned::to_owned),
path_id: Some(context.path_id),
class_id: Some(context.class_id),
field_path: Some(field.path.clone()),
field_offset: Some(field.offset),
field_byte_size: Some(field.byte_size),
version: context.version.to_string(),
context: unit_context,
});
}
UnitySerializedValue::Object(fields) => {
for field in fields {
collect_field_text(units, context, field);
}
}
UnitySerializedValue::ManagedReference {
metadata, fields, ..
} => {
let managed_context = context.with_managed_reference(metadata.as_ref());
for field in fields {
collect_managed_reference_child_text(units, &managed_context, field);
}
}
UnitySerializedValue::ManagedReferenceRegistry { references, fields } => {
if references.is_empty() {
let fallback_metadata = managed_reference_metadata_from_fields(fields);
for field in fields {
let sibling_metadata =
managed_reference_metadata_from_sibling_fields(fields, field);
let managed_context = context.with_managed_reference(
sibling_metadata.as_ref().or(fallback_metadata.as_ref()),
);
collect_managed_reference_child_text(units, &managed_context, field);
}
} else {
for reference in references {
collect_managed_reference_record_text(units, context, reference);
}
}
}
UnitySerializedValue::Array(values) | UnitySerializedValue::Map(values) => {
for field in values {
collect_field_text(units, context, field);
}
}
_ => {}
}
}
fn collect_managed_reference_record_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
reference: &'a UnityManagedReferenceRecord,
) {
let managed_context = context.with_managed_reference(Some(&reference.metadata));
for field in &reference.fields {
collect_field_text(units, &managed_context, field);
}
}
fn collect_managed_reference_child_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
field: &'a UnitySerializedField,
) {
let key = normalized_text_metadata_key(&field.name);
if is_managed_reference_metadata_text_key(&key) {
return;
}
if is_managed_reference_payload_text_key(&key) {
if let Some(children) = serialized_field_children(field) {
for child in children {
collect_field_text(units, context, child);
}
} else {
collect_field_text(units, context, field);
}
return;
}
if let Some(children) = serialized_field_children(field) {
if let Some(metadata) = managed_reference_metadata_from_fields(children) {
let managed_context = context.with_managed_reference(Some(&metadata));
for child in children {
collect_managed_reference_child_text(units, &managed_context, child);
}
return;
}
}
collect_field_text(units, context, field);
}
fn managed_reference_metadata_from_sibling_fields(
fields: &[UnitySerializedField],
field: &UnitySerializedField,
) -> Option<UnityManagedReferenceMetadata> {
let record_prefix = managed_reference_record_path_prefix(&field.path)?;
let grouped_fields = fields
.iter()
.filter(|candidate| {
candidate.path == record_prefix
|| candidate
.path
.strip_prefix(record_prefix)
.is_some_and(|suffix| suffix.starts_with('.'))
})
.cloned()
.collect::<Vec<_>>();
managed_reference_metadata_from_fields(&grouped_fields)
}
fn managed_reference_record_path_prefix(path: &str) -> Option<&str> {
if let Some(index_end) = path.rfind(']') {
return Some(&path[..=index_end]);
}
path.rsplit_once('.')
.map(|(parent, _)| parent)
.filter(|parent| !parent.is_empty())
}
fn insert_managed_reference_context(
context: &mut BTreeMap<String, String>,
metadata: &UnityManagedReferenceMetadata,
) {
if let Some(reference_id) = metadata.reference_id {
context.insert("managed_reference_id".to_string(), reference_id.to_string());
}
if let Some(value) = &metadata.full_type_name {
context.insert(
"managed_reference_full_type_name".to_string(),
value.clone(),
);
}
if let Some(value) = &metadata.type_name {
context.insert("managed_reference_type".to_string(), value.clone());
}
if let Some(value) = &metadata.namespace {
context.insert("managed_reference_namespace".to_string(), value.clone());
}
if let Some(value) = &metadata.assembly_name {
context.insert("managed_reference_assembly".to_string(), value.clone());
}
}
fn serialized_field_children(field: &UnitySerializedField) -> Option<&[UnitySerializedField]> {
match &field.value {
UnitySerializedValue::Object(fields)
| UnitySerializedValue::Array(fields)
| UnitySerializedValue::Map(fields)
| UnitySerializedValue::ManagedReference { fields, .. }
| UnitySerializedValue::ManagedReferenceRegistry { fields, .. } => Some(fields),
_ => None,
}
}
fn normalized_text_metadata_key(name: &str) -> String {
name.strip_prefix("m_")
.unwrap_or(name)
.chars()
.filter(|character| character.is_ascii_alphanumeric())
.flat_map(char::to_lowercase)
.collect()
}
fn is_managed_reference_metadata_text_key(key: &str) -> bool {
matches!(
key,
"rid"
| "id"
| "identifier"
| "refid"
| "referenceid"
| "managedreferenceid"
| "managedreferenceids"
| "managedreferencesid"
| "managedreferencesids"
| "serializedreferenceid"
| "serializedreferenceids"
| "refids"
| "type"
| "typeid"
| "typeinfo"
| "typename"
| "fullname"
| "fulltypename"
| "class"
| "classname"
| "managedreferenceclassname"
| "serializedreferenceclassname"
| "klass"
| "managedtype"
| "managedreferencetype"
| "managedreferencefullname"
| "managedreferencefulltypename"
| "serializedreferencetype"
| "serializedreferencefullname"
| "serializedreferencefulltypename"
| "assemblyqualifiedname"
| "ns"
| "namespace"
| "namespacename"
| "managedreferencenamespace"
| "managedreferencenamespacename"
| "serializedreferencenamespace"
| "serializedreferencenamespacename"
| "asm"
| "asmname"
| "assembly"
| "assemblyname"
| "managedreferenceassembly"
| "managedreferenceassemblyname"
| "serializedreferenceassembly"
| "serializedreferenceassemblyname"
)
}
fn is_managed_reference_payload_text_key(key: &str) -> bool {
matches!(
key,
"data"
| "payload"
| "value"
| "object"
| "instance"
| "managedreferencepayload"
| "referencepayload"
| "serializedreferencepayload"
| "managedreferencevalue"
| "referencevalue"
| "serializedreferencevalue"
| "managedreferenceobject"
| "referenceobject"
| "serializedreferenceobject"
| "managedreferencedata"
| "referencedata"
| "serializeddata"
| "serializedreferencedata"
)
}
fn decode_text_payload(bytes: &[u8]) -> Option<(&'static str, String)> {
let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
if bytes.contains(&0) {
return None;
}
let text = std::str::from_utf8(bytes).ok()?.to_string();
if text.trim().is_empty()
|| text
.chars()
.any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t'))
{
return None;
}
let trimmed = text.trim();
if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
return Some(("json", text));
}
if text.lines().any(|line| line.contains('\t')) {
return Some(("tsv", text));
}
if text.lines().count() > 1 && text.lines().any(|line| line.contains(',')) {
return Some(("csv", text));
}
Some(("plain", text))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::serialized::UnitySerializedTextAsset;
use crate::types::{
ParsedAssetBundle, UnityFsBlockInfo, UnityFsDirectoryInfo, UnityFsHeader,
UnitySerializedParseError,
};
#[test]
fn extracts_textasset_and_writes_jsonl() {
let bundle = ParsedAssetBundle {
unity_version: "2021.3.56f2".to_string(),
assets: vec!["CAB-test".to_string()],
raw_data: Vec::new(),
unityfs_header: Some(UnityFsHeader {
format_version: 8,
target_version: "5.x.x".to_string(),
unity_version: "2021.3.56f2".to_string(),
total_size: 0,
compressed_blocks_info_size: 0,
uncompressed_blocks_info_size: 0,
flags: 0,
}),
blocks: vec![UnityFsBlockInfo {
uncompressed_size: 0,
compressed_size: 0,
flags: 0,
compression: crate::types::UnityFsCompression::None,
}],
directories: vec![UnityFsDirectoryInfo {
offset: 0,
size: 0,
flags: 0,
path: "CAB-test".to_string(),
}],
files: Vec::new(),
serialized_files: Vec::new(),
text_assets: vec![UnitySerializedTextAsset {
source_path: Some("CAB-test".to_string()),
path_id: 1,
name: "dialogue.json".to_string(),
bytes: br#"{"text":"hello"}"#.to_vec(),
}],
serialized_parse_errors: Vec::<UnitySerializedParseError>::new(),
};
let report = TextUnitExtractor::new().extract_bundle_with_context(
&bundle,
Some("dialogue.bundle"),
Some("assets/dialogue.bundle"),
);
assert_eq!(report.units.len(), 1);
assert_eq!(
report.units[0].archive_entry.as_deref(),
Some("assets/dialogue.bundle")
);
assert_eq!(
report.units[0].context.get("format"),
Some(&"json".to_string())
);
assert_eq!(
text_units_to_jsonl(&report.units).unwrap(),
format!("{}\n", serde_json::to_string(&report.units[0]).unwrap())
);
}
#[test]
fn extracts_managed_reference_payload_without_metadata_strings() {
let payload_field = string_field(
"m_ManagedReferences.references[0].data.message",
"message",
"こんにちは",
);
let metadata_field = string_field(
"m_ManagedReferences.references[0].managedReferenceFullTypeName",
"managedReferenceFullTypeName",
"Game BA.Text.ScenarioLine",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferenceRegistry".to_string(),
offset: 0,
byte_size: 64,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: vec![UnityManagedReferenceRecord {
metadata: UnityManagedReferenceMetadata {
reference_id: Some(42),
full_type_name: Some("Game BA.Text.ScenarioLine".to_string()),
type_name: Some("ScenarioLine".to_string()),
namespace: Some("BA.Text".to_string()),
assembly_name: Some("Game".to_string()),
},
fields: vec![payload_field.clone()],
}],
fields: vec![metadata_field],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.references[0].data.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_id"),
Some(&"42".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_full_type_name"),
Some(&"Game BA.Text.ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_payload_alias_without_metadata_strings() {
let payload_field = string_field(
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
"message",
"こんにちは",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 96,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
string_field(
"m_ManagedReferences.RefIds[0].serializedReferenceFullTypeName",
"serializedReferenceFullTypeName",
"Game BA.Text.ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeInfo",
"typeInfo",
"Game BA.Text.ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.className",
"className",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.namespaceName",
"namespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.asmName",
"asmName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
name: "managedReferenceData".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![payload_field]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_full_type_name"),
Some(&"Game BA.Text.ScenarioLine".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_payload_family_alias_with_context() {
let payload_field = string_field(
"m_ManagedReferences.RefIds[0].serializedReferencePayload.message",
"message",
"こんにちは",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 96,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].serializedReferencePayload"
.to_string(),
name: "serializedReferencePayload".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![payload_field]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].serializedReferencePayload.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_sibling_records_with_separate_context() {
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 160,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
name: "managedReferenceData".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![string_field(
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
"message",
"こんにちは",
)]),
},
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceClassName",
"managedReferenceClassName",
"ChoiceLine",
),
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[1].referencePayload".to_string(),
name: "referencePayload".to_string(),
type_name: "managedReference".to_string(),
offset: 120,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![string_field(
"m_ManagedReferences.RefIds[1].referencePayload.message",
"message",
"選択肢",
)]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 2);
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[1].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[1].referencePayload.message")
);
assert_eq!(units[1].source_text, "選択肢");
assert_eq!(
units[1].context.get("managed_reference_type"),
Some(&"ChoiceLine".to_string())
);
}
fn string_field(path: &str, name: &str, value: &str) -> UnitySerializedField {
UnitySerializedField {
path: path.to_string(),
name: name.to_string(),
type_name: "string".to_string(),
offset: 0,
byte_size: value.len() + 4,
type_tree_node_index: None,
value: UnitySerializedValue::String(value.to_string()),
}
}
}
+1
View File
@@ -14,6 +14,7 @@ bat-core = { path = "../core" }
bat-adapters = { path = "../adapters" } bat-adapters = { path = "../adapters" }
bat-assetbundle = { path = "../crates/bat-assetbundle" } bat-assetbundle = { path = "../crates/bat-assetbundle" }
bat-cas-engine = { path = "../crates/bat-cas-engine" } bat-cas-engine = { path = "../crates/bat-cas-engine" }
bat-patch = { path = "../crates/bat-patch" }
anyhow.workspace = true anyhow.workspace = true
thiserror.workspace = true thiserror.workspace = true
serde.workspace = true serde.workspace = true
File diff suppressed because it is too large Load Diff
+27 -2
View File
@@ -2,9 +2,10 @@
use bat_adapters::manifest::GenericManifest; use bat_adapters::manifest::GenericManifest;
use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry}; use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry};
use bat_core::domain::{Resource, ResourceEntry, ResourceType}; use bat_assetbundle::TextUnitExtractor;
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository}; use bat_core::repositories::{CasRepository, ResourceRepository};
use std::collections::BTreeMap; use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
/// 待导入的 AssetBundle 数据。 /// 待导入的 AssetBundle 数据。
@@ -97,6 +98,12 @@ pub struct UnityFsImportSummary {
pub text_assets: Vec<String>, pub text_assets: Vec<String>,
/// 非致命 serialized-file 解析诊断数量。 /// 非致命 serialized-file 解析诊断数量。
pub serialized_parse_error_count: usize, pub serialized_parse_error_count: usize,
/// 从 TextAsset 和 TypeTree 字段提取出的 TextUnit 数量。
pub text_unit_count: usize,
/// TextUnit 格式标签。
pub text_unit_formats: Vec<String>,
/// TextUnit 提取阶段的非致命诊断数量。
pub text_unit_error_count: usize,
} }
/// Manifest 导入报告。 /// Manifest 导入报告。
@@ -208,6 +215,7 @@ impl<'a> ResourceImportService<'a> {
id: resource_id_for_path(&entry.path), id: resource_id_for_path(&entry.path),
local_path: PathBuf::from(&entry.path), local_path: PathBuf::from(&entry.path),
entry: stored_entry, entry: stored_entry,
metadata: ResourceMetadata::default(),
}; };
let id = self.resources.add(resource).await?; let id = self.resources.add(resource).await?;
added_resources.push(id.clone()); added_resources.push(id.clone());
@@ -266,6 +274,14 @@ impl<'a> ResourceImportService<'a> {
manifest_path, error manifest_path, error
)) ))
})?; })?;
let text_units = TextUnitExtractor::new().extract_bundle(&parsed, Some(manifest_path));
let text_unit_formats = text_units
.units
.iter()
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
Ok(UnityFsImportSummary { Ok(UnityFsImportSummary {
unity_version: parsed.unity_version, unity_version: parsed.unity_version,
@@ -285,6 +301,9 @@ impl<'a> ResourceImportService<'a> {
.map(|asset| asset.name) .map(|asset| asset.name)
.collect(), .collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(), serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
text_unit_error_count: text_units.errors.len(),
}) })
} }
} }
@@ -715,6 +734,9 @@ mod tests {
assert_eq!(unityfs.text_asset_count, 0); assert_eq!(unityfs.text_asset_count, 0);
assert!(unityfs.text_assets.is_empty()); assert!(unityfs.text_assets.is_empty());
assert_eq!(unityfs.serialized_parse_error_count, 0); assert_eq!(unityfs.serialized_parse_error_count, 0);
assert_eq!(unityfs.text_unit_count, 0);
assert!(unityfs.text_unit_formats.is_empty());
assert_eq!(unityfs.text_unit_error_count, 0);
assert_eq!( assert_eq!(
report.imported[1].category, report.imported[1].category,
ResourceImportCategory::TextAsset ResourceImportCategory::TextAsset
@@ -799,6 +821,9 @@ mod tests {
assert_eq!(unityfs.text_asset_count, 1); assert_eq!(unityfs.text_asset_count, 1);
assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]); assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]);
assert_eq!(unityfs.serialized_parse_error_count, 0); assert_eq!(unityfs.serialized_parse_error_count, 0);
assert_eq!(unityfs.text_unit_count, 1);
assert_eq!(unityfs.text_unit_formats, vec!["plain".to_string()]);
assert_eq!(unityfs.text_unit_error_count, 0);
} }
#[tokio::test] #[tokio::test]
+47 -3
View File
@@ -13,13 +13,18 @@
pub mod cas; pub mod cas;
mod curl_transfer; mod curl_transfer;
pub mod import; pub mod import;
pub mod localized_patch;
pub mod official_changes;
pub mod official_download; pub mod official_download;
pub mod official_game_main_config; pub mod official_game_main_config;
pub mod official_launcher; pub mod official_launcher;
pub mod official_parse; pub mod official_parse;
pub mod official_pull; pub mod official_pull;
pub mod official_repository;
pub mod official_sync; pub mod official_sync;
pub mod official_textunit_queue;
pub mod official_update; pub mod official_update;
pub mod patch_ops;
pub mod path_security; pub mod path_security;
pub mod resources; pub mod resources;
mod zip_validation; mod zip_validation;
@@ -32,6 +37,24 @@ pub use import::{
BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport, BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport,
ResourceImportService, ResourceImportService,
}; };
pub use localized_patch::{
read_localized_patch_manifest_at, read_localized_version_state, LocalizedPatchConfig,
LocalizedPatchFile, LocalizedPatchIntegrity, LocalizedPatchManifest, LocalizedPatchOperation,
LocalizedPatchReport, LocalizedPatchRollbackInfo, LocalizedPatchService,
LocalizedTextAssetPatch, LocalizedVersionState, LOCALIZED_CURRENT_LINK,
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_PATCH_MANIFEST_VERSION, LOCALIZED_STAGING_DIR,
LOCALIZED_VERSIONS_DIR, LOCALIZED_VERSION_STATE_FILE,
};
pub use official_changes::{
read_resource_change_set_at, write_crowdin_translation_handoff_at,
write_official_resource_change_handoff, write_resource_change_set_at,
CrowdinTranslationHandoff, OfficialResourceChange, OfficialResourceChangeHandoffReport,
OfficialResourceChangeKind, OfficialResourceChangeSet, OfficialResourceChangeSummary,
OfficialResourceDescriptor, TranslationHandoffProvider, TranslationHandoffResource,
TranslationHandoffStatus, CROWDIN_TRANSLATION_HANDOFF_FILE,
CROWDIN_TRANSLATION_HANDOFF_VERSION, OFFICIAL_RESOURCE_CHANGES_FILE,
OFFICIAL_RESOURCE_CHANGES_VERSION,
};
pub use official_download::{ pub use official_download::{
read_download_manifest_at, DownloadError, OfficialDownloadManifest, read_download_manifest_at, DownloadError, OfficialDownloadManifest,
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem, OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
@@ -47,20 +70,35 @@ pub use official_launcher::{
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest, YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
}; };
pub use official_parse::{ pub use official_parse::{
read_parse_cache_at, write_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, query_textunit_index_errors, query_textunit_index_units, read_parse_cache_at,
OfficialParseCacheService, OfficialParseConfig, OfficialParseReport, read_textunit_index_at, write_parse_cache_at, write_textunit_index_at, OfficialParseCache,
OfficialParseCacheEntry, OfficialParseCacheService, OfficialParseConfig, OfficialParseReport,
OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus, OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus,
OfficialParseSummary, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION, OfficialParseSummary, OfficialTextUnitIndex, OfficialTextUnitIndexError,
OfficialTextUnitIndexSummary, OfficialTextUnitIndexUnit, OfficialTextUnitQuery,
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION, OFFICIAL_TEXTUNIT_INDEX_FILE,
OFFICIAL_TEXTUNIT_INDEX_VERSION,
}; };
pub use official_pull::{ pub use official_pull::{
build_official_pull_plan, build_official_pull_plan_for_platform_inventory, build_official_pull_plan, build_official_pull_plan_for_platform_inventory,
build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory, build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory,
OfficialResourcePullPlan, OfficialResourcePullPlan,
}; };
pub use official_repository::{
OfficialReleaseImportConfig, OfficialReleaseImportReport, OfficialReleaseImportService,
};
pub use official_sync::{ pub use official_sync::{
build_official_sync_plan, changed_endpoint_urls, classify_sync_decision, build_official_sync_plan, changed_endpoint_urls, classify_sync_decision,
default_official_platforms, OfficialSyncDecision, OfficialSyncPlan, default_official_platforms, OfficialSyncDecision, OfficialSyncPlan,
}; };
pub use official_textunit_queue::{
read_textunit_task_queue_at, write_crowdin_textunit_queue_at, write_official_textunit_queues,
write_textunit_task_queue_at, CrowdinTextUnitQueue, CrowdinTextUnitQueueItem,
OfficialTextUnitQueueReport, OfficialTextUnitTask, OfficialTextUnitTaskQueue,
OfficialTextUnitTaskStatus, OfficialTextUnitTaskSummary, CROWDIN_TEXTUNIT_QUEUE_FILE,
CROWDIN_TEXTUNIT_QUEUE_VERSION, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE,
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
};
pub use official_update::{ pub use official_update::{
cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging, cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging,
read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot, read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot,
@@ -71,6 +109,12 @@ pub use official_update::{
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary, OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap, OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap,
}; };
pub use patch_ops::{
apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file,
apply_unityfs_text_asset_patch_file, PatchApplyKind, PatchApplyParams, PatchApplyReport,
UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams,
UnityFsTextAssetPatchParams,
};
pub use path_security::{ pub use path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute, ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
open_append_file, read_file_no_symlink, set_file_mode, validate_output_root, open_append_file, read_file_no_symlink, set_file_mode, validate_output_root,
+857
View File
@@ -0,0 +1,857 @@
//! Localized release publishing for verified TextAsset patches.
use bat_assetbundle::{patch_unityfs_text_asset, TextAssetPatch};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
read_file_no_symlink, write_file_atomic, STATE_FILE_MODE,
};
/// Atomic current pointer under the localized output root.
pub const LOCALIZED_CURRENT_LINK: &str = "current";
/// Staging directory under the localized output root.
pub const LOCALIZED_STAGING_DIR: &str = ".staging";
/// Version directory under the localized output root.
pub const LOCALIZED_VERSIONS_DIR: &str = "versions";
/// Persisted localized release state file name.
pub const LOCALIZED_VERSION_STATE_FILE: &str = "localized-version-state.json";
/// Per-release patch manifest file name.
pub const LOCALIZED_PATCH_MANIFEST_FILE: &str = "localized-patch-manifest.json";
/// Current localized patch manifest schema version.
pub const LOCALIZED_PATCH_MANIFEST_VERSION: u32 = 1;
/// One patch operation against a bundle in an official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedTextAssetPatch {
/// Relative path of the UnityFS bundle under the official release.
pub bundle_path: String,
/// TextAsset replacement inside the bundle.
pub text_asset: TextAssetPatch,
}
/// Configuration for one localized release publication.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedPatchConfig {
/// Immutable, verified official release root.
pub official_release_root: PathBuf,
/// Separate localized publication root.
pub localized_output_root: PathBuf,
/// Version identifier shared with the official release.
pub release_id: String,
/// Patch operations to apply.
pub patches: Vec<LocalizedTextAssetPatch>,
}
impl LocalizedPatchConfig {
/// Creates a localized patch configuration.
pub fn new(
official_release_root: impl Into<PathBuf>,
localized_output_root: impl Into<PathBuf>,
release_id: impl Into<String>,
patches: Vec<LocalizedTextAssetPatch>,
) -> Self {
Self {
official_release_root: official_release_root.into(),
localized_output_root: localized_output_root.into(),
release_id: release_id.into(),
patches,
}
}
}
/// Persisted localized release state.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedVersionState {
/// State schema version.
pub state_version: u32,
/// Official release ID used as the patch source.
pub official_release_id: String,
/// Published localized release ID.
pub current_release_id: Option<String>,
/// Stable status label.
pub status: String,
/// Last update time.
pub updated_unix_seconds: u64,
}
/// One changed file in a localized patch manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchFile {
/// Relative bundle path.
pub path: String,
/// BLAKE3 before applying the patch.
pub original_blake3: String,
/// BLAKE3 after applying the patch.
pub localized_blake3: String,
/// Original file size in bytes.
#[serde(default)]
pub original_bytes: u64,
/// Localized file size in bytes.
#[serde(default)]
pub localized_bytes: u64,
/// Localized minus original byte size.
#[serde(default)]
pub byte_delta: i64,
/// TextAsset operations applied to this file.
#[serde(default)]
pub text_asset_operations: Vec<LocalizedPatchOperation>,
}
/// One TextAsset patch operation recorded in the localized patch manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchOperation {
/// UnityFS directory path of the serialized file.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Expected TextAsset name, when provided.
pub expected_name: Option<String>,
/// Replacement payload size.
pub replacement_bytes: u64,
/// BLAKE3 of the replacement payload.
pub replacement_blake3: String,
}
/// Rollback information recorded for a localized publication.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchRollbackInfo {
/// Previous `current` symlink target before this publication.
pub previous_current_target: Option<PathBuf>,
/// Version directory that should be removed when rolling this publication back.
pub remove_version_path: PathBuf,
}
/// Integrity summary for a published localized release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchIntegrity {
/// Number of changed files verified against the manifest.
pub verified_changed_file_count: usize,
/// Number of TextAsset operations recorded in the manifest.
pub verified_text_asset_operation_count: usize,
/// Whether `current` points at this localized release.
pub current_points_to_release: bool,
}
/// Persisted manifest for one localized release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchManifest {
/// Manifest schema version.
#[serde(default = "default_patch_manifest_version")]
pub manifest_version: u32,
/// Official release ID used as the patch source.
pub official_release_id: String,
/// Published localized release ID.
pub localized_release_id: String,
/// Manifest generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Changed file count.
pub file_count: usize,
/// TextAsset operation count.
pub text_asset_operation_count: usize,
/// Changed files and their before/after hashes.
pub files: Vec<LocalizedPatchFile>,
/// Rollback information for this release.
pub rollback: LocalizedPatchRollbackInfo,
}
impl LocalizedPatchManifest {
/// Converts the localized TextAsset manifest to the generic patch manifest model.
pub fn to_patch_manifest(&self) -> bat_patch::PatchManifest {
bat_patch::PatchManifest {
version: bat_patch::PATCH_MANIFEST_VERSION,
patch_id: self.localized_release_id.clone(),
source_version: self.official_release_id.clone(),
target_version: self.localized_release_id.clone(),
files: self
.files
.iter()
.map(|file| bat_patch::PatchManifestFile {
path: PathBuf::from(&file.path),
patch_kind: bat_patch::PatchKind::UnityFsTextAsset,
source_blake3: file.original_blake3.clone(),
target_blake3: file.localized_blake3.clone(),
source_size: file.original_bytes,
target_size: file.localized_bytes,
})
.collect(),
rollback: bat_patch::PatchRollback {
previous_current_target: self.rollback.previous_current_target.clone(),
remove_target_path: Some(self.rollback.remove_version_path.clone()),
},
}
}
}
/// Result of a successful localized release publication.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedPatchReport {
/// Published version directory.
pub version_path: PathBuf,
/// Atomic current pointer.
pub current_path: PathBuf,
/// Version state path.
pub state_path: PathBuf,
/// Patch manifest path.
pub patch_manifest_path: PathBuf,
/// Changed files.
pub files: Vec<LocalizedPatchFile>,
/// Persisted patch manifest.
pub manifest: LocalizedPatchManifest,
/// Integrity check performed after publication.
pub integrity: LocalizedPatchIntegrity,
}
/// Applies TextAsset patches and atomically publishes a localized release.
#[derive(Debug, Default, Clone, Copy)]
pub struct LocalizedPatchService;
impl LocalizedPatchService {
/// Creates the publisher.
pub fn new() -> Self {
Self
}
/// Copies the official release, applies patches in staging and publishes it.
pub fn publish(&self, config: &LocalizedPatchConfig) -> anyhow::Result<LocalizedPatchReport> {
let staging = config
.localized_output_root
.join(LOCALIZED_STAGING_DIR)
.join(&config.release_id);
let version_path = config
.localized_output_root
.join(LOCALIZED_VERSIONS_DIR)
.join(&config.release_id);
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
let previous_current_target = current_symlink_target(&current_path).ok().flatten();
let version_existed_before = version_path.exists();
match self.publish_inner(config, previous_current_target.clone()) {
Ok(report) => Ok(report),
Err(error) => {
if let Err(rollback_error) = rollback_failed_publish(
&config.localized_output_root,
&staging,
&version_path,
!version_existed_before,
&current_path,
previous_current_target.as_ref(),
) {
return Err(anyhow::anyhow!(
"{error}; rollback failed: {rollback_error}"
));
}
Err(error)
}
}
}
fn publish_inner(
&self,
config: &LocalizedPatchConfig,
previous_current_target: Option<PathBuf>,
) -> anyhow::Result<LocalizedPatchReport> {
validate_config(config).map_err(anyhow::Error::msg)?;
let staging = config
.localized_output_root
.join(LOCALIZED_STAGING_DIR)
.join(&config.release_id);
let version_path = config
.localized_output_root
.join(LOCALIZED_VERSIONS_DIR)
.join(&config.release_id);
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
let state_path = config
.localized_output_root
.join(LOCALIZED_VERSION_STATE_FILE);
let patch_manifest_path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
if version_path.exists() {
return Err(anyhow::anyhow!(
"localized release already exists: {}",
version_path.display()
));
}
remove_owned_staging(&staging)?;
fs::create_dir_all(&staging)?;
copy_tree(&config.official_release_root, &staging)?;
let mut changed_files = Vec::with_capacity(config.patches.len());
for operation in &config.patches {
let target = staging.join(Path::new(&operation.bundle_path));
ensure_path_within_root(&staging, &target).map_err(anyhow::Error::msg)?;
ensure_safe_file_target(&staging, &target, "汉化 patch 输入")
.map_err(anyhow::Error::msg)?;
let original = fs::read(&target)?;
let patched = patch_unityfs_text_asset(&original, &operation.text_asset)
.map_err(|error| anyhow::anyhow!("{}: {error}", operation.bundle_path))?;
if original == patched {
return Err(anyhow::anyhow!(
"patch produced no change: {}",
operation.bundle_path
));
}
write_file_atomic(&target, &patched, STATE_FILE_MODE, "汉化 patch 输出")
.map_err(anyhow::Error::msg)?;
let original_blake3 = blake3::hash(&original).to_hex().to_string();
let localized_blake3 = blake3::hash(&patched).to_hex().to_string();
changed_files.push(LocalizedPatchFile {
path: operation.bundle_path.clone(),
original_blake3,
localized_blake3,
original_bytes: original.len() as u64,
localized_bytes: patched.len() as u64,
byte_delta: patched.len() as i64 - original.len() as i64,
text_asset_operations: vec![LocalizedPatchOperation::from_text_asset_patch(
&operation.text_asset,
)],
});
}
let manifest = LocalizedPatchManifest {
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
official_release_id: config.release_id.clone(),
localized_release_id: config.release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
file_count: changed_files.len(),
text_asset_operation_count: changed_files
.iter()
.map(|file| file.text_asset_operations.len())
.sum(),
files: changed_files.clone(),
rollback: LocalizedPatchRollbackInfo {
previous_current_target: previous_current_target.clone(),
remove_version_path: version_path.clone(),
},
};
write_file_atomic(
&staging.join(LOCALIZED_PATCH_MANIFEST_FILE),
&serde_json::to_vec_pretty(&manifest)?,
STATE_FILE_MODE,
"汉化 patch manifest",
)
.map_err(anyhow::Error::msg)?;
verify_patch_manifest_files(&config.official_release_root, &staging, &manifest)?;
fs::create_dir_all(config.localized_output_root.join(LOCALIZED_VERSIONS_DIR))?;
fs::rename(&staging, &version_path)?;
switch_current_symlink(
&config.localized_output_root,
&current_path,
&config.release_id,
)?;
let state = LocalizedVersionState {
state_version: 1,
official_release_id: config.release_id.clone(),
current_release_id: Some(config.release_id.clone()),
status: "localized".to_string(),
updated_unix_seconds: unix_seconds_now(),
};
write_file_atomic(
&state_path,
&serde_json::to_vec_pretty(&state)?,
STATE_FILE_MODE,
"汉化版本状态",
)
.map_err(anyhow::Error::msg)?;
let integrity = verify_published_localized_release(
&config.official_release_root,
&version_path,
&current_path,
)?;
Ok(LocalizedPatchReport {
version_path,
current_path,
state_path,
patch_manifest_path,
files: changed_files,
manifest,
integrity,
})
}
}
/// Reads the localized release state without following a symlink at the file
/// path. A missing state file means no localized release has been published.
pub fn read_localized_version_state(
localized_output_root: &Path,
) -> anyhow::Result<Option<LocalizedVersionState>> {
let path = localized_output_root.join(LOCALIZED_VERSION_STATE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "汉化版本状态").map_err(anyhow::Error::msg)?
else {
return Ok(None);
};
let state: LocalizedVersionState = serde_json::from_slice(&bytes)?;
if state.state_version != 1 {
return Err(anyhow::anyhow!(
"不支持的汉化版本状态 schema:{},当前版本=1",
state.state_version
));
}
Ok(Some(state))
}
/// Reads a localized patch manifest from a published version directory.
pub fn read_localized_patch_manifest_at(
version_path: &Path,
) -> anyhow::Result<Option<LocalizedPatchManifest>> {
let path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
let Some(bytes) =
read_file_no_symlink(&path, "汉化 patch manifest").map_err(anyhow::Error::msg)?
else {
return Ok(None);
};
let manifest: LocalizedPatchManifest = serde_json::from_slice(&bytes)?;
if manifest.manifest_version != LOCALIZED_PATCH_MANIFEST_VERSION {
return Err(anyhow::anyhow!(
"不支持的汉化 patch manifest schema{},当前版本={}",
manifest.manifest_version,
LOCALIZED_PATCH_MANIFEST_VERSION
));
}
Ok(Some(manifest))
}
impl LocalizedPatchOperation {
fn from_text_asset_patch(patch: &TextAssetPatch) -> Self {
Self {
serialized_file_path: patch.serialized_file_path.clone(),
path_id: patch.path_id,
expected_name: patch.expected_name.clone(),
replacement_bytes: patch.replacement.len() as u64,
replacement_blake3: blake3::hash(&patch.replacement).to_hex().to_string(),
}
}
}
fn verify_published_localized_release(
official_release_root: &Path,
version_path: &Path,
current_path: &Path,
) -> anyhow::Result<LocalizedPatchIntegrity> {
let manifest = read_localized_patch_manifest_at(version_path)?.ok_or_else(|| {
anyhow::anyhow!(
"缺少汉化 patch manifest{}",
version_path.join(LOCALIZED_PATCH_MANIFEST_FILE).display()
)
})?;
let mut integrity =
verify_patch_manifest_files(official_release_root, version_path, &manifest)?;
integrity.current_points_to_release = current_points_to_version(current_path, version_path)?;
if !integrity.current_points_to_release {
return Err(anyhow::anyhow!(
"汉化 current 未指向发布版本:current={} version={}",
current_path.display(),
version_path.display()
));
}
Ok(integrity)
}
fn verify_patch_manifest_files(
official_release_root: &Path,
localized_release_root: &Path,
manifest: &LocalizedPatchManifest,
) -> anyhow::Result<LocalizedPatchIntegrity> {
let mut operation_count = 0usize;
for file in &manifest.files {
let relative = Path::new(&file.path);
let official_path = official_release_root.join(relative);
let localized_path = localized_release_root.join(relative);
ensure_path_within_root(official_release_root, &official_path)
.map_err(anyhow::Error::msg)?;
ensure_path_within_root(localized_release_root, &localized_path)
.map_err(anyhow::Error::msg)?;
ensure_safe_file_target(official_release_root, &official_path, "官方 patch 原文件")
.map_err(anyhow::Error::msg)?;
ensure_safe_file_target(localized_release_root, &localized_path, "汉化 patch 产物")
.map_err(anyhow::Error::msg)?;
let original = fs::read(&official_path)?;
let localized = fs::read(&localized_path)?;
let original_hash = blake3::hash(&original).to_hex().to_string();
let localized_hash = blake3::hash(&localized).to_hex().to_string();
if original_hash != file.original_blake3 || original.len() as u64 != file.original_bytes {
return Err(anyhow::anyhow!(
"汉化 manifest 原文件校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
file.path,
file.original_blake3,
file.original_bytes,
original_hash,
original.len()
));
}
if localized_hash != file.localized_blake3 || localized.len() as u64 != file.localized_bytes
{
return Err(anyhow::anyhow!(
"汉化 manifest 产物校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
file.path,
file.localized_blake3,
file.localized_bytes,
localized_hash,
localized.len()
));
}
if localized_hash == original_hash {
return Err(anyhow::anyhow!(
"汉化 manifest 文件未发生变化:{}",
file.path
));
}
operation_count += file.text_asset_operations.len();
}
if manifest.file_count != manifest.files.len() {
return Err(anyhow::anyhow!(
"汉化 manifest file_count 不一致:声明 {},实际 {}",
manifest.file_count,
manifest.files.len()
));
}
if manifest.text_asset_operation_count != operation_count {
return Err(anyhow::anyhow!(
"汉化 manifest operation_count 不一致:声明 {},实际 {}",
manifest.text_asset_operation_count,
operation_count
));
}
Ok(LocalizedPatchIntegrity {
verified_changed_file_count: manifest.files.len(),
verified_text_asset_operation_count: operation_count,
current_points_to_release: false,
})
}
fn current_symlink_target(current_path: &Path) -> anyhow::Result<Option<PathBuf>> {
match fs::symlink_metadata(current_path) {
Ok(metadata) if metadata.file_type().is_symlink() => Ok(Some(fs::read_link(current_path)?)),
Ok(_) => Err(anyhow::anyhow!(
"汉化 current 已存在但不是 symlink{}",
current_path.display()
)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error.into()),
}
}
fn current_points_to_version(current_path: &Path, version_path: &Path) -> anyhow::Result<bool> {
let metadata = fs::symlink_metadata(current_path)?;
if !metadata.file_type().is_symlink() {
return Ok(false);
}
Ok(fs::canonicalize(current_path)? == fs::canonicalize(version_path)?)
}
fn rollback_failed_publish(
localized_output_root: &Path,
staging: &Path,
version_path: &Path,
remove_version_path: bool,
current_path: &Path,
previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
remove_owned_path(staging)?;
if remove_version_path {
remove_owned_path(version_path)?;
}
remove_owned_path(&localized_output_root.join(".current.tmp"))?;
remove_owned_path(&localized_output_root.join(".current.rollback.tmp"))?;
let failed_target = version_path
.file_name()
.map(|release_id| Path::new(LOCALIZED_VERSIONS_DIR).join(release_id));
if failed_target.as_ref().is_some_and(|target| {
fs::read_link(current_path)
.map(|current_target| current_target == *target)
.unwrap_or(false)
}) {
restore_current_symlink(localized_output_root, current_path, previous_current_target)?;
}
Ok(())
}
fn remove_owned_path(path: &Path) -> anyhow::Result<()> {
if let Ok(metadata) = fs::symlink_metadata(path) {
if metadata.file_type().is_symlink() || metadata.is_file() {
fs::remove_file(path)?;
} else if metadata.is_dir() {
fs::remove_dir_all(path)?;
}
}
Ok(())
}
#[cfg(unix)]
fn restore_current_symlink(
root: &Path,
current_path: &Path,
previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
use std::os::unix::fs::symlink;
if let Some(previous_target) = previous_current_target {
let temporary = root.join(".current.rollback.tmp");
remove_owned_path(&temporary)?;
symlink(previous_target, &temporary)?;
fs::rename(temporary, current_path)?;
} else if fs::symlink_metadata(current_path)
.map(|metadata| metadata.file_type().is_symlink())
.unwrap_or(false)
{
fs::remove_file(current_path)?;
}
Ok(())
}
#[cfg(not(unix))]
fn restore_current_symlink(
_root: &Path,
_current_path: &Path,
_previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
Ok(())
}
fn validate_config(config: &LocalizedPatchConfig) -> Result<(), String> {
let official = lexical_absolute(&config.official_release_root)?;
let localized = lexical_absolute(&config.localized_output_root)?;
if official == localized || official.starts_with(&localized) || localized.starts_with(&official)
{
return Err(format!(
"官方 release 与汉化输出目录不能相同或互相嵌套:官方={} 汉化={}",
official.display(),
localized.display()
));
}
if config.release_id.is_empty()
|| config.release_id.contains('/')
|| config.release_id.contains('\\')
|| config.release_id == "."
|| config.release_id == ".."
{
return Err(format!("非法汉化 release id{}", config.release_id));
}
ensure_safe_directory_path(&config.official_release_root, "官方 release")?;
ensure_safe_directory_path(&config.localized_output_root, "汉化输出目录")?;
Ok(())
}
fn copy_tree(source: &Path, destination: &Path) -> anyhow::Result<()> {
let metadata = fs::symlink_metadata(source)?;
if metadata.file_type().is_symlink() {
return Err(anyhow::anyhow!(
"官方 release 不能包含 symlink: {}",
source.display()
));
}
if metadata.is_dir() {
fs::create_dir_all(destination)?;
for entry in fs::read_dir(source)? {
let entry = entry?;
copy_tree(&entry.path(), &destination.join(entry.file_name()))?;
}
} else if metadata.is_file() {
if let Some(parent) = destination.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(source, destination)?;
} else {
return Err(anyhow::anyhow!(
"官方 release 中存在非普通文件: {}",
source.display()
));
}
Ok(())
}
fn remove_owned_staging(path: &Path) -> anyhow::Result<()> {
if let Ok(metadata) = fs::symlink_metadata(path) {
if metadata.file_type().is_symlink() {
return Err(anyhow::anyhow!(
"汉化 staging 不能是 symlink: {}",
path.display()
));
}
if metadata.is_dir() {
fs::remove_dir_all(path)?;
} else {
fs::remove_file(path)?;
}
}
Ok(())
}
#[cfg(unix)]
fn switch_current_symlink(root: &Path, current: &Path, release_id: &str) -> anyhow::Result<()> {
use std::os::unix::fs::symlink;
let temporary = root.join(".current.tmp");
if let Ok(metadata) = fs::symlink_metadata(&temporary) {
if metadata.file_type().is_symlink() || metadata.is_file() {
fs::remove_file(&temporary)?;
} else if metadata.is_dir() {
fs::remove_dir_all(&temporary)?;
}
}
symlink(
Path::new(LOCALIZED_VERSIONS_DIR).join(release_id),
&temporary,
)?;
fs::rename(temporary, current)?;
Ok(())
}
#[cfg(not(unix))]
fn switch_current_symlink(_root: &Path, _current: &Path, _release_id: &str) -> anyhow::Result<()> {
Err(anyhow::anyhow!(
"localized release publication requires a Unix symlink-capable platform"
))
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_patch_manifest_version() -> u32 {
LOCALIZED_PATCH_MANIFEST_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn localized_manifest_converts_to_generic_patch_manifest() {
let source = b"source";
let target = b"target";
let manifest = LocalizedPatchManifest {
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
official_release_id: "official-v1".to_string(),
localized_release_id: "localized-v1".to_string(),
generated_unix_seconds: 123,
file_count: 1,
text_asset_operation_count: 1,
files: vec![LocalizedPatchFile {
path: "Bundles/file.bundle".to_string(),
original_blake3: blake3::hash(source).to_hex().to_string(),
localized_blake3: blake3::hash(target).to_hex().to_string(),
original_bytes: source.len() as u64,
localized_bytes: target.len() as u64,
byte_delta: target.len() as i64 - source.len() as i64,
text_asset_operations: vec![LocalizedPatchOperation {
serialized_file_path: "CAB-asset".to_string(),
path_id: 1,
expected_name: Some("Text".to_string()),
replacement_bytes: target.len() as u64,
replacement_blake3: blake3::hash(target).to_hex().to_string(),
}],
}],
rollback: LocalizedPatchRollbackInfo {
previous_current_target: Some(PathBuf::from("versions/previous")),
remove_version_path: PathBuf::from("versions/localized-v1"),
},
};
let patch_manifest = manifest.to_patch_manifest();
assert_eq!(patch_manifest.version, bat_patch::PATCH_MANIFEST_VERSION);
assert_eq!(patch_manifest.patch_id, "localized-v1");
assert_eq!(patch_manifest.source_version, "official-v1");
assert_eq!(patch_manifest.target_version, "localized-v1");
assert_eq!(patch_manifest.files.len(), 1);
assert_eq!(
patch_manifest.files[0].patch_kind,
bat_patch::PatchKind::UnityFsTextAsset
);
assert_eq!(
patch_manifest.rollback.previous_current_target,
Some(PathBuf::from("versions/previous"))
);
assert_eq!(
patch_manifest.rollback.remove_target_path,
Some(PathBuf::from("versions/localized-v1"))
);
}
#[cfg(unix)]
#[test]
fn publishes_a_separate_localized_release_atomically() {
let temp = TempDir::new().unwrap();
let official = temp.path().join("official-release");
let localized = temp.path().join("localized");
fs::create_dir_all(official.join("TableBundles")).unwrap();
fs::write(
official.join("TableBundles/TableCatalog.bytes"),
b"official",
)
.unwrap();
let report = LocalizedPatchService::new()
.publish(&LocalizedPatchConfig::new(
&official,
&localized,
"release-1",
Vec::new(),
))
.unwrap();
assert_eq!(
fs::read(report.version_path.join("TableBundles/TableCatalog.bytes")).unwrap(),
b"official"
);
assert_eq!(
fs::read_link(report.current_path).unwrap(),
PathBuf::from("versions/release-1")
);
let state: LocalizedVersionState =
serde_json::from_slice(&fs::read(report.state_path).unwrap()).unwrap();
assert_eq!(state.status, "localized");
assert_eq!(state.current_release_id.as_deref(), Some("release-1"));
assert!(report.patch_manifest_path.is_file());
assert_eq!(report.manifest.file_count, 0);
assert_eq!(report.integrity.verified_changed_file_count, 0);
assert!(report.integrity.current_points_to_release);
let manifest = read_localized_patch_manifest_at(&report.version_path)
.unwrap()
.unwrap();
assert_eq!(manifest.localized_release_id, "release-1");
assert_eq!(manifest.rollback.previous_current_target, None);
}
#[cfg(unix)]
#[test]
fn failed_patch_publish_cleans_staging_and_unpublished_version() {
let temp = TempDir::new().unwrap();
let official = temp.path().join("official-release");
let localized = temp.path().join("localized");
fs::create_dir_all(official.join("Bundles")).unwrap();
fs::write(official.join("Bundles/bad.bundle"), b"not-unityfs").unwrap();
let error = LocalizedPatchService::new()
.publish(&LocalizedPatchConfig::new(
&official,
&localized,
"release-1",
vec![LocalizedTextAssetPatch {
bundle_path: "Bundles/bad.bundle".to_string(),
text_asset: TextAssetPatch::new("CAB-bad", 1, b"replacement".to_vec()),
}],
))
.unwrap_err();
assert!(error.to_string().contains("Bundles/bad.bundle"));
assert!(!localized
.join(LOCALIZED_STAGING_DIR)
.join("release-1")
.exists());
assert!(!localized
.join(LOCALIZED_VERSIONS_DIR)
.join("release-1")
.exists());
assert!(!localized.join(LOCALIZED_CURRENT_LINK).exists());
}
}
+653
View File
@@ -0,0 +1,653 @@
//! Official resource change sets and translation handoff files.
//!
//! This module is intentionally file-based. Official update generates the
//! durable change set after a new release has been fully downloaded and
//! verified; parser and translation modules can then consume the same immutable
//! handoff without depending on daemon internals.
use crate::official_download::{
read_download_manifest_at, OfficialDownloadManifest, OfficialDownloadManifestEntry,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
/// Current resource-change-set schema version.
pub const OFFICIAL_RESOURCE_CHANGES_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const OFFICIAL_RESOURCE_CHANGES_FILE: &str = "official-resource-changes.json";
/// Current Crowdin handoff schema version.
pub const CROWDIN_TRANSLATION_HANDOFF_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const CROWDIN_TRANSLATION_HANDOFF_FILE: &str = "crowdin-translation-handoff.json";
/// Change kind for one official resource destination.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialResourceChangeKind {
/// Destination did not exist in the previous complete release.
Added,
/// Destination exists in both releases, but verified size or BLAKE3 changed.
Modified,
/// Destination existed in the previous release but is absent from the new one.
Removed,
}
impl OfficialResourceChangeKind {
/// Returns the stable JSON/RPC label for the change kind.
pub fn as_str(self) -> &'static str {
match self {
Self::Added => "added",
Self::Modified => "modified",
Self::Removed => "removed",
}
}
/// Returns true when the changed resource should be offered to parser and
/// translation modules.
pub fn is_incremental_candidate(self) -> bool {
matches!(self, Self::Added | Self::Modified)
}
}
/// Verified manifest attributes for one official resource.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceDescriptor {
/// Official URL from the download manifest.
pub url: String,
/// Relative path under the official release root.
pub destination: String,
/// Verified byte count from the download manifest.
pub bytes: u64,
/// Verified BLAKE3 digest from the download manifest.
pub blake3: String,
}
impl From<&OfficialDownloadManifestEntry> for OfficialResourceDescriptor {
fn from(entry: &OfficialDownloadManifestEntry) -> Self {
Self {
url: entry.url.clone(),
destination: entry.destination.clone(),
bytes: entry.bytes,
blake3: entry.blake3.clone(),
}
}
}
/// One resource-level change between two complete official releases.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChange {
/// Relative path used as the stable comparison key.
pub destination: String,
/// Change kind for this destination.
pub kind: OfficialResourceChangeKind,
/// Previous release descriptor, when the destination existed before.
pub previous: Option<OfficialResourceDescriptor>,
/// Current release descriptor, when the destination exists now.
pub current: Option<OfficialResourceDescriptor>,
/// Whether parser modules should inspect this resource in incremental mode.
pub parse_candidate: bool,
/// Whether translation modules should enqueue this resource in incremental mode.
pub translation_candidate: bool,
}
impl OfficialResourceChange {
fn new(
destination: String,
kind: OfficialResourceChangeKind,
previous: Option<OfficialResourceDescriptor>,
current: Option<OfficialResourceDescriptor>,
) -> Self {
let is_candidate = kind.is_incremental_candidate();
Self {
destination,
kind,
previous,
current,
parse_candidate: is_candidate,
translation_candidate: is_candidate,
}
}
}
/// Aggregate counters for one official resource change set.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeSummary {
/// Whether a previous complete release manifest was available.
pub previous_manifest_present: bool,
/// Number of entries in the previous release manifest.
pub previous_manifest_entry_count: usize,
/// Number of entries in the current release manifest.
pub current_manifest_entry_count: usize,
/// Number of newly added destinations.
pub added_count: usize,
/// Number of destinations whose verified bytes or BLAKE3 changed.
pub modified_count: usize,
/// Number of destinations removed from the current release.
pub removed_count: usize,
/// Number of resources to offer to parser modules.
pub parse_candidate_count: usize,
/// Number of resources to offer to translation modules.
pub translation_candidate_count: usize,
}
/// Durable comparison result between a previous complete official release and
/// the newly published official release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeSet {
/// Change-set schema version.
#[serde(default = "default_resource_changes_version")]
pub change_set_version: u32,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Previous complete release root, when available.
pub previous_resource_root: Option<PathBuf>,
/// Current complete release root.
pub current_resource_root: PathBuf,
/// Aggregate counters.
pub summary: OfficialResourceChangeSummary,
/// Stable, destination-sorted list of changed resources.
pub changes: Vec<OfficialResourceChange>,
}
impl OfficialResourceChangeSet {
/// Builds a change set from already loaded manifests.
pub fn from_manifests(
official_release_id: impl Into<String>,
previous_release_id: Option<String>,
previous_resource_root: Option<PathBuf>,
current_resource_root: PathBuf,
previous_manifest: Option<&OfficialDownloadManifest>,
current_manifest: &OfficialDownloadManifest,
) -> Self {
let previous_by_destination = previous_manifest
.map(entries_by_destination)
.unwrap_or_default();
let current_by_destination = entries_by_destination(current_manifest);
let destinations = previous_by_destination
.keys()
.chain(current_by_destination.keys())
.cloned()
.collect::<BTreeSet<_>>();
let mut changes = Vec::new();
let mut summary = OfficialResourceChangeSummary {
previous_manifest_present: previous_manifest.is_some(),
previous_manifest_entry_count: previous_manifest
.map(|manifest| manifest.entries.len())
.unwrap_or(0),
current_manifest_entry_count: current_manifest.entries.len(),
..OfficialResourceChangeSummary::default()
};
for destination in destinations {
match (
previous_by_destination.get(&destination),
current_by_destination.get(&destination),
) {
(None, Some(current)) => {
summary.added_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Added,
None,
Some((*current).into()),
));
}
(Some(previous), Some(current)) if content_changed(previous, current) => {
summary.modified_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Modified,
Some((*previous).into()),
Some((*current).into()),
));
}
(Some(previous), None) => {
summary.removed_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Removed,
Some((*previous).into()),
None,
));
}
_ => {}
}
}
summary.parse_candidate_count = changes
.iter()
.filter(|change| change.parse_candidate)
.count();
summary.translation_candidate_count = changes
.iter()
.filter(|change| change.translation_candidate)
.count();
Self {
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
official_release_id: official_release_id.into(),
previous_release_id,
generated_unix_seconds: unix_seconds_now(),
previous_resource_root,
current_resource_root,
summary,
changes,
}
}
/// Returns the resources that parser modules should inspect for
/// incremental work.
pub fn parse_candidates(&self) -> Vec<&OfficialResourceChange> {
self.changes
.iter()
.filter(|change| change.parse_candidate)
.collect()
}
/// Returns the resources that translation modules should enqueue.
pub fn translation_candidates(&self) -> Vec<&OfficialResourceChange> {
self.changes
.iter()
.filter(|change| change.translation_candidate)
.collect()
}
}
/// Provider reserved for translation handoff consumers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationHandoffProvider {
/// Crowdin provider. The handoff file does not make a network request.
Crowdin,
}
impl TranslationHandoffProvider {
/// Returns the stable provider label.
pub fn as_str(self) -> &'static str {
match self {
Self::Crowdin => "crowdin",
}
}
}
/// Status of a generated translation handoff.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationHandoffStatus {
/// The handoff was written locally and is waiting for a translation worker.
QueuedOffline,
}
impl TranslationHandoffStatus {
/// Returns the stable status label.
pub fn as_str(self) -> &'static str {
match self {
Self::QueuedOffline => "queued_offline",
}
}
}
/// One resource entry queued for translation-provider processing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TranslationHandoffResource {
/// Relative path under the official release root.
pub destination: String,
/// Change kind that caused this resource to be queued.
pub kind: OfficialResourceChangeKind,
/// Current official URL.
pub url: String,
/// Verified byte count.
pub bytes: u64,
/// Verified BLAKE3 digest.
pub blake3: String,
}
/// Crowdin-ready local queue file for added or modified official resources.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTranslationHandoff {
/// Handoff schema version.
#[serde(default = "default_crowdin_handoff_version")]
pub handoff_version: u32,
/// Translation provider reserved for this queue.
pub provider: TranslationHandoffProvider,
/// Current queue status.
pub status: TranslationHandoffStatus,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Number of queued resources.
pub resource_count: usize,
/// Added or modified resources to pass into parsing/translation workers.
pub resources: Vec<TranslationHandoffResource>,
}
impl CrowdinTranslationHandoff {
/// Builds a local Crowdin handoff from a resource change set.
pub fn from_change_set(change_set: &OfficialResourceChangeSet) -> Self {
let resources = change_set
.translation_candidates()
.into_iter()
.filter_map(|change| {
let current = change.current.as_ref()?;
Some(TranslationHandoffResource {
destination: change.destination.clone(),
kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
})
})
.collect::<Vec<_>>();
Self {
handoff_version: CROWDIN_TRANSLATION_HANDOFF_VERSION,
provider: TranslationHandoffProvider::Crowdin,
status: TranslationHandoffStatus::QueuedOffline,
official_release_id: change_set.official_release_id.clone(),
previous_release_id: change_set.previous_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
resource_count: resources.len(),
resources,
}
}
}
/// Generates a change set for two complete release roots and writes both the
/// change set and the Crowdin handoff under the current release root.
pub fn write_official_resource_change_handoff(
previous_resource_root: Option<&Path>,
current_resource_root: &Path,
official_release_id: &str,
previous_release_id: Option<String>,
) -> Result<OfficialResourceChangeHandoffReport, String> {
let current_manifest = read_download_manifest_at(current_resource_root)?.ok_or_else(|| {
format!(
"缺少当前官方下载 manifest,无法生成资源变更集:{}",
current_resource_root.display()
)
})?;
let previous_manifest = match previous_resource_root {
Some(root) => read_download_manifest_at(root)?,
None => None,
};
let previous_manifest_root = previous_manifest
.as_ref()
.and(previous_resource_root)
.map(Path::to_path_buf);
let previous_release_id = previous_manifest.as_ref().and(previous_release_id);
let change_set = OfficialResourceChangeSet::from_manifests(
official_release_id,
previous_release_id,
previous_manifest_root,
current_resource_root.to_path_buf(),
previous_manifest.as_ref(),
&current_manifest,
);
write_resource_change_set_at(current_resource_root, &change_set)?;
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
write_crowdin_translation_handoff_at(current_resource_root, &handoff)?;
Ok(OfficialResourceChangeHandoffReport {
change_set_path: current_resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE),
crowdin_handoff_path: current_resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE),
summary: change_set.summary,
})
}
/// Paths and summary produced after writing a resource-change handoff.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeHandoffReport {
/// Path to `official-resource-changes.json`.
pub change_set_path: PathBuf,
/// Path to `crowdin-translation-handoff.json`.
pub crowdin_handoff_path: PathBuf,
/// Aggregate change counters.
pub summary: OfficialResourceChangeSummary,
}
/// Reads a generated resource change set from a release root.
pub fn read_resource_change_set_at(
resource_root: &Path,
) -> Result<Option<OfficialResourceChangeSet>, String> {
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方资源变更集")? else {
return Ok(None);
};
let change_set: OfficialResourceChangeSet = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析官方资源变更集失败 {}{error}", path.display()))?;
if change_set.change_set_version != OFFICIAL_RESOURCE_CHANGES_VERSION {
return Err(format!(
"不支持的官方资源变更集版本 {},文件 {}",
change_set.change_set_version,
path.display()
));
}
Ok(Some(change_set))
}
/// Writes a generated resource change set under a release root.
pub fn write_resource_change_set_at(
resource_root: &Path,
change_set: &OfficialResourceChangeSet,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方资源变更集")?;
let bytes = serde_json::to_vec_pretty(change_set)
.map_err(|error| format!("序列化官方资源变更集失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方资源变更集")
}
/// Writes a generated Crowdin handoff under a release root.
pub fn write_crowdin_translation_handoff_at(
resource_root: &Path,
handoff: &CrowdinTranslationHandoff,
) -> Result<(), String> {
let path = resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "Crowdin 翻译 handoff")?;
let bytes = serde_json::to_vec_pretty(handoff)
.map_err(|error| format!("序列化 Crowdin 翻译 handoff 失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin 翻译 handoff")
}
fn entries_by_destination(
manifest: &OfficialDownloadManifest,
) -> BTreeMap<String, &OfficialDownloadManifestEntry> {
manifest
.entries
.values()
.map(|entry| (entry.destination.clone(), entry))
.collect()
}
fn content_changed(
previous: &OfficialDownloadManifestEntry,
current: &OfficialDownloadManifestEntry,
) -> bool {
previous.bytes != current.bytes || previous.blake3 != current.blake3
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_resource_changes_version() -> u32 {
OFFICIAL_RESOURCE_CHANGES_VERSION
}
fn default_crowdin_handoff_version() -> u32 {
CROWDIN_TRANSLATION_HANDOFF_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
fn manifest(entries: &[(&str, &str, &[u8])]) -> OfficialDownloadManifest {
let mut manifest = OfficialDownloadManifest::default();
for (url, destination, bytes) in entries {
manifest.entries.insert(
(*url).to_string(),
OfficialDownloadManifestEntry {
url: (*url).to_string(),
destination: (*destination).to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
);
}
manifest
}
fn write_manifest(root: &Path, manifest: &OfficialDownloadManifest) {
std::fs::create_dir_all(root).unwrap();
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(manifest).unwrap(),
)
.unwrap();
}
#[test]
fn change_set_classifies_added_modified_and_removed_resources() {
let previous = manifest(&[
("https://old/a", "TableBundles/a.bytes", b"old-a"),
("https://old/b", "TableBundles/b.bytes", b"same"),
("https://old/c", "TableBundles/c.bytes", b"removed"),
(
"https://old/u",
"TableBundles/url-only.bytes",
b"same-url-only",
),
]);
let current = manifest(&[
("https://new/a", "TableBundles/a.bytes", b"new-a"),
("https://new/b", "TableBundles/b.bytes", b"same"),
("https://new/d", "TableBundles/d.bytes", b"added"),
(
"https://changed-host/u",
"TableBundles/url-only.bytes",
b"same-url-only",
),
]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
Some("release-old".to_string()),
Some(PathBuf::from("/previous")),
PathBuf::from("/current"),
Some(&previous),
&current,
);
assert_eq!(change_set.summary.added_count, 1);
assert_eq!(change_set.summary.modified_count, 1);
assert_eq!(change_set.summary.removed_count, 1);
assert_eq!(change_set.summary.parse_candidate_count, 2);
assert_eq!(change_set.summary.translation_candidate_count, 2);
let destinations = change_set
.translation_candidates()
.into_iter()
.map(|change| change.destination.as_str())
.collect::<Vec<_>>();
assert_eq!(
destinations,
vec!["TableBundles/a.bytes", "TableBundles/d.bytes"]
);
}
#[test]
fn first_release_treats_all_current_resources_as_added() {
let current = manifest(&[
("https://new/a", "a.bundle", b"a"),
("https://new/b", "b.bundle", b"b"),
]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
None,
None,
PathBuf::from("/current"),
None,
&current,
);
assert!(!change_set.summary.previous_manifest_present);
assert_eq!(change_set.summary.added_count, 2);
assert_eq!(change_set.summary.translation_candidate_count, 2);
assert_eq!(change_set.changes.len(), 2);
}
#[test]
fn crowdin_handoff_excludes_removed_resources() {
let previous = manifest(&[("https://old/a", "a.bundle", b"a")]);
let current = manifest(&[("https://new/b", "b.bundle", b"b")]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
Some("release-old".to_string()),
None,
PathBuf::from("/current"),
Some(&previous),
&current,
);
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
assert_eq!(handoff.provider, TranslationHandoffProvider::Crowdin);
assert_eq!(handoff.status, TranslationHandoffStatus::QueuedOffline);
assert_eq!(handoff.resource_count, 1);
assert_eq!(handoff.resources[0].destination, "b.bundle");
}
#[test]
fn write_handoff_persists_change_set_and_crowdin_queue() {
let temp = tempfile::TempDir::new().unwrap();
let previous_root = temp.path().join("previous");
let current_root = temp.path().join("current");
write_manifest(
&previous_root,
&manifest(&[("https://old/a", "a.bundle", b"old")]),
);
write_manifest(
&current_root,
&manifest(&[
("https://new/a", "a.bundle", b"new"),
("https://new/b", "b.bundle", b"added"),
]),
);
let report = write_official_resource_change_handoff(
Some(&previous_root),
&current_root,
"release-new",
Some("release-old".to_string()),
)
.unwrap();
assert_eq!(report.summary.modified_count, 1);
assert_eq!(report.summary.added_count, 1);
assert!(report.change_set_path.exists());
assert!(report.crowdin_handoff_path.exists());
let change_set = read_resource_change_set_at(&current_root).unwrap().unwrap();
assert_eq!(change_set.summary.translation_candidate_count, 2);
}
}
+7 -5
View File
@@ -115,8 +115,8 @@ pub struct OfficialResourcePullProgress {
/// ///
/// `Started` events report the currently completed count before the URL /// `Started` events report the currently completed count before the URL
/// finishes; `Finished` events report the count after completion. This is /// finishes; `Finished` events report the count after completion. This is
/// intentionally not the URL's plan position, because concurrent downloads /// intentionally not the URL's plan position, so status percentages stay
/// finish out of plan order and status percentages must not move backward. /// monotonic even if execution order or skip/resume mix changes.
pub index: usize, pub index: usize,
/// Total URL count in the pull plan. /// Total URL count in the pull plan.
pub total: usize, pub total: usize,
@@ -2272,14 +2272,16 @@ exit 22
} }
fn write_shell_script(path: &Path, script: &str) { fn write_shell_script(path: &Path, script: &str) {
fs::write(path, script).unwrap(); let temp_path = path.with_extension("tmp");
fs::write(&temp_path, script).unwrap();
#[cfg(unix)] #[cfg(unix)]
{ {
use std::os::unix::fs::PermissionsExt; use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(path).unwrap().permissions(); let mut permissions = fs::metadata(&temp_path).unwrap().permissions();
permissions.set_mode(0o755); permissions.set_mode(0o755);
fs::set_permissions(path, permissions).unwrap(); fs::set_permissions(&temp_path, permissions).unwrap();
} }
fs::rename(temp_path, path).unwrap();
} }
fn write_fake_curl(path: &Path) { fn write_fake_curl(path: &Path) {
+101 -10
View File
@@ -9,7 +9,8 @@ use crate::official_download::DownloadError;
use crate::official_launcher::launcher_package_url; use crate::official_launcher::launcher_package_url;
use crate::official_launcher::OfficialLauncherBootstrapService; use crate::official_launcher::OfficialLauncherBootstrapService;
use crate::official_launcher::{ use crate::official_launcher::{
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherRemoteManifest, YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl,
YostarJpLauncherRemoteManifest,
}; };
use crate::zip_validation::validate_zip_structure; use crate::zip_validation::validate_zip_structure;
use bat_adapters::official::game_main_config::YostarJpGameMainConfig; use bat_adapters::official::game_main_config::YostarJpGameMainConfig;
@@ -39,12 +40,44 @@ pub struct OfficialGameMainConfigBootstrap {
/// may instead point to a directory source plus per-file entries; in that /// may instead point to a directory source plus per-file entries; in that
/// case this is the direct `resources.assets` URL. /// case this is the direct `resources.assets` URL.
pub game_zip_url: String, pub game_zip_url: String,
/// Remote launcher manifest used for this bootstrap.
pub remote_manifest: YostarJpLauncherRemoteManifest,
/// Exact source selected to obtain `GameMainConfig`.
pub selected_source: OfficialGameMainConfigSelectedSource,
/// Number of files declared by the remote manifest. /// Number of files declared by the remote manifest.
pub manifest_file_count: usize, pub manifest_file_count: usize,
/// Decrypted `GameMainConfig`. /// Decrypted `GameMainConfig`.
pub game_main_config: YostarJpGameMainConfig, pub game_main_config: YostarJpGameMainConfig,
} }
/// Kind of official launcher package artifact selected for `GameMainConfig`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OfficialGameMainConfigSourceKind {
/// Older launcher manifests point to a single game ZIP archive.
Archive,
/// Current launcher manifests expose a directory plus per-file entries.
ManifestFile,
}
/// Exact launcher artifact selected to obtain `resources.assets`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialGameMainConfigSelectedSource {
/// Source kind.
pub kind: OfficialGameMainConfigSourceKind,
/// Official URL fetched for this source.
pub url: String,
/// Relative path under the official launcher package CDN root.
pub relative_path: String,
/// Original manifest file path when the source is a manifest file entry.
pub manifest_path: Option<String>,
/// Declared file size from the manifest, when available.
pub declared_size: Option<u64>,
/// Official launcher manifest `hash` field, when available.
pub official_hash: Option<String>,
/// Official launcher manifest per-file `vc`, when available.
pub vc: Option<String>,
}
/// Loads and decrypts the official `GameMainConfig` by following the official /// Loads and decrypts the official `GameMainConfig` by following the official
/// launcher package chain. /// launcher package chain.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -92,6 +125,18 @@ impl OfficialGameMainConfigBootstrapService {
/// launcher API, extracts `resources.assets`, and decrypts `GameMainConfig`. /// launcher API, extracts `resources.assets`, and decrypts `GameMainConfig`.
pub fn fetch_bootstrap(&self) -> Result<OfficialGameMainConfigBootstrap, DownloadError> { pub fn fetch_bootstrap(&self) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
let (game_config, manifest_url, manifest) = self.launcher.fetch_latest_remote_manifest()?; let (game_config, manifest_url, manifest) = self.launcher.fetch_latest_remote_manifest()?;
let cdn_config = self.launcher.fetch_cdn_config()?;
self.fetch_bootstrap_from_parts(game_config, cdn_config, manifest_url, manifest)
}
/// Extracts `GameMainConfig` from already fetched launcher metadata.
pub fn fetch_bootstrap_from_parts(
&self,
game_config: YostarJpLauncherGameConfig,
cdn_config: YostarJpLauncherCdnConfig,
manifest_url: YostarJpLauncherManifestUrl,
manifest: YostarJpLauncherRemoteManifest,
) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
let manifest_source = manifest let manifest_source = manifest
.source .source
.clone() .clone()
@@ -102,9 +147,8 @@ impl OfficialGameMainConfigBootstrapService {
"官方启动器远端 manifest 缺少 source", "官方启动器远端 manifest 缺少 source",
) )
})?; })?;
let cdn_config = self.launcher.fetch_cdn_config()?;
let temp_dir = TempDir::new().map_err(|error| format!("创建临时目录失败:{error}"))?; let temp_dir = TempDir::new().map_err(|error| format!("创建临时目录失败:{error}"))?;
let (game_zip_url, resources_assets) = self.fetch_resources_assets( let (game_zip_url, resources_assets, selected_source) = self.fetch_resources_assets(
&game_config, &game_config,
&manifest, &manifest,
&manifest_source, &manifest_source,
@@ -113,6 +157,7 @@ impl OfficialGameMainConfigBootstrapService {
)?; )?;
let game_main_config = YostarJpGameMainConfig::from_resources_assets(resources_assets) let game_main_config = YostarJpGameMainConfig::from_resources_assets(resources_assets)
.map_err(|error| DownloadError::new(ErrorCode::GAME_MAIN_CONFIG_FAILED, error))?; .map_err(|error| DownloadError::new(ErrorCode::GAME_MAIN_CONFIG_FAILED, error))?;
let manifest_file_count = manifest.files.len();
Ok(OfficialGameMainConfigBootstrap { Ok(OfficialGameMainConfigBootstrap {
game_config, game_config,
@@ -120,7 +165,9 @@ impl OfficialGameMainConfigBootstrapService {
manifest_url: manifest_url.url, manifest_url: manifest_url.url,
manifest_source: Some(manifest_source), manifest_source: Some(manifest_source),
game_zip_url, game_zip_url,
manifest_file_count: manifest.files.len(), remote_manifest: manifest,
selected_source,
manifest_file_count,
game_main_config, game_main_config,
}) })
} }
@@ -203,10 +250,12 @@ impl OfficialGameMainConfigBootstrapService {
manifest_source: &str, manifest_source: &str,
cdn_config: &YostarJpLauncherCdnConfig, cdn_config: &YostarJpLauncherCdnConfig,
temp_root: &Path, temp_root: &Path,
) -> Result<(String, PathBuf), DownloadError> { ) -> Result<(String, PathBuf, OfficialGameMainConfigSelectedSource), DownloadError> {
let selected_source =
resolve_game_main_config_source(game_config, manifest, manifest_source, cdn_config)?;
match select_game_main_config_source(game_config, manifest_source, manifest)? { match select_game_main_config_source(game_config, manifest_source, manifest)? {
GameMainConfigSource::Archive(package_path) => { GameMainConfigSource::Archive(package_path) => {
let game_zip_url = launcher_package_url(&cdn_config.primary_cdn, package_path)?; let game_zip_url = selected_source.url.clone();
let archive_path = temp_root.join("official-game.zip"); let archive_path = temp_root.join("official-game.zip");
self.download_file_with_fallback( self.download_file_with_fallback(
&game_zip_url, &game_zip_url,
@@ -227,12 +276,11 @@ impl OfficialGameMainConfigBootstrapService {
"官方启动器包内没有找到 resources.assets", "官方启动器包内没有找到 resources.assets",
) )
})?; })?;
Ok((game_zip_url, resources_assets)) Ok((game_zip_url, resources_assets, selected_source))
} }
GameMainConfigSource::ManifestFile { source_dir, file } => { GameMainConfigSource::ManifestFile { source_dir, file } => {
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)?; let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)?;
let resources_assets_url = let resources_assets_url = selected_source.url.clone();
launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
let resources_assets = temp_root.join("resources.assets"); let resources_assets = temp_root.join("resources.assets");
self.download_file_with_fallback( self.download_file_with_fallback(
&resources_assets_url, &resources_assets_url,
@@ -241,7 +289,7 @@ impl OfficialGameMainConfigBootstrapService {
&resources_assets, &resources_assets,
)?; )?;
verify_manifest_file_size(&resources_assets, file)?; verify_manifest_file_size(&resources_assets, file)?;
Ok((resources_assets_url, resources_assets)) Ok((resources_assets_url, resources_assets, selected_source))
} }
} }
} }
@@ -283,6 +331,49 @@ impl OfficialGameMainConfigBootstrapService {
} }
} }
/// Resolves the exact official launcher artifact used to obtain `GameMainConfig`.
pub fn resolve_game_main_config_source(
game_config: &YostarJpLauncherGameConfig,
manifest: &YostarJpLauncherRemoteManifest,
manifest_source: &str,
cdn_config: &YostarJpLauncherCdnConfig,
) -> Result<OfficialGameMainConfigSelectedSource, DownloadError> {
match select_game_main_config_source(game_config, manifest_source, manifest)? {
GameMainConfigSource::Archive(package_path) => {
let url = launcher_package_url(&cdn_config.primary_cdn, package_path)?;
Ok(OfficialGameMainConfigSelectedSource {
kind: OfficialGameMainConfigSourceKind::Archive,
url,
relative_path: package_path.to_string(),
manifest_path: None,
declared_size: None,
official_hash: None,
vc: None,
})
}
GameMainConfigSource::ManifestFile { source_dir, file } => {
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)
.map_err(|error| DownloadError::new(ErrorCode::LAUNCHER_RESPONSE_INVALID, error))?;
let url = launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
let declared_size = file.size.parse::<u64>().map_err(|error| {
DownloadError::new(
ErrorCode::LAUNCHER_RESPONSE_INVALID,
format!("官方启动器 manifest 中 {} 的 size 无效:{error}", file.path),
)
})?;
Ok(OfficialGameMainConfigSelectedSource {
kind: OfficialGameMainConfigSourceKind::ManifestFile,
url,
relative_path,
manifest_path: Some(file.path.clone()),
declared_size: Some(declared_size),
official_hash: Some(file.hash.clone()),
vc: file.vc.clone(),
})
}
}
}
#[derive(Debug)] #[derive(Debug)]
enum GameMainConfigSource<'a> { enum GameMainConfigSource<'a> {
Archive(&'a str), Archive(&'a str),
+567 -45
View File
@@ -10,9 +10,11 @@ use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE, STATE_FILE_MODE,
}; };
use bat_assetbundle::{Parser, UnityFsParser}; use bat_assetbundle::{
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::BTreeMap; use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::Command; use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@@ -20,7 +22,11 @@ use std::time::{SystemTime, UNIX_EPOCH};
/// Parse-cache file name stored under a published official resource root. /// Parse-cache file name stored under a published official resource root.
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json"; pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
/// Current parse-cache schema version. /// Current parse-cache schema version.
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1; pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
/// TextUnit detail index file name stored under a published official resource root.
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
/// Current TextUnit detail-index schema version.
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
/// Configuration for one official resource parse-cache refresh. /// Configuration for one official resource parse-cache refresh.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
@@ -55,6 +61,10 @@ pub struct OfficialParseReport {
pub cache_path: PathBuf, pub cache_path: PathBuf,
/// Aggregate parse-cache summary. /// Aggregate parse-cache summary.
pub summary: OfficialParseSummary, pub summary: OfficialParseSummary,
/// TextUnit detail-index path written by the refresh.
pub textunit_index_path: PathBuf,
/// Aggregate TextUnit detail-index summary.
pub textunit_index_summary: OfficialTextUnitIndexSummary,
} }
/// Aggregate counters for a parse-cache refresh. /// Aggregate counters for a parse-cache refresh.
@@ -78,6 +88,15 @@ pub struct OfficialParseSummary {
pub failed_count: usize, pub failed_count: usize,
/// Total TextAsset objects found in parsed Unity serialized files. /// Total TextAsset objects found in parsed Unity serialized files.
pub text_asset_count: usize, pub text_asset_count: usize,
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
#[serde(default)]
pub text_unit_count: usize,
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Number of non-fatal TypeTree field extraction diagnostics.
#[serde(default)]
pub text_unit_error_count: usize,
} }
/// Persistent parse cache for one official resource root. /// Persistent parse cache for one official resource root.
@@ -127,10 +146,162 @@ pub struct OfficialParseCacheEntry {
pub text_assets: Vec<String>, pub text_assets: Vec<String>,
/// Non-fatal serialized-file parse diagnostic count. /// Non-fatal serialized-file parse diagnostic count.
pub serialized_parse_error_count: usize, pub serialized_parse_error_count: usize,
/// Number of translation-ready TextUnit entries extracted from this bundle.
#[serde(default)]
pub text_unit_count: usize,
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
#[serde(default)]
pub text_unit_formats: Vec<String>,
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Non-fatal TypeTree field extraction diagnostic count.
#[serde(default)]
pub text_unit_error_count: usize,
/// Human-readable error or skip reason. /// Human-readable error or skip reason.
pub error: Option<String>, pub error: Option<String>,
} }
/// Persistent TextUnit detail index for one official resource root.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndex {
/// Index schema version.
#[serde(default = "default_textunit_index_version")]
pub version: u32,
/// Index generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Official resource root used to generate this index.
pub resource_root: PathBuf,
/// Aggregate counters for this index.
pub summary: OfficialTextUnitIndexSummary,
/// Extracted TextUnit details in deterministic order.
#[serde(default)]
pub units: Vec<OfficialTextUnitIndexUnit>,
/// Parse and extraction diagnostics in deterministic order.
#[serde(default)]
pub errors: Vec<OfficialTextUnitIndexError>,
}
/// Aggregate counters for a TextUnit detail index.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexSummary {
/// Number of TextUnit detail entries.
pub unit_count: usize,
/// Number of parse/extraction diagnostics.
pub error_count: usize,
/// Number of binary or invalid TextAsset payloads skipped by extraction.
pub skipped_binary_text_asset_count: usize,
}
/// One TextUnit detail entry.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexUnit {
/// Stable unit ID within the official release.
pub id: String,
/// Parse-cache entry key that produced this unit.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Unity editor version when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unity_version: Option<String>,
/// Original source text.
pub source_text: String,
/// Unity serialized file path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// TextUnit payload format such as json/csv/tsv/plain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
/// Extraction source kind such as TextAsset or TypeTreeField.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_source_kind: Option<String>,
/// TextAsset name when the unit came from a TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset_name: Option<String>,
/// Stable extraction context copied from the parser.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub context: BTreeMap<String, String>,
}
/// One parse or extraction diagnostic in the TextUnit detail index.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexError {
/// Stable diagnostic ID within the official release.
pub id: String,
/// Parse-cache entry key associated with this diagnostic.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Parse status associated with this diagnostic.
pub status: OfficialParseStatus,
/// Unity serialized file path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable diagnostic.
pub error: String,
}
/// Query filters for TextUnit detail entries.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialTextUnitQuery {
/// Filter by destination path.
pub destination: Option<String>,
/// Filter by destination glob pattern.
pub path_pattern: Option<String>,
/// Filter by archive entry.
pub archive_entry: Option<String>,
/// Filter by Unity object path ID.
pub path_id: Option<i64>,
/// Filter by Unity class ID.
pub class_id: Option<i32>,
/// Filter by field path.
pub field_path: Option<String>,
/// Filter by TextUnit format.
pub format: Option<String>,
}
/// Source kind for a parse-cache entry. /// Source kind for a parse-cache entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
@@ -187,21 +358,36 @@ impl OfficialParseCacheService {
) )
})?; })?;
let previous_cache = read_parse_cache_at(&config.resource_root)?; let previous_cache = read_parse_cache_at(&config.resource_root)?;
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
let mut summary = OfficialParseSummary { let mut summary = OfficialParseSummary {
manifest_entry_count: manifest.entries.len(), manifest_entry_count: manifest.entries.len(),
..OfficialParseSummary::default() ..OfficialParseSummary::default()
}; };
let mut entries = BTreeMap::new(); let mut entries = BTreeMap::new();
let mut units = Vec::new();
let mut errors = Vec::new();
for manifest_entry in manifest.entries.values() { for manifest_entry in manifest.entries.values() {
let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref()); let produced = process_manifest_entry(
for entry in produced { config,
summary.record_entry(&entry); manifest_entry,
entries.insert(entry.key.clone(), entry); previous_cache.as_ref(),
previous_textunit_index.as_ref(),
);
for produced in produced {
summary.record_entry(&produced.entry);
units.extend(produced.units);
errors.extend(produced.errors);
entries.insert(produced.entry.key.clone(), produced.entry);
} }
} }
summary.cache_entry_count = entries.len(); summary.cache_entry_count = entries.len();
let textunit_index_summary = OfficialTextUnitIndexSummary {
unit_count: units.len(),
error_count: errors.len(),
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
};
let cache = OfficialParseCache { let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION, version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: unix_seconds_now(), generated_unix_seconds: unix_seconds_now(),
@@ -209,15 +395,67 @@ impl OfficialParseCacheService {
entries, entries,
}; };
write_parse_cache_at(&config.resource_root, &cache)?; write_parse_cache_at(&config.resource_root, &cache)?;
let textunit_index = OfficialTextUnitIndex {
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
generated_unix_seconds: cache.generated_unix_seconds,
resource_root: config.resource_root.clone(),
summary: textunit_index_summary.clone(),
units,
errors,
};
write_textunit_index_at(&config.resource_root, &textunit_index)?;
Ok(OfficialParseReport { Ok(OfficialParseReport {
resource_root: config.resource_root.clone(), resource_root: config.resource_root.clone(),
cache_path: config.cache_path(), cache_path: config.cache_path(),
summary, summary,
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
textunit_index_summary,
}) })
} }
} }
struct ParseProduced {
entry: OfficialParseCacheEntry,
units: Vec<OfficialTextUnitIndexUnit>,
errors: Vec<OfficialTextUnitIndexError>,
}
impl ParseProduced {
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
let errors = error_from_status_entry(&entry).into_iter().collect();
Self {
entry,
units: Vec::new(),
errors,
}
}
fn from_cached(
mut entry: OfficialParseCacheEntry,
previous_index: Option<&OfficialTextUnitIndex>,
) -> Self {
entry.reused_from_previous_cache = true;
let units = previous_index
.into_iter()
.flat_map(|index| index.units.iter())
.filter(|unit| unit.parse_entry_key == entry.key)
.cloned()
.collect();
let errors = previous_index
.into_iter()
.flat_map(|index| index.errors.iter())
.filter(|error| error.parse_entry_key == entry.key)
.cloned()
.collect();
Self {
entry,
units,
errors,
}
}
}
impl OfficialParseSummary { impl OfficialParseSummary {
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) { fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
match entry.source_kind { match entry.source_kind {
@@ -234,6 +472,9 @@ impl OfficialParseSummary {
OfficialParseStatus::Parsed => { OfficialParseStatus::Parsed => {
self.parsed_bundle_count += 1; self.parsed_bundle_count += 1;
self.text_asset_count += entry.text_asset_count; self.text_asset_count += entry.text_asset_count;
self.text_unit_count += entry.text_unit_count;
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
self.text_unit_error_count += entry.text_unit_error_count;
} }
OfficialParseStatus::SkippedUnsupported => { OfficialParseStatus::SkippedUnsupported => {
self.unsupported_count += 1; self.unsupported_count += 1;
@@ -279,21 +520,81 @@ pub fn write_parse_cache_at(
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存") write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
} }
/// Reads the TextUnit detail index under a published official resource root.
pub fn read_textunit_index_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitIndex>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
return Ok(None);
};
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
return Ok(None);
};
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
return Ok(None);
}
Ok(Some(index))
}
/// Writes the TextUnit detail index under a published official resource root.
pub fn write_textunit_index_at(
resource_root: &Path,
index: &OfficialTextUnitIndex,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
let bytes = serde_json::to_vec_pretty(index)
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
}
/// Returns TextUnit detail entries matching a query.
pub fn query_textunit_index_units<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexUnit> {
index
.units
.iter()
.filter(|unit| textunit_unit_matches(unit, query))
.collect()
}
/// Returns parse/extraction diagnostics matching a query.
pub fn query_textunit_index_errors<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexError> {
index
.errors
.iter()
.filter(|error| textunit_error_matches(error, query))
.collect()
}
fn process_manifest_entry( fn process_manifest_entry(
config: &OfficialParseConfig, config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry, manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>, previous_cache: Option<&OfficialParseCache>,
) -> Vec<OfficialParseCacheEntry> { previous_index: Option<&OfficialTextUnitIndex>,
) -> Vec<ParseProduced> {
let fingerprint = fingerprint_for(manifest_entry); let fingerprint = fingerprint_for(manifest_entry);
if looks_like_zip_source(manifest_entry) { if looks_like_zip_source(manifest_entry) {
return process_zip_entry(config, manifest_entry, previous_cache, fingerprint); return process_zip_entry(
config,
manifest_entry,
previous_cache,
previous_index,
fingerprint,
);
} }
if looks_like_direct_bundle_source(manifest_entry) { if looks_like_direct_bundle_source(manifest_entry) {
let key = direct_key(&manifest_entry.url); let key = direct_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) { if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true; return vec![ParseProduced::from_cached(cached, previous_index)];
return vec![cached];
} }
return vec![parse_direct_bundle( return vec![parse_direct_bundle(
config, config,
@@ -304,61 +605,58 @@ fn process_manifest_entry(
} }
let key = unsupported_key(&manifest_entry.url); let key = unsupported_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) { if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true; return vec![ParseProduced::from_cached(cached, previous_index)];
return vec![cached];
} }
vec![unsupported_entry( vec![ParseProduced::from_entry(unsupported_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::Unsupported, OfficialParseSourceKind::Unsupported,
fingerprint, fingerprint,
key, key,
"非 UnityFS 候选资源", "非 UnityFS 候选资源",
)] ))]
} }
fn process_zip_entry( fn process_zip_entry(
config: &OfficialParseConfig, config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry, manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>, previous_cache: Option<&OfficialParseCache>,
previous_index: Option<&OfficialTextUnitIndex>,
fingerprint: OfficialParseSourceFingerprint, fingerprint: OfficialParseSourceFingerprint,
) -> Vec<OfficialParseCacheEntry> { ) -> Vec<ParseProduced> {
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint); let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
if !cached_entries.is_empty() { if !cached_entries.is_empty() {
return cached_entries return cached_entries
.into_iter() .into_iter()
.map(|mut entry| { .map(|entry| ParseProduced::from_cached(entry, previous_index))
entry.reused_from_previous_cache = true;
entry
})
.collect(); .collect();
} }
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) { let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path, Ok(path) => path,
Err(error) => { Err(error) => {
return vec![failed_entry( return vec![ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::ZipEntry, OfficialParseSourceKind::ZipEntry,
fingerprint, fingerprint,
zip_list_key(&manifest_entry.url), zip_list_key(&manifest_entry.url),
error, error,
)] ))]
} }
}; };
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) { let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
Ok(entries) => entries, Ok(entries) => entries,
Err(error) => { Err(error) => {
return vec![failed_entry( return vec![ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::ZipEntry, OfficialParseSourceKind::ZipEntry,
fingerprint, fingerprint,
zip_list_key(&manifest_entry.url), zip_list_key(&manifest_entry.url),
error, error,
)] ))]
} }
}; };
@@ -369,14 +667,14 @@ fn process_zip_entry(
{ {
Ok(bytes) => bytes, Ok(bytes) => bytes,
Err(error) => { Err(error) => {
produced.push(failed_entry( produced.push(ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
Some(archive_entry), Some(archive_entry),
OfficialParseSourceKind::ZipEntry, OfficialParseSourceKind::ZipEntry,
fingerprint.clone(), fingerprint.clone(),
key, key,
error, error,
)); )));
continue; continue;
} }
}; };
@@ -390,14 +688,14 @@ fn process_zip_entry(
} }
if produced.is_empty() { if produced.is_empty() {
produced.push(unsupported_entry( produced.push(ParseProduced::from_entry(unsupported_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::Unsupported, OfficialParseSourceKind::Unsupported,
fingerprint, fingerprint,
zip_list_key(&manifest_entry.url), zip_list_key(&manifest_entry.url),
"ZIP 内没有可检查文件条目", "ZIP 内没有可检查文件条目",
)); )));
} }
produced produced
} }
@@ -407,42 +705,42 @@ fn parse_direct_bundle(
manifest_entry: &OfficialDownloadManifestEntry, manifest_entry: &OfficialDownloadManifestEntry,
key: String, key: String,
fingerprint: OfficialParseSourceFingerprint, fingerprint: OfficialParseSourceFingerprint,
) -> OfficialParseCacheEntry { ) -> ParseProduced {
let path = match resource_path_for(&config.resource_root, manifest_entry) { let path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path, Ok(path) => path,
Err(error) => { Err(error) => {
return failed_entry( return ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::DirectBundle, OfficialParseSourceKind::DirectBundle,
fingerprint, fingerprint,
key, key,
error, error,
) ))
} }
}; };
let bytes = match read_resource_file(&path) { let bytes = match read_resource_file(&path) {
Ok(bytes) => bytes, Ok(bytes) => bytes,
Err(error) => { Err(error) => {
return failed_entry( return ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::DirectBundle, OfficialParseSourceKind::DirectBundle,
fingerprint, fingerprint,
key, key,
error, error,
) ))
} }
}; };
if !UnityFsParser::has_unityfs_signature(&bytes) { if !UnityFsParser::has_unityfs_signature(&bytes) {
return unsupported_entry( return ParseProduced::from_entry(unsupported_entry(
manifest_entry, manifest_entry,
None, None,
OfficialParseSourceKind::DirectBundle, OfficialParseSourceKind::DirectBundle,
fingerprint, fingerprint,
key, key,
"文件不是 UnityFS bundle", "文件不是 UnityFS bundle",
); ));
} }
parsed_bundle_entry( parsed_bundle_entry(
manifest_entry, manifest_entry,
@@ -460,16 +758,16 @@ fn parse_zip_inner_file(
fingerprint: OfficialParseSourceFingerprint, fingerprint: OfficialParseSourceFingerprint,
key: String, key: String,
bytes: &[u8], bytes: &[u8],
) -> OfficialParseCacheEntry { ) -> ParseProduced {
if !UnityFsParser::has_unityfs_signature(bytes) { if !UnityFsParser::has_unityfs_signature(bytes) {
return unsupported_entry( return ParseProduced::from_entry(unsupported_entry(
manifest_entry, manifest_entry,
Some(archive_entry), Some(archive_entry),
OfficialParseSourceKind::ZipEntry, OfficialParseSourceKind::ZipEntry,
fingerprint, fingerprint,
key, key,
"ZIP 条目不是 UnityFS bundle", "ZIP 条目不是 UnityFS bundle",
); ));
} }
parsed_bundle_entry( parsed_bundle_entry(
manifest_entry, manifest_entry,
@@ -488,10 +786,24 @@ fn parsed_bundle_entry(
fingerprint: OfficialParseSourceFingerprint, fingerprint: OfficialParseSourceFingerprint,
key: String, key: String,
bytes: &[u8], bytes: &[u8],
) -> OfficialParseCacheEntry { ) -> ParseProduced {
let parser = UnityFsParser::new(); let parser = UnityFsParser::new();
match parser.parse(bytes) { match parser.parse(bytes) {
Ok(parsed) => OfficialParseCacheEntry { Ok(parsed) => {
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
&parsed,
Some(&manifest_entry.destination),
archive_entry.as_deref(),
);
let text_unit_formats = text_units
.units
.iter()
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let entry = OfficialParseCacheEntry {
key, key,
source_url: manifest_entry.url.clone(), source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(), destination: manifest_entry.destination.clone(),
@@ -510,16 +822,218 @@ fn parsed_bundle_entry(
.map(|asset| asset.name.clone()) .map(|asset| asset.name.clone())
.collect(), .collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(), serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
text_unit_error_count: text_units.errors.len(),
error: None, error: None,
}, };
Err(error) => failed_entry( let units = indexed_text_units_for_entry(&entry, &text_units.units);
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
ParseProduced {
entry,
units,
errors,
}
}
Err(error) => ParseProduced::from_entry(failed_entry(
manifest_entry, manifest_entry,
archive_entry, archive_entry,
source_kind, source_kind,
fingerprint, fingerprint,
key, key,
error.to_string(), error.to_string(),
), )),
}
}
fn indexed_text_units_for_entry(
entry: &OfficialParseCacheEntry,
units: &[TextUnit],
) -> Vec<OfficialTextUnitIndexUnit> {
units
.iter()
.enumerate()
.map(|(index, unit)| OfficialTextUnitIndexUnit {
id: format!("{}#unit:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
unity_version: entry.unity_version.clone(),
source_text: unit.source_text.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
field_offset: unit.field_offset,
field_byte_size: unit.field_byte_size,
format: unit.context.get("format").cloned(),
text_source_kind: unit.context.get("source_kind").cloned(),
asset_name: unit.context.get("asset_name").cloned(),
context: unit.context.clone(),
})
.collect()
}
fn indexed_extraction_errors_for_entry(
entry: &OfficialParseCacheEntry,
errors: &[TextUnitExtractionError],
) -> Vec<OfficialTextUnitIndexError> {
errors
.iter()
.enumerate()
.map(|(index, error)| OfficialTextUnitIndexError {
id: format!("{}#extract-error:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: error.serialized_file.clone(),
path_id: error.path_id,
class_id: error.class_id,
field_path: error.field_path.clone(),
offset: error.offset,
error: error.error.clone(),
})
.collect()
}
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
let error = entry.error.as_ref()?;
Some(OfficialTextUnitIndexError {
id: format!("{}#status", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: None,
path_id: None,
class_id: None,
field_path: None,
offset: None,
error: error.clone(),
})
}
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &unit.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| unit.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| unit.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
{
return false;
}
if query
.format
.as_ref()
.is_some_and(|format| unit.format.as_ref() != Some(format))
{
return false;
}
true
}
fn textunit_error_matches(
error: &OfficialTextUnitIndexError,
query: &OfficialTextUnitQuery,
) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &error.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| error.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| error.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
{
return false;
}
true
}
fn glob_matches(pattern: &str, value: &str) -> bool {
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
}
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
match pattern.split_first() {
None => value.is_empty(),
Some((&b'*', rest)) => {
glob_matches_bytes(rest, value)
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
}
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
Some((&literal, rest)) => value
.split_first()
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
} }
} }
@@ -585,6 +1099,10 @@ fn status_entry(
text_asset_count: 0, text_asset_count: 0,
text_assets: Vec::new(), text_assets: Vec::new(),
serialized_parse_error_count: 0, serialized_parse_error_count: 0,
text_unit_count: 0,
text_unit_formats: Vec::new(),
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: Some(reason), error: Some(reason),
} }
} }
@@ -735,6 +1253,10 @@ fn default_parse_cache_version() -> u32 {
OFFICIAL_PARSE_CACHE_VERSION OFFICIAL_PARSE_CACHE_VERSION
} }
fn default_textunit_index_version() -> u32 {
OFFICIAL_TEXTUNIT_INDEX_VERSION
}
fn unix_seconds_now() -> u64 { fn unix_seconds_now() -> u64 {
SystemTime::now() SystemTime::now()
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
+543
View File
@@ -0,0 +1,543 @@
//! Import of a verified official release into CAS and ResourceRepository.
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus,
OfficialParseSummary,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink,
};
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// Configuration for importing one already-published official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialReleaseImportConfig {
/// Published release root containing `official-download-manifest.json`.
pub release_root: PathBuf,
/// Official release ID associated with this root, when known.
pub official_release_id: Option<String>,
}
impl OfficialReleaseImportConfig {
/// Creates an import configuration.
pub fn new(release_root: impl Into<PathBuf>) -> Self {
Self {
release_root: release_root.into(),
official_release_id: None,
}
}
/// Attaches the official release ID that should be stored in resource metadata.
pub fn with_official_release_id(mut self, release_id: impl Into<String>) -> Self {
self.official_release_id = Some(release_id.into());
self
}
}
/// Summary returned by an official release repository import.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialReleaseImportReport {
/// Number of verified manifest entries.
pub manifest_entry_count: usize,
/// Number of new or changed repository rows.
pub imported_count: usize,
/// Number of rows already pointing at the same CAS object.
pub unchanged_count: usize,
/// Number of unchanged rows whose metadata was refreshed from parse cache.
#[serde(default)]
pub metadata_updated_count: usize,
/// Number of resources classified as AssetBundle.
pub asset_bundle_count: usize,
/// Number of resources classified as text-like payloads.
pub text_asset_count: usize,
/// Number of resources classified as tables.
pub table_count: usize,
/// Number of resources classified as media.
pub media_count: usize,
/// Parse-cache summary associated with this release, when available.
pub parse_summary: Option<OfficialParseSummary>,
/// Non-fatal cleanup warnings, such as an old CAS reference that could not
/// be decremented after a successful row replacement.
pub warnings: Vec<String>,
}
/// Imports verified official resources into CAS and the resource index.
pub struct OfficialReleaseImportService<'a> {
cas: &'a dyn CasRepository,
resources: &'a dyn ResourceRepository,
}
impl<'a> OfficialReleaseImportService<'a> {
/// Creates an import service.
pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self {
Self { cas, resources }
}
/// Imports every entry in one published release manifest.
///
/// The source files remain untouched. Every file is checked against the
/// verified download manifest before it can enter CAS. Repository IDs are
/// stable by destination, making repeated imports idempotent.
pub async fn import_release(
&self,
config: &OfficialReleaseImportConfig,
) -> bat_core::Result<OfficialReleaseImportReport> {
let manifest = read_download_manifest_at(&config.release_root)
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| {
bat_core::Error::NotFound(format!(
"官方下载 manifest 不存在:{}",
config.release_root.display()
))
})?;
let parse_cache =
read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?;
let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone());
let parse_entries_by_destination = parse_cache
.as_ref()
.map(parse_entries_by_destination)
.unwrap_or_default();
let release_id = config
.official_release_id
.clone()
.or_else(|| release_id_from_root(&config.release_root));
let mut report = OfficialReleaseImportReport {
manifest_entry_count: manifest.entries.len(),
imported_count: 0,
unchanged_count: 0,
metadata_updated_count: 0,
asset_bundle_count: 0,
text_asset_count: 0,
table_count: 0,
media_count: 0,
parse_summary,
warnings: Vec::new(),
};
for entry in manifest.entries.values() {
let parse_entries = parse_entries_by_destination
.get(&entry.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
self.import_entry(
&config.release_root,
release_id.as_deref(),
entry,
parse_entries,
&mut report,
)
.await?;
}
Ok(report)
}
async fn import_entry(
&self,
release_root: &Path,
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
report: &mut OfficialReleaseImportReport,
) -> bat_core::Result<()> {
let path = release_root.join(Path::new(&manifest_entry.destination));
ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?;
ensure_safe_file_target(release_root, &path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?;
let bytes = read_file_no_symlink(&path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?;
if bytes.len() as u64 != manifest_entry.bytes {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 size 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination,
manifest_entry.bytes,
bytes.len()
)));
}
let actual_hash = blake3::hash(&bytes).to_hex().to_string();
if actual_hash != manifest_entry.blake3 {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination, manifest_entry.blake3, actual_hash
)));
}
let resource_type = resource_type_for_path(&manifest_entry.destination);
let metadata =
metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries);
let resource_id = resource_id_for_destination(&manifest_entry.destination);
let previous = match self.resources.find_by_id(&resource_id).await {
Ok(resource) => Some(resource),
Err(bat_core::Error::NotFound(_)) => None,
Err(error) => return Err(error),
};
if previous
.as_ref()
.is_some_and(|resource| resource.entry.hash == actual_hash)
{
if let Some(mut resource) = previous {
let should_refresh_metadata = resource.metadata != metadata
|| resource.entry.resource_type != resource_type
|| resource.entry.size != manifest_entry.bytes
|| resource.local_path.as_path() != Path::new(&manifest_entry.destination);
if should_refresh_metadata {
resource.local_path = PathBuf::from(&manifest_entry.destination);
resource.entry.size = manifest_entry.bytes;
resource.entry.resource_type = resource_type;
resource.metadata = metadata;
self.resources.update(resource).await?;
report.metadata_updated_count += 1;
}
}
report.unchanged_count += 1;
count_resource_type(report, resource_type);
return Ok(());
}
let object_id = self.cas.store(&bytes).await?;
let resource = Resource {
id: resource_id,
local_path: PathBuf::from(&manifest_entry.destination),
entry: ResourceEntry {
path: manifest_entry.destination.clone(),
hash: object_id.clone(),
size: manifest_entry.bytes,
resource_type,
address: None,
dependencies: Vec::new(),
crc: None,
},
metadata,
};
if let Err(error) = self.resources.add(resource).await {
let _ = self.cas.remove_reference(&object_id).await;
return Err(error);
}
if let Some(previous) = previous {
if previous.entry.hash != object_id {
match self.cas.remove_reference(&previous.entry.hash).await {
Ok(_) => {}
Err(error) => report.warnings.push(format!(
"旧 CAS 引用清理失败 {}{}",
previous.entry.hash, error
)),
}
}
}
report.imported_count += 1;
count_resource_type(report, resource_type);
Ok(())
}
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn metadata_for_manifest_entry(
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
) -> ResourceMetadata {
let mut metadata = ResourceMetadata {
official_release_id: official_release_id.map(ToOwned::to_owned),
platform: platform_for_destination(&manifest_entry.destination),
bundle_path: Some(manifest_entry.destination.clone()),
..ResourceMetadata::default()
};
let mut archive_entries = BTreeSet::new();
let mut parse_statuses = BTreeSet::new();
let mut unity_versions = BTreeSet::new();
let mut text_assets = BTreeSet::new();
let mut text_unit_formats = BTreeSet::new();
for entry in parse_entries {
if let Some(archive_entry) = entry.archive_entry.as_ref() {
archive_entries.insert(archive_entry.clone());
}
parse_statuses.insert(parse_status_label(entry.status).to_string());
if let Some(unity_version) = entry.unity_version.as_ref() {
unity_versions.insert(unity_version.clone());
}
metadata.unityfs_file_count += entry.file_count as u64;
metadata.serialized_file_count += entry.serialized_file_count as u64;
metadata.text_asset_count += entry.text_asset_count as u64;
metadata.text_unit_count += entry.text_unit_count as u64;
metadata.text_unit_error_count += entry.text_unit_error_count as u64;
text_assets.extend(entry.text_assets.iter().cloned());
text_unit_formats.extend(entry.text_unit_formats.iter().cloned());
}
metadata.archive_entries = archive_entries.into_iter().collect();
metadata.parse_statuses = parse_statuses.into_iter().collect();
metadata.unity_versions = unity_versions.into_iter().collect();
metadata.text_assets = text_assets.into_iter().collect();
metadata.text_unit_formats = text_unit_formats.into_iter().collect();
metadata
}
fn parse_status_label(status: OfficialParseStatus) -> &'static str {
match status {
OfficialParseStatus::Parsed => "parsed",
OfficialParseStatus::SkippedUnsupported => "skipped_unsupported",
OfficialParseStatus::Failed => "failed",
}
}
fn platform_for_destination(destination: &str) -> Option<String> {
let normalized = destination.replace('\\', "/").to_ascii_lowercase();
if normalized.contains("windows") || normalized.contains("/win/") {
Some("windows".to_string())
} else if normalized.contains("android") {
Some("android".to_string())
} else {
None
}
}
fn release_id_from_root(release_root: &Path) -> Option<String> {
release_root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty() && *name != "current")
.map(ToOwned::to_owned)
}
fn resource_id_for_destination(destination: &str) -> String {
format!("official/{}", destination.replace('\\', "/"))
}
fn resource_type_for_path(path: &str) -> ResourceType {
let normalized = path.replace('\\', "/").to_ascii_lowercase();
if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") {
ResourceType::AssetBundle
} else if normalized.contains("tablebundles/") {
ResourceType::TableBundle
} else if normalized.contains("textassets/")
|| matches!(
normalized.rsplit('.').next(),
Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml")
)
{
ResourceType::TextAsset
} else if normalized.contains("mediaresources/")
|| matches!(
normalized.rsplit('.').next(),
Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp")
)
{
ResourceType::Media
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
ResourceType::Manifest
} else {
ResourceType::Other
}
}
fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) {
match resource_type {
ResourceType::AssetBundle => report.asset_bundle_count += 1,
ResourceType::TextAsset => report.text_asset_count += 1,
ResourceType::TableBundle => report.table_count += 1,
ResourceType::Media => report.media_count += 1,
ResourceType::Manifest | ResourceType::Other => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{FileSystemCasRepository, InMemoryResourceRepository};
use crate::{
OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION,
};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use std::collections::BTreeMap;
use tempfile::TempDir;
fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) {
let path = root.join(destination);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, bytes).unwrap();
let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialDownloadManifestEntry {
url: url.clone(),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
};
let manifest = serde_json::json!({
"version": 1,
"entries": BTreeMap::from([(url, entry)]),
});
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
}
fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) {
let source_url =
format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialParseCacheEntry {
key: format!("direct:{source_url}"),
source_url: source_url.clone(),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url,
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: 1,
text_assets: vec!["Scenario".to_string()],
serialized_parse_error_count: 0,
text_unit_count: 2,
text_unit_formats: vec!["json".to_string(), "plain".to_string()],
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
error: None,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 123,
summary: OfficialParseSummary {
manifest_entry_count: 1,
cache_entry_count: 1,
candidate_file_count: 1,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 0,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
},
entries: BTreeMap::from([(entry.key.clone(), entry)]),
};
std::fs::write(
root.join("official-parse-cache.json"),
serde_json::to_vec(&cache).unwrap(),
)
.unwrap();
}
#[tokio::test]
async fn imports_verified_release_idempotently() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog");
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let config = OfficialReleaseImportConfig::new(temp.path());
let first = service.import_release(&config).await.unwrap();
let second = service.import_release(&config).await.unwrap();
assert_eq!(first.imported_count, 1);
assert_eq!(second.imported_count, 0);
assert_eq!(second.unchanged_count, 1);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TableBundle))
.await
.unwrap(),
1
);
let id = resource_id_for_destination("TableBundles/TableCatalog.bytes");
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
cas.get_reference_count(&resource.entry.hash).await.unwrap(),
1
);
}
#[tokio::test]
async fn imports_parse_cache_metadata_into_resource_index() {
let temp = TempDir::new().unwrap();
let destination = "Windows/Bundles/scenario.bundle";
let bytes = b"bundle-bytes";
write_manifest(temp.path(), destination, bytes);
write_parse_cache(temp.path(), destination, bytes);
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let report = service
.import_release(
&OfficialReleaseImportConfig::new(temp.path())
.with_official_release_id("release-current"),
)
.await
.unwrap();
assert_eq!(report.imported_count, 1);
assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2);
let id = resource_id_for_destination(destination);
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
resource.metadata.official_release_id.as_deref(),
Some("release-current")
);
assert_eq!(resource.metadata.platform.as_deref(), Some("windows"));
assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination));
assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]);
assert_eq!(
resource.metadata.unity_versions,
vec!["2021.3.56f2".to_string()]
);
assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(resource.metadata.text_unit_count, 2);
assert_eq!(
resource.metadata.text_unit_formats,
vec!["json".to_string(), "plain".to_string()]
);
assert_eq!(resource.metadata.text_unit_error_count, 1);
}
#[tokio::test]
async fn rejects_manifest_hash_mismatch_before_cas_write() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original");
let path = temp.path().join("TextAssets/dialogue.txt");
std::fs::write(&path, b"tampered").unwrap();
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let error = service
.import_release(&OfficialReleaseImportConfig::new(temp.path()))
.await
.unwrap_err();
assert!(error.to_string().contains("BLAKE3"));
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
}
}
@@ -0,0 +1,834 @@
//! Incremental TextUnit task and Crowdin offline queues for official releases.
//!
//! The queue is derived only from a verified official release, its
//! `official-resource-changes.json`, and its `official-parse-cache.json`.
//! It does not call Crowdin or any network service.
use crate::official_changes::{
read_resource_change_set_at, OfficialResourceChange, OfficialResourceChangeKind,
OfficialResourceChangeSet, TranslationHandoffProvider, TranslationHandoffStatus,
};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceKind,
OfficialParseStatus,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
/// Current TextUnit task-queue schema version.
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE: &str = "official-textunit-tasks.json";
/// Current Crowdin TextUnit queue schema version.
pub const CROWDIN_TEXTUNIT_QUEUE_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const CROWDIN_TEXTUNIT_QUEUE_FILE: &str = "crowdin-textunit-queue.json";
/// Processing status for one incremental TextUnit task candidate.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialTextUnitTaskStatus {
/// The task has parse metadata and at least one TextUnit.
QueuedOffline,
/// No parse-cache entry exists for this changed resource yet.
SkippedNoParseEntry,
/// The resource parsed successfully, but did not produce TextUnits.
SkippedNoTextUnit,
/// Parse cache says the candidate failed to parse.
SkippedParseFailed,
/// Parse cache says the candidate is unsupported.
SkippedUnsupported,
}
impl OfficialTextUnitTaskStatus {
/// Returns the stable status label.
pub fn as_str(self) -> &'static str {
match self {
Self::QueuedOffline => "queued_offline",
Self::SkippedNoParseEntry => "skipped_no_parse_entry",
Self::SkippedNoTextUnit => "skipped_no_text_unit",
Self::SkippedParseFailed => "skipped_parse_failed",
Self::SkippedUnsupported => "skipped_unsupported",
}
}
}
/// One parse-cache-backed incremental task.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTask {
/// Stable task ID.
pub task_id: String,
/// Current official release ID.
pub official_release_id: String,
/// Relative resource path under the official release root.
pub destination: String,
/// Change kind that caused this task candidate.
pub change_kind: OfficialResourceChangeKind,
/// Current official URL.
pub url: String,
/// Verified byte count.
pub bytes: u64,
/// Verified BLAKE3 digest.
pub blake3: String,
/// Parse-cache entry key, when available.
pub parse_entry_key: Option<String>,
/// ZIP/archive entry containing the parsed bundle, when available.
pub archive_entry: Option<String>,
/// Parse-cache source kind, when available.
pub source_kind: Option<OfficialParseSourceKind>,
/// Parse status, when available.
pub parse_status: Option<OfficialParseStatus>,
/// Number of TextAsset objects in this parse entry.
pub text_asset_count: usize,
/// TextAsset names in this parse entry.
pub text_assets: Vec<String>,
/// Number of TextUnits in this parse entry.
pub text_unit_count: usize,
/// TextUnit format labels.
pub text_unit_formats: Vec<String>,
/// Non-fatal TextUnit extraction diagnostic count.
pub text_unit_error_count: usize,
/// Queue status.
pub status: OfficialTextUnitTaskStatus,
/// Diagnostic reason for skipped tasks.
pub reason: Option<String>,
}
/// Aggregate counters for an incremental TextUnit task queue.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTaskSummary {
/// Number of added/modified resources considered.
pub resource_candidate_count: usize,
/// Number of parse-cache entries inspected for those resources.
pub parse_entry_count: usize,
/// Number of tasks queued for offline translation.
pub queued_task_count: usize,
/// Number of changed resources with no parse-cache entry.
pub skipped_no_parse_entry_count: usize,
/// Number of parse entries with zero TextUnits.
pub skipped_no_text_unit_count: usize,
/// Number of failed parse entries.
pub skipped_parse_failed_count: usize,
/// Number of unsupported parse entries.
pub skipped_unsupported_count: usize,
/// Total queued TextUnit count.
pub text_unit_count: usize,
}
impl OfficialTextUnitTaskSummary {
fn record(&mut self, task: &OfficialTextUnitTask) {
match task.status {
OfficialTextUnitTaskStatus::QueuedOffline => {
self.queued_task_count += 1;
self.text_unit_count += task.text_unit_count;
}
OfficialTextUnitTaskStatus::SkippedNoParseEntry => {
self.skipped_no_parse_entry_count += 1;
}
OfficialTextUnitTaskStatus::SkippedNoTextUnit => {
self.skipped_no_text_unit_count += 1;
}
OfficialTextUnitTaskStatus::SkippedParseFailed => {
self.skipped_parse_failed_count += 1;
}
OfficialTextUnitTaskStatus::SkippedUnsupported => {
self.skipped_unsupported_count += 1;
}
}
}
}
/// Durable incremental TextUnit task queue for one official release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTaskQueue {
/// Queue schema version.
#[serde(default = "default_textunit_task_queue_version")]
pub queue_version: u32,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Current official release root.
pub current_resource_root: PathBuf,
/// Queue summary.
pub summary: OfficialTextUnitTaskSummary,
/// All task candidates, including skipped diagnostics.
pub tasks: Vec<OfficialTextUnitTask>,
}
impl OfficialTextUnitTaskQueue {
/// Builds a task queue from an official change set and parse cache.
pub fn from_change_set_and_parse_cache(
change_set: &OfficialResourceChangeSet,
parse_cache: &OfficialParseCache,
) -> Self {
let parse_entries = parse_entries_by_destination(parse_cache);
let mut summary = OfficialTextUnitTaskSummary {
resource_candidate_count: change_set.parse_candidates().len(),
..OfficialTextUnitTaskSummary::default()
};
let mut tasks = Vec::new();
for change in change_set.parse_candidates() {
let entries = parse_entries
.get(&change.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
if entries.is_empty() {
let task = skipped_no_parse_entry_task(change_set, change);
summary.record(&task);
tasks.push(task);
continue;
}
summary.parse_entry_count += entries.len();
for entry in entries {
let task = task_from_parse_entry(change_set, change, entry);
summary.record(&task);
tasks.push(task);
}
}
Self {
queue_version: OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
official_release_id: change_set.official_release_id.clone(),
previous_release_id: change_set.previous_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
current_resource_root: change_set.current_resource_root.clone(),
summary,
tasks,
}
}
fn matches_derivation(&self, expected: &Self) -> bool {
self.official_release_id == expected.official_release_id
&& self.previous_release_id == expected.previous_release_id
&& self.current_resource_root == expected.current_resource_root
&& self.summary == expected.summary
&& self.tasks == expected.tasks
}
}
/// One Crowdin offline queue item referencing a TextUnit task.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTextUnitQueueItem {
/// Stable TextUnit task ID.
pub task_id: String,
/// Relative resource path under the official release root.
pub destination: String,
/// ZIP/archive entry, when applicable.
pub archive_entry: Option<String>,
/// Number of queued TextUnits.
pub text_unit_count: usize,
/// TextUnit format labels.
pub text_unit_formats: Vec<String>,
}
/// Crowdin offline queue derived from queued TextUnit tasks.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTextUnitQueue {
/// Queue schema version.
#[serde(default = "default_crowdin_textunit_queue_version")]
pub queue_version: u32,
/// Provider reserved for this queue.
pub provider: TranslationHandoffProvider,
/// Queue status.
pub status: TranslationHandoffStatus,
/// Current official release ID.
pub official_release_id: String,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Path to the source TextUnit task queue.
pub textunit_task_queue_path: PathBuf,
/// Number of queued task items.
pub task_count: usize,
/// Total queued TextUnit count.
pub text_unit_count: usize,
/// Queued items for the future Crowdin worker.
pub items: Vec<CrowdinTextUnitQueueItem>,
}
impl CrowdinTextUnitQueue {
/// Builds a Crowdin offline queue from queued TextUnit tasks.
pub fn from_textunit_task_queue(
task_queue_path: PathBuf,
task_queue: &OfficialTextUnitTaskQueue,
) -> Self {
let items = task_queue
.tasks
.iter()
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
.map(|task| CrowdinTextUnitQueueItem {
task_id: task.task_id.clone(),
destination: task.destination.clone(),
archive_entry: task.archive_entry.clone(),
text_unit_count: task.text_unit_count,
text_unit_formats: task.text_unit_formats.clone(),
})
.collect::<Vec<_>>();
let text_unit_count = items.iter().map(|item| item.text_unit_count).sum();
Self {
queue_version: CROWDIN_TEXTUNIT_QUEUE_VERSION,
provider: TranslationHandoffProvider::Crowdin,
status: TranslationHandoffStatus::QueuedOffline,
official_release_id: task_queue.official_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
textunit_task_queue_path: task_queue_path,
task_count: items.len(),
text_unit_count,
items,
}
}
fn matches_derivation(&self, expected: &Self) -> bool {
self.provider == expected.provider
&& self.status == expected.status
&& self.official_release_id == expected.official_release_id
&& self.textunit_task_queue_path == expected.textunit_task_queue_path
&& self.task_count == expected.task_count
&& self.text_unit_count == expected.text_unit_count
&& self.items == expected.items
}
}
/// Paths and summary produced after writing incremental TextUnit queues.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitQueueReport {
/// Path to `official-textunit-tasks.json`.
pub textunit_task_queue_path: PathBuf,
/// Path to `crowdin-textunit-queue.json`.
pub crowdin_textunit_queue_path: PathBuf,
/// Aggregate queue counters.
pub summary: OfficialTextUnitTaskSummary,
}
/// Reads change/parse derived queues from a release root and writes them back.
pub fn write_official_textunit_queues(
resource_root: &Path,
) -> Result<OfficialTextUnitQueueReport, String> {
let change_set = read_resource_change_set_at(resource_root)?.ok_or_else(|| {
format!(
"缺少官方资源变更集,无法生成 TextUnit 队列:{}",
resource_root.display()
)
})?;
let parse_cache = read_parse_cache_at(resource_root)?.ok_or_else(|| {
format!(
"缺少官方解析缓存,无法生成 TextUnit 队列:{}",
resource_root.display()
)
})?;
let task_queue =
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
write_textunit_task_queue_at(resource_root, &task_queue)?;
let task_queue_path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
let crowdin_queue =
CrowdinTextUnitQueue::from_textunit_task_queue(task_queue_path.clone(), &task_queue);
write_crowdin_textunit_queue_at(resource_root, &crowdin_queue)?;
Ok(OfficialTextUnitQueueReport {
textunit_task_queue_path: task_queue_path,
crowdin_textunit_queue_path: resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE),
summary: task_queue.summary,
})
}
/// Reads a generated TextUnit task queue from a release root.
pub fn read_textunit_task_queue_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitTaskQueue>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 任务队列")? else {
return Ok(None);
};
let queue: OfficialTextUnitTaskQueue = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析官方 TextUnit 任务队列失败 {}{error}", path.display()))?;
if queue.queue_version != OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION {
return Err(format!(
"不支持的官方 TextUnit 任务队列版本 {},文件 {}",
queue.queue_version,
path.display()
));
}
Ok(Some(queue))
}
/// Returns whether the persisted TextUnit queue still matches current inputs.
pub fn is_textunit_task_queue_current(
resource_root: &Path,
queue: &OfficialTextUnitTaskQueue,
) -> Result<bool, String> {
let Some(change_set) = read_resource_change_set_at(resource_root)? else {
return Ok(false);
};
let Some(parse_cache) = read_parse_cache_at(resource_root)? else {
return Ok(false);
};
let expected =
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
Ok(queue.matches_derivation(&expected))
}
/// Writes a TextUnit task queue under a release root.
pub fn write_textunit_task_queue_at(
resource_root: &Path,
queue: &OfficialTextUnitTaskQueue,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 任务队列")?;
let bytes = serde_json::to_vec_pretty(queue)
.map_err(|error| format!("序列化官方 TextUnit 任务队列失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 任务队列")
}
/// Reads a generated Crowdin offline TextUnit queue from a release root.
pub fn read_crowdin_textunit_queue_at(
resource_root: &Path,
) -> Result<Option<CrowdinTextUnitQueue>, String> {
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "Crowdin TextUnit 离线队列")? else {
return Ok(None);
};
let queue: CrowdinTextUnitQueue = serde_json::from_slice(&bytes).map_err(|error| {
format!(
"解析 Crowdin TextUnit 离线队列失败 {}{error}",
path.display()
)
})?;
if queue.queue_version != CROWDIN_TEXTUNIT_QUEUE_VERSION {
return Err(format!(
"不支持的 Crowdin TextUnit 离线队列版本 {},文件 {}",
queue.queue_version,
path.display()
));
}
Ok(Some(queue))
}
/// Returns whether the persisted Crowdin queue still matches a TextUnit queue.
pub fn is_crowdin_textunit_queue_current(
resource_root: &Path,
task_queue: &OfficialTextUnitTaskQueue,
) -> Result<bool, String> {
let Some(queue) = read_crowdin_textunit_queue_at(resource_root)? else {
return Ok(false);
};
let expected = CrowdinTextUnitQueue::from_textunit_task_queue(
resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
task_queue,
);
Ok(queue.matches_derivation(&expected))
}
/// Writes a Crowdin offline TextUnit queue under a release root.
pub fn write_crowdin_textunit_queue_at(
resource_root: &Path,
queue: &CrowdinTextUnitQueue,
) -> Result<(), String> {
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "Crowdin TextUnit 离线队列")?;
let bytes = serde_json::to_vec_pretty(queue)
.map_err(|error| format!("序列化 Crowdin TextUnit 离线队列失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin TextUnit 离线队列")
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn skipped_no_parse_entry_task(
change_set: &OfficialResourceChangeSet,
change: &OfficialResourceChange,
) -> OfficialTextUnitTask {
let current = change
.current
.as_ref()
.expect("parse candidate has current");
OfficialTextUnitTask {
task_id: task_id_for(&change_set.official_release_id, &change.destination, None),
official_release_id: change_set.official_release_id.clone(),
destination: change.destination.clone(),
change_kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
parse_entry_key: None,
archive_entry: None,
source_kind: None,
parse_status: None,
text_asset_count: 0,
text_assets: Vec::new(),
text_unit_count: 0,
text_unit_formats: Vec::new(),
text_unit_error_count: 0,
status: OfficialTextUnitTaskStatus::SkippedNoParseEntry,
reason: Some("parse cache entry not found for changed resource".to_string()),
}
}
fn task_from_parse_entry(
change_set: &OfficialResourceChangeSet,
change: &OfficialResourceChange,
entry: &OfficialParseCacheEntry,
) -> OfficialTextUnitTask {
let current = change
.current
.as_ref()
.expect("parse candidate has current");
let (status, reason) = status_for_parse_entry(entry);
OfficialTextUnitTask {
task_id: task_id_for(
&change_set.official_release_id,
&change.destination,
entry.archive_entry.as_deref(),
),
official_release_id: change_set.official_release_id.clone(),
destination: change.destination.clone(),
change_kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
parse_entry_key: Some(entry.key.clone()),
archive_entry: entry.archive_entry.clone(),
source_kind: Some(entry.source_kind),
parse_status: Some(entry.status),
text_asset_count: entry.text_asset_count,
text_assets: entry.text_assets.clone(),
text_unit_count: entry.text_unit_count,
text_unit_formats: entry.text_unit_formats.clone(),
text_unit_error_count: entry.text_unit_error_count,
status,
reason,
}
}
fn status_for_parse_entry(
entry: &OfficialParseCacheEntry,
) -> (OfficialTextUnitTaskStatus, Option<String>) {
match entry.status {
OfficialParseStatus::Parsed if entry.text_unit_count > 0 => {
(OfficialTextUnitTaskStatus::QueuedOffline, None)
}
OfficialParseStatus::Parsed => (
OfficialTextUnitTaskStatus::SkippedNoTextUnit,
Some("parsed resource has no TextUnit".to_string()),
),
OfficialParseStatus::Failed => (
OfficialTextUnitTaskStatus::SkippedParseFailed,
entry.error.clone(),
),
OfficialParseStatus::SkippedUnsupported => (
OfficialTextUnitTaskStatus::SkippedUnsupported,
entry.error.clone(),
),
}
}
fn task_id_for(
official_release_id: &str,
destination: &str,
archive_entry: Option<&str>,
) -> String {
match archive_entry {
Some(entry) => format!("textunit/{official_release_id}/{destination}#{entry}"),
None => format!("textunit/{official_release_id}/{destination}"),
}
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_textunit_task_queue_version() -> u32 {
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION
}
fn default_crowdin_textunit_queue_version() -> u32 {
CROWDIN_TEXTUNIT_QUEUE_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
use crate::official_changes::{
OfficialResourceChangeSet, OfficialResourceDescriptor, OFFICIAL_RESOURCE_CHANGES_VERSION,
};
use crate::official_parse::{
OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceFingerprint,
OfficialParseSourceKind, OfficialParseSummary, OFFICIAL_PARSE_CACHE_VERSION,
};
fn change_set(root: &Path) -> OfficialResourceChangeSet {
OfficialResourceChangeSet {
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
official_release_id: "release-new".to_string(),
previous_release_id: Some("release-old".to_string()),
generated_unix_seconds: 123,
previous_resource_root: None,
current_resource_root: root.to_path_buf(),
summary: crate::OfficialResourceChangeSummary {
previous_manifest_present: true,
previous_manifest_entry_count: 2,
current_manifest_entry_count: 3,
added_count: 2,
modified_count: 1,
removed_count: 1,
parse_candidate_count: 3,
translation_candidate_count: 3,
},
changes: vec![
change("Bundles/a.bundle", OfficialResourceChangeKind::Added, b"a"),
change(
"Bundles/b.bundle",
OfficialResourceChangeKind::Modified,
b"b",
),
change("Bundles/c.bundle", OfficialResourceChangeKind::Added, b"c"),
crate::OfficialResourceChange {
destination: "Bundles/removed.bundle".to_string(),
kind: OfficialResourceChangeKind::Removed,
previous: Some(OfficialResourceDescriptor {
url: "https://old/removed".to_string(),
destination: "Bundles/removed.bundle".to_string(),
bytes: 1,
blake3: "old".to_string(),
}),
current: None,
parse_candidate: false,
translation_candidate: false,
},
],
}
}
fn change(
destination: &str,
kind: OfficialResourceChangeKind,
bytes: &[u8],
) -> crate::OfficialResourceChange {
crate::OfficialResourceChange {
destination: destination.to_string(),
kind,
previous: None,
current: Some(OfficialResourceDescriptor {
url: format!("https://new/{destination}"),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
}),
parse_candidate: true,
translation_candidate: true,
}
}
fn parse_cache() -> OfficialParseCache {
OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 124,
summary: OfficialParseSummary {
manifest_entry_count: 3,
cache_entry_count: 2,
candidate_file_count: 2,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 1,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
},
entries: BTreeMap::from([
(
"direct:a".to_string(),
parse_entry(
"direct:a",
"Bundles/a.bundle",
OfficialParseStatus::Parsed,
2,
),
),
(
"direct:b".to_string(),
parse_entry(
"direct:b",
"Bundles/b.bundle",
OfficialParseStatus::SkippedUnsupported,
0,
),
),
]),
}
}
fn parse_entry(
key: &str,
destination: &str,
status: OfficialParseStatus,
text_unit_count: usize,
) -> OfficialParseCacheEntry {
OfficialParseCacheEntry {
key: key.to_string(),
source_url: format!("https://new/{destination}"),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url: format!("https://new/{destination}"),
destination: destination.to_string(),
bytes: 1,
blake3: "hash".to_string(),
},
status,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: usize::from(text_unit_count > 0),
text_assets: if text_unit_count > 0 {
vec!["Scenario".to_string()]
} else {
Vec::new()
},
serialized_parse_error_count: 0,
text_unit_count,
text_unit_formats: if text_unit_count > 0 {
vec!["plain".to_string()]
} else {
Vec::new()
},
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: (status != OfficialParseStatus::Parsed).then(|| "unsupported".to_string()),
}
}
#[test]
fn textunit_queue_uses_only_added_and_modified_parse_candidates() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
assert_eq!(queue.summary.resource_candidate_count, 3);
assert_eq!(queue.summary.parse_entry_count, 2);
assert_eq!(queue.summary.queued_task_count, 1);
assert_eq!(queue.summary.skipped_unsupported_count, 1);
assert_eq!(queue.summary.skipped_no_parse_entry_count, 1);
assert_eq!(queue.summary.text_unit_count, 2);
assert_eq!(
queue
.tasks
.iter()
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
.map(|task| task.destination.as_str())
.collect::<Vec<_>>(),
vec!["Bundles/a.bundle"]
);
}
#[test]
fn crowdin_queue_contains_only_queued_textunit_tasks() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
&queue,
);
assert_eq!(crowdin.provider, TranslationHandoffProvider::Crowdin);
assert_eq!(crowdin.status, TranslationHandoffStatus::QueuedOffline);
assert_eq!(crowdin.task_count, 1);
assert_eq!(crowdin.text_unit_count, 2);
assert_eq!(crowdin.items[0].destination, "Bundles/a.bundle");
}
#[test]
fn write_textunit_queues_persists_files() {
let temp = tempfile::TempDir::new().unwrap();
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
let report = write_official_textunit_queues(temp.path()).unwrap();
assert_eq!(report.summary.queued_task_count, 1);
assert!(report.textunit_task_queue_path.is_file());
assert!(report.crowdin_textunit_queue_path.is_file());
let persisted = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
assert_eq!(persisted.summary.text_unit_count, 2);
}
#[test]
fn textunit_queue_current_detects_parse_cache_changes() {
let temp = tempfile::TempDir::new().unwrap();
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
write_official_textunit_queues(temp.path()).unwrap();
let queue = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
assert!(is_textunit_task_queue_current(temp.path(), &queue).unwrap());
let mut changed_cache = parse_cache();
changed_cache.entries.insert(
"direct:c".to_string(),
parse_entry(
"direct:c",
"Bundles/c.bundle",
OfficialParseStatus::Parsed,
1,
),
);
crate::write_parse_cache_at(temp.path(), &changed_cache).unwrap();
assert!(!is_textunit_task_queue_current(temp.path(), &queue).unwrap());
}
#[test]
fn crowdin_queue_current_requires_matching_file() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
write_textunit_task_queue_at(temp.path(), &queue).unwrap();
assert!(!is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
&queue,
);
write_crowdin_textunit_queue_at(temp.path(), &crowdin).unwrap();
assert!(is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
}
}
File diff suppressed because it is too large Load Diff
+424
View File
@@ -0,0 +1,424 @@
//! File-level Patch and UnityFS write operations.
use std::path::{Path, PathBuf};
use bat_assetbundle::{
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
StringFieldPatch, TextAssetPatch, UnitySerializedReplacementValue,
};
use serde::{Deserialize, Serialize};
use crate::path_security::{
ensure_safe_file_target, lexical_absolute, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
/// File patch algorithm selected by `patch.apply`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PatchApplyKind {
/// Deterministic binary hunk patch.
Binary,
/// RFC 6902 JSON Patch.
Json,
/// UTF-8 Text Patch.
Text,
}
impl PatchApplyKind {
/// Stable RPC/CLI label.
pub fn as_str(self) -> &'static str {
match self {
Self::Binary => "binary",
Self::Json => "json",
Self::Text => "text",
}
}
}
/// Parameters for applying one file-level patch.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PatchApplyParams {
/// Patch algorithm.
pub kind: PatchApplyKind,
/// Source file path.
pub source_path: PathBuf,
/// Patch document path.
pub patch_path: PathBuf,
/// Target file path written atomically.
pub target_path: PathBuf,
}
/// Parameters for patching one UnityFS TextAsset object.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsTextAssetPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Replacement raw bytes path.
pub replacement_path: PathBuf,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected TextAsset name.
#[serde(default)]
pub expected_name: Option<String>,
}
/// Parameters for patching one TypeTree string field inside a UnityFS bundle.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsStringFieldPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Stable TypeTree field path.
pub field_path: String,
/// Replacement string supplied inline.
#[serde(default)]
pub replacement_text: Option<String>,
/// Replacement UTF-8 file path.
#[serde(default)]
pub replacement_path: Option<PathBuf>,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected source string.
#[serde(default)]
pub expected_value: Option<String>,
}
/// Parameters for patching one semantic TypeTree field inside a UnityFS bundle.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsFieldPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Stable TypeTree field path.
pub field_path: String,
/// Replacement value encoded according to the current TypeTree field type.
pub replacement: UnitySerializedReplacementValue,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected source value.
#[serde(default)]
pub expected_value: Option<UnitySerializedReplacementValue>,
}
/// Result of a file-level patch operation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct PatchApplyReport {
/// RPC/CLI command name.
pub command: &'static str,
/// Operation status.
pub status: &'static str,
/// Human-readable summary.
pub message: &'static str,
/// Patch algorithm.
pub kind: PatchApplyKind,
/// Absolute source path.
pub source_path: PathBuf,
/// Absolute patch path.
pub patch_path: PathBuf,
/// Absolute target path.
pub target_path: PathBuf,
/// Source byte length.
pub source_size: u64,
/// Patch document byte length.
pub patch_size: u64,
/// Target byte length.
pub target_size: u64,
/// Source BLAKE3 hash.
pub source_blake3: String,
/// Patch document BLAKE3 hash.
pub patch_blake3: String,
/// Target BLAKE3 hash.
pub target_blake3: String,
}
/// Result of a UnityFS write operation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct UnityFsPatchReport {
/// RPC/CLI command name.
pub command: &'static str,
/// Operation status.
pub status: &'static str,
/// Human-readable summary.
pub message: &'static str,
/// Absolute source UnityFS bundle path.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Optional TypeTree field path for string-field patches.
#[serde(skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Absolute target bundle path.
pub target_path: PathBuf,
/// Source bundle byte length.
pub source_size: u64,
/// Replacement byte length.
pub replacement_size: u64,
/// Target bundle byte length.
pub target_size: u64,
/// Source bundle BLAKE3 hash.
pub source_blake3: String,
/// Replacement BLAKE3 hash.
pub replacement_blake3: String,
/// Target bundle BLAKE3 hash.
pub target_blake3: String,
}
/// Applies a Binary/JSON/Text patch document to one source file.
pub fn apply_patch_file(params: &PatchApplyParams) -> anyhow::Result<PatchApplyReport> {
let source_path = lexical_absolute(&params.source_path).map_err(anyhow::Error::msg)?;
let patch_path = lexical_absolute(&params.patch_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&source_path, &patch_path])?;
let source = read_required_file(&source_path, "patch source")?;
let patch = read_required_file(&patch_path, "patch document")?;
let target = match params.kind {
PatchApplyKind::Binary => bat_patch::binary::apply_patch(&source, &patch)?,
PatchApplyKind::Json => {
let source = std::str::from_utf8(&source)
.map_err(|error| anyhow::anyhow!("JSON patch source 不是 UTF-8{error}"))?;
let patch = std::str::from_utf8(&patch)
.map_err(|error| anyhow::anyhow!("JSON patch document 不是 UTF-8{error}"))?;
bat_patch::json::apply_json_patch(source, patch)?.into_bytes()
}
PatchApplyKind::Text => bat_patch::text::apply_patch_bytes(&source, &patch)?,
};
write_output_file(&target_path, &target, "patch target")?;
Ok(PatchApplyReport {
command: "patch.apply",
status: "patched",
message: "patch 已应用并原子写入目标文件",
kind: params.kind,
source_path,
patch_path,
target_path,
source_size: source.len() as u64,
patch_size: patch.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&source),
patch_blake3: blake3_hex(&patch),
target_blake3: blake3_hex(&target),
})
}
/// Patches one UnityFS TextAsset and writes the rebuilt bundle atomically.
pub fn apply_unityfs_text_asset_patch_file(
params: &UnityFsTextAssetPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let replacement_path =
lexical_absolute(&params.replacement_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&bundle_path, &replacement_path])?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let replacement = read_required_file(&replacement_path, "TextAsset replacement")?;
let mut patch = TextAssetPatch::new(
params.serialized_file_path.clone(),
params.path_id,
replacement.clone(),
);
patch.expected_name = params.expected_name.clone();
let target = patch_unityfs_text_asset(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_text_asset",
status: "patched",
message: "UnityFS TextAsset patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: None,
target_path,
source_size: bundle.len() as u64,
replacement_size: replacement.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(&replacement),
target_blake3: blake3_hex(&target),
})
}
/// Patches one TypeTree string field and writes the rebuilt UnityFS bundle atomically.
pub fn apply_unityfs_string_field_patch_file(
params: &UnityFsStringFieldPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
let replacement = replacement_text(params)?;
let extra_inputs = params
.replacement_path
.as_ref()
.map(|path| lexical_absolute(path).map_err(anyhow::Error::msg))
.transpose()?;
let mut inputs = vec![bundle_path.as_path()];
if let Some(path) = extra_inputs.as_ref() {
inputs.push(path.as_path());
}
ensure_target_is_not_input(&target_path, &inputs)?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let patch = StringFieldPatch {
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: params.field_path.clone(),
expected_value: params.expected_value.clone(),
replacement: replacement.clone(),
};
let target = patch_unityfs_string_field(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_string_field",
status: "patched",
message: "UnityFS TypeTree string field patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: Some(params.field_path.clone()),
target_path,
source_size: bundle.len() as u64,
replacement_size: replacement.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(replacement.as_bytes()),
target_blake3: blake3_hex(&target),
})
}
/// Patches one semantic TypeTree field and writes the rebuilt UnityFS bundle atomically.
pub fn apply_unityfs_field_patch_file(
params: &UnityFsFieldPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&bundle_path])?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let replacement_size = serde_json::to_vec(&params.replacement)
.map_err(anyhow::Error::from)?
.len() as u64;
let patch = FieldPatch {
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: params.field_path.clone(),
expected_value: params.expected_value.clone(),
replacement: params.replacement.clone(),
};
let target = patch_unityfs_field(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_field",
status: "patched",
message: "UnityFS TypeTree field patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: Some(params.field_path.clone()),
target_path,
source_size: bundle.len() as u64,
replacement_size,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(&serde_json::to_vec(&params.replacement)?),
target_blake3: blake3_hex(&target),
})
}
fn replacement_text(params: &UnityFsStringFieldPatchParams) -> anyhow::Result<String> {
match (&params.replacement_text, &params.replacement_path) {
(Some(_), Some(_)) => Err(anyhow::anyhow!(
"replacement_text 和 replacement_path 只能指定一个"
)),
(Some(text), None) => Ok(text.clone()),
(None, Some(path)) => {
let path = lexical_absolute(path).map_err(anyhow::Error::msg)?;
let bytes = read_required_file(&path, "string replacement")?;
String::from_utf8(bytes)
.map_err(|error| anyhow::anyhow!("string replacement 不是 UTF-8{error}"))
}
(None, None) => Err(anyhow::anyhow!(
"必须指定 replacement_text 或 replacement_path"
)),
}
}
fn read_required_file(path: &Path, label: &str) -> anyhow::Result<Vec<u8>> {
read_file_no_symlink(path, label)
.map_err(anyhow::Error::msg)?
.ok_or_else(|| anyhow::anyhow!("{label} 不存在:{}", path.display()))
}
fn write_output_file(path: &Path, bytes: &[u8], label: &str) -> anyhow::Result<()> {
let parent = path
.parent()
.ok_or_else(|| anyhow::anyhow!("{label} 缺少父目录:{}", path.display()))?;
ensure_safe_file_target(parent, path, label).map_err(anyhow::Error::msg)?;
write_file_atomic(path, bytes, STATE_FILE_MODE, label).map_err(anyhow::Error::msg)
}
fn ensure_target_is_not_input(target: &Path, inputs: &[&Path]) -> anyhow::Result<()> {
for input in inputs {
if target == *input {
return Err(anyhow::anyhow!(
"target_path 不能与输入文件相同:{}",
target.display()
));
}
}
Ok(())
}
fn blake3_hex(bytes: &[u8]) -> String {
blake3::hash(bytes).to_hex().to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn patch_apply_rejects_in_place_target() {
let path = PathBuf::from("/tmp/source.bin");
let params = PatchApplyParams {
kind: PatchApplyKind::Binary,
source_path: path.clone(),
patch_path: PathBuf::from("/tmp/patch.json"),
target_path: path,
};
let error = apply_patch_file(&params).unwrap_err().to_string();
assert!(error.contains("target_path 不能与输入文件相同"));
}
#[test]
fn string_field_params_require_one_replacement_source() {
let params = UnityFsStringFieldPatchParams {
bundle_path: PathBuf::from("/tmp/source.bundle"),
serialized_file_path: "CAB".to_string(),
path_id: 1,
field_path: "message".to_string(),
replacement_text: Some("a".to_string()),
replacement_path: Some(PathBuf::from("/tmp/replacement.txt")),
target_path: PathBuf::from("/tmp/target.bundle"),
expected_value: None,
};
let error = replacement_text(&params).unwrap_err().to_string();
assert!(error.contains("只能指定一个"));
}
}
+59 -11
View File
@@ -1,7 +1,7 @@
//! 内存资源仓储实现。 //! 内存资源仓储实现。
use async_trait::async_trait; use async_trait::async_trait;
use bat_core::domain::{Resource, ResourceEntry, ResourceType}; use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository}; use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult};
use sqlx::{QueryBuilder, Sqlite, SqlitePool}; use sqlx::{QueryBuilder, Sqlite, SqlitePool};
@@ -130,7 +130,8 @@ impl SqliteResourceRepository {
local_path TEXT NOT NULL, local_path TEXT NOT NULL,
address TEXT, address TEXT,
dependencies_json TEXT NOT NULL DEFAULT '[]', dependencies_json TEXT NOT NULL DEFAULT '[]',
crc INTEGER crc INTEGER,
metadata_json TEXT NOT NULL DEFAULT '{}'
) )
"#, "#,
), ),
@@ -140,6 +141,13 @@ impl SqliteResourceRepository {
// 向后兼容:早于 crc 列的旧库缺少该列,按需补加(新建库已含该列, // 向后兼容:早于 crc 列的旧库缺少该列,按需补加(新建库已含该列,
// pragma 检查后不会重复 ALTER)。 // pragma 检查后不会重复 ALTER)。
Self::ensure_column(&self.pool, "resources", "crc", "INTEGER").await?; Self::ensure_column(&self.pool, "resources", "crc", "INTEGER").await?;
Self::ensure_column(
&self.pool,
"resources",
"metadata_json",
"TEXT NOT NULL DEFAULT '{}'",
)
.await?;
Self::execute_query( Self::execute_query(
&self.pool, &self.pool,
@@ -250,9 +258,32 @@ impl SqliteResourceRepository {
.map_err(|error| bat_core::Error::Serialization(error.to_string())) .map_err(|error| bat_core::Error::Serialization(error.to_string()))
} }
fn metadata_to_json(metadata: &ResourceMetadata) -> bat_core::Result<String> {
serde_json::to_string(metadata)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
}
fn metadata_from_json(value: &str) -> bat_core::Result<ResourceMetadata> {
if value.trim().is_empty() {
return Ok(ResourceMetadata::default());
}
serde_json::from_str(value)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
}
fn resource_from_row(row: ResourceRow) -> bat_core::Result<Resource> { fn resource_from_row(row: ResourceRow) -> bat_core::Result<Resource> {
let (id, path, hash, size, resource_type, local_path, address, dependencies_json, crc) = let (
row; id,
path,
hash,
size,
resource_type,
local_path,
address,
dependencies_json,
crc,
metadata_json,
) = row;
Ok(Resource { Ok(Resource {
id, id,
local_path: PathBuf::from(local_path), local_path: PathBuf::from(local_path),
@@ -265,6 +296,7 @@ impl SqliteResourceRepository {
dependencies: Self::dependencies_from_json(&dependencies_json)?, dependencies: Self::dependencies_from_json(&dependencies_json)?,
crc: crc.and_then(|value| u32::try_from(value).ok()), crc: crc.and_then(|value| u32::try_from(value).ok()),
}, },
metadata: Self::metadata_from_json(&metadata_json)?,
}) })
} }
@@ -303,7 +335,7 @@ impl SqliteResourceRepository {
limit: Option<usize>, limit: Option<usize>,
) -> bat_core::Result<Vec<Resource>> { ) -> bat_core::Result<Vec<Resource>> {
let mut builder = QueryBuilder::<Sqlite>::new( let mut builder = QueryBuilder::<Sqlite>::new(
"SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc FROM resources", "SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json FROM resources",
); );
Self::apply_filters(&mut builder, query)?; Self::apply_filters(&mut builder, query)?;
builder.push(" ORDER BY id"); builder.push(" ORDER BY id");
@@ -337,14 +369,15 @@ impl SqliteResourceRepository {
impl ResourceRepository for SqliteResourceRepository { impl ResourceRepository for SqliteResourceRepository {
async fn add(&self, resource: Resource) -> bat_core::Result<String> { async fn add(&self, resource: Resource) -> bat_core::Result<String> {
let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?; let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?;
let metadata = Self::metadata_to_json(&resource.metadata)?;
Self::execute_query( Self::execute_query(
&self.pool, &self.pool,
sqlx::query( sqlx::query(
r#" r#"
INSERT INTO resources ( INSERT INTO resources (
id, path, hash, size, resource_type, local_path, address, dependencies_json, crc id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
) )
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
ON CONFLICT(id) DO UPDATE SET ON CONFLICT(id) DO UPDATE SET
path = excluded.path, path = excluded.path,
hash = excluded.hash, hash = excluded.hash,
@@ -353,7 +386,8 @@ impl ResourceRepository for SqliteResourceRepository {
local_path = excluded.local_path, local_path = excluded.local_path,
address = excluded.address, address = excluded.address,
dependencies_json = excluded.dependencies_json, dependencies_json = excluded.dependencies_json,
crc = excluded.crc crc = excluded.crc,
metadata_json = excluded.metadata_json
"#, "#,
) )
.bind(resource.id.clone()) .bind(resource.id.clone())
@@ -364,7 +398,8 @@ impl ResourceRepository for SqliteResourceRepository {
.bind(resource.local_path.to_string_lossy().to_string()) .bind(resource.local_path.to_string_lossy().to_string())
.bind(resource.entry.address.clone()) .bind(resource.entry.address.clone())
.bind(dependencies) .bind(dependencies)
.bind(resource.entry.crc.map(i64::from)), .bind(resource.entry.crc.map(i64::from))
.bind(metadata),
) )
.await?; .await?;
@@ -374,7 +409,7 @@ impl ResourceRepository for SqliteResourceRepository {
async fn find_by_id(&self, id: &str) -> bat_core::Result<Resource> { async fn find_by_id(&self, id: &str) -> bat_core::Result<Resource> {
let row: Option<ResourceRow> = sqlx::query_as( let row: Option<ResourceRow> = sqlx::query_as(
r#" r#"
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
FROM resources FROM resources
WHERE id = ?1 WHERE id = ?1
"#, "#,
@@ -392,7 +427,7 @@ impl ResourceRepository for SqliteResourceRepository {
async fn find_by_hash(&self, hash: &str) -> bat_core::Result<Resource> { async fn find_by_hash(&self, hash: &str) -> bat_core::Result<Resource> {
let row: Option<ResourceRow> = sqlx::query_as( let row: Option<ResourceRow> = sqlx::query_as(
r#" r#"
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
FROM resources FROM resources
WHERE hash = ?1 WHERE hash = ?1
ORDER BY id ORDER BY id
@@ -455,6 +490,7 @@ type ResourceRow = (
Option<String>, Option<String>,
String, String,
Option<i64>, Option<i64>,
String,
); );
fn glob_to_like(pattern: &str) -> String { fn glob_to_like(pattern: &str) -> String {
@@ -544,6 +580,7 @@ mod tests {
dependencies: Vec::new(), dependencies: Vec::new(),
crc: None, crc: None,
}, },
metadata: ResourceMetadata::default(),
} }
} }
@@ -638,6 +675,10 @@ mod tests {
.entry .entry
.dependencies .dependencies
.push("assets/shared.bundle".to_string()); .push("assets/shared.bundle".to_string());
resource.metadata.official_release_id = Some("release-1".to_string());
resource.metadata.platform = Some("windows".to_string());
resource.metadata.text_assets = vec!["Scenario".to_string()];
resource.metadata.text_unit_count = 3;
repository.add(resource.clone()).await.unwrap(); repository.add(resource.clone()).await.unwrap();
@@ -647,6 +688,13 @@ mod tests {
by_id.entry.dependencies, by_id.entry.dependencies,
vec!["assets/shared.bundle".to_string()] vec!["assets/shared.bundle".to_string()]
); );
assert_eq!(
by_id.metadata.official_release_id.as_deref(),
Some("release-1")
);
assert_eq!(by_id.metadata.platform.as_deref(), Some("windows"));
assert_eq!(by_id.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(by_id.metadata.text_unit_count, 3);
assert_eq!( assert_eq!(
repository.find_by_hash("hash-sqlite-a").await.unwrap().id, repository.find_by_hash("hash-sqlite-a").await.unwrap().id,
resource.id resource.id
@@ -367,14 +367,16 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
let curl_failure_state = harness.temp.path().join("curl-failure.state"); let curl_failure_state = harness.temp.path().join("curl-failure.state");
write_executable( write_executable(
&harness.curl_script, &harness.curl_script,
&official_curl_script( &official_curl_script(OfficialCurlScriptFixture {
&harness.curl_log, curl_log: &harness.curl_log,
&zip_fixture, zip_fixture: &zip_fixture,
&resources_assets_path, resources_assets_fixture: &resources_assets_path,
fs::metadata(&resources_assets_path).unwrap().len() as usize, resources_assets_size: fs::metadata(&resources_assets_path).unwrap().len() as usize,
TEST_LAUNCHER_MANIFEST_SOURCE, manifest_source: TEST_LAUNCHER_MANIFEST_SOURCE,
Some(&curl_failure_state), failure_state: Some(&curl_failure_state),
), clientpatch_unavailable: false,
seed_unavailable_only: false,
}),
); );
let config = harness.sync_config("failed-staging-output"); let config = harness.sync_config("failed-staging-output");
@@ -399,6 +401,77 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
assert!(version_state.current_completed_version.is_some()); assert!(version_state.current_completed_version.is_some());
} }
#[test]
fn official_update_waits_when_client_patch_markers_are_not_ready() {
let harness = TestHarness::new_with_clientpatch_unavailable(false);
let config = harness.sync_config("clientpatch-marker-wait-output");
let mut events = Vec::<OfficialUpdateProgress>::new();
let report = OfficialUpdateService::new()
.run_with_progress(&config, |event| events.push(event))
.unwrap();
assert_eq!(
report.update_status,
OfficialUpdateStatus::WaitingForOfficialResources
);
assert!(report.waiting_for_official_resources);
assert!(!report.should_download);
assert!(report
.unavailable_endpoints
.iter()
.any(|endpoint| endpoint.url.ends_with("/TableBundles/TableCatalog.hash")));
assert!(report
.unavailable_endpoints
.iter()
.all(|endpoint| endpoint.error_kind == "http_forbidden"));
assert!(report.staging_path.is_none());
assert!(report.published_version_path.is_none());
assert!(!config.output_root.join("current").exists());
assert!(!config.output_root.join(".staging").exists());
assert!(read_version_state(&report.version_state_path)
.unwrap()
.is_none());
assert!(events
.iter()
.any(|event| event.stage == "upstream" && event.message.contains("尚未开放")));
}
#[test]
fn official_update_waits_when_required_seed_catalog_is_not_ready() {
let harness = TestHarness::new_with_clientpatch_unavailable(true);
let config = harness.sync_config("clientpatch-seed-wait-output");
let mut events = Vec::<OfficialUpdateProgress>::new();
let report = OfficialUpdateService::new()
.run_with_progress(&config, |event| events.push(event))
.unwrap();
assert_eq!(
report.update_status,
OfficialUpdateStatus::WaitingForOfficialResources
);
assert!(report.waiting_for_official_resources);
assert_eq!(report.unavailable_endpoints.len(), 1);
assert!(report.unavailable_endpoints[0]
.url
.ends_with("/TableBundles/TableCatalog.bytes"));
assert_eq!(report.unavailable_endpoints[0].http_status, Some(403));
assert!(report.staging_path.is_none());
assert!(report.published_version_path.is_none());
assert!(!config.output_root.join("current").exists());
assert!(!config.output_root.join(".staging").exists());
assert!(read_version_state(&report.version_state_path)
.unwrap()
.is_none());
assert!(events
.iter()
.any(|event| event.stage == "catalog" && event.message.contains("TableCatalog.bytes")));
assert!(events
.iter()
.any(|event| event.stage == "upstream" && event.message.contains("必需资源")));
}
struct TestHarness { struct TestHarness {
temp: TempDir, temp: TempDir,
curl_script: std::path::PathBuf, curl_script: std::path::PathBuf,
@@ -412,6 +485,18 @@ impl TestHarness {
} }
fn new_with_manifest_source(manifest_source: &str) -> Self { fn new_with_manifest_source(manifest_source: &str) -> Self {
Self::new_with_options(manifest_source, false, false)
}
fn new_with_clientpatch_unavailable(seed_only: bool) -> Self {
Self::new_with_options(TEST_LAUNCHER_MANIFEST_SOURCE, true, seed_only)
}
fn new_with_options(
manifest_source: &str,
clientpatch_unavailable: bool,
seed_unavailable_only: bool,
) -> Self {
let temp = TempDir::new().unwrap(); let temp = TempDir::new().unwrap();
let resources_assets = synthetic_resources_assets(); let resources_assets = synthetic_resources_assets();
@@ -425,14 +510,16 @@ impl TestHarness {
let curl_script = temp.path().join("fake-curl"); let curl_script = temp.path().join("fake-curl");
write_executable( write_executable(
&curl_script, &curl_script,
&official_curl_script( &official_curl_script(OfficialCurlScriptFixture {
&curl_log, curl_log: &curl_log,
&zip_fixture, zip_fixture: &zip_fixture,
&resources_assets_path, resources_assets_fixture: &resources_assets_path,
resources_assets.len(), resources_assets_size: resources_assets.len(),
manifest_source, manifest_source,
None, failure_state: None,
), clientpatch_unavailable,
seed_unavailable_only,
}),
); );
let unzip_script = temp.path().join("fake-unzip"); let unzip_script = temp.path().join("fake-unzip");
@@ -542,15 +629,20 @@ fn one_file_zip(name: &[u8], data: &[u8]) -> Vec<u8> {
bytes bytes
} }
fn official_curl_script( struct OfficialCurlScriptFixture<'a> {
curl_log: &Path, curl_log: &'a Path,
zip_fixture: &Path, zip_fixture: &'a Path,
resources_assets_fixture: &Path, resources_assets_fixture: &'a Path,
resources_assets_size: usize, resources_assets_size: usize,
manifest_source: &str, manifest_source: &'a str,
failure_state: Option<&Path>, failure_state: Option<&'a Path>,
) -> String { clientpatch_unavailable: bool,
let failure_state = failure_state seed_unavailable_only: bool,
}
fn official_curl_script(fixture: OfficialCurlScriptFixture<'_>) -> String {
let failure_state = fixture
.failure_state
.map(shell_quote) .map(shell_quote)
.unwrap_or_else(|| "''".to_string()); .unwrap_or_else(|| "''".to_string());
format!( format!(
@@ -614,6 +706,31 @@ maybe_fail_once() {{
fi fi
}} }}
clientpatch_is_unavailable() {{
if [[ "{clientpatch_unavailable}" != "true" ]]; then
return 1
fi
if [[ "$url" != "{addressables_root}/"* ]]; then
return 1
fi
if [[ "{seed_unavailable_only}" == "true" ]]; then
case "$url" in
*/TableBundles/TableCatalog.bytes|*/BundlePackingInfo.bytes|*/Catalog/MediaCatalog.bytes)
return 0
;;
*)
return 1
;;
esac
fi
return 0
}}
if clientpatch_is_unavailable; then
echo "curl: (22) The requested URL returned error: 403" >&2
exit 22
fi
if [[ "$url" == "https://api-launcher-jp.yo-star.com/api/launcher/game/config" ]]; then if [[ "$url" == "https://api-launcher-jp.yo-star.com/api/launcher/game/config" ]]; then
cat <<'JSON' cat <<'JSON'
{{"code":200,"message":"ok","data":{{"game_latest_version":"{launcher_latest_version}","game_latest_file_path":"{launcher_latest_file_path}"}}}} {{"code":200,"message":"ok","data":{{"game_latest_version":"{launcher_latest_version}","game_latest_file_path":"{launcher_latest_file_path}"}}}}
@@ -679,16 +796,16 @@ else
exit 1 exit 1
fi fi
"#, "#,
shell_quote(curl_log), shell_quote(fixture.curl_log),
shell_quote(zip_fixture), shell_quote(fixture.zip_fixture),
shell_quote(resources_assets_fixture), shell_quote(fixture.resources_assets_fixture),
launcher_latest_version = TEST_LAUNCHER_LATEST_VERSION, launcher_latest_version = TEST_LAUNCHER_LATEST_VERSION,
launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH, launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH,
launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL, launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL,
launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL, launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL,
launcher_manifest_source = manifest_source, launcher_manifest_source = fixture.manifest_source,
launcher_resources_assets_url = TEST_LAUNCHER_RESOURCES_ASSETS_URL, launcher_resources_assets_url = TEST_LAUNCHER_RESOURCES_ASSETS_URL,
resources_assets_size = resources_assets_size, resources_assets_size = fixture.resources_assets_size,
server_info_url = TEST_SERVER_INFO_URL, server_info_url = TEST_SERVER_INFO_URL,
connection_group = TEST_CONNECTION_GROUP, connection_group = TEST_CONNECTION_GROUP,
addressables_root = TEST_ADDRESSABLES_ROOT, addressables_root = TEST_ADDRESSABLES_ROOT,
@@ -698,6 +815,8 @@ fi
android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"), android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"),
android_media_catalog_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"), android_media_catalog_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"),
failure_state = failure_state, failure_state = failure_state,
clientpatch_unavailable = fixture.clientpatch_unavailable,
seed_unavailable_only = fixture.seed_unavailable_only,
) )
} }