feat(sync): 接入解析缓存与汉化发布前置

补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
2026-07-31 00:38:45 +08:00
parent f4880a71bd
commit 2079c6a307
25 changed files with 17287 additions and 225 deletions
Generated
+2
View File
@@ -88,6 +88,7 @@ dependencies = [
"hex",
"lz4",
"lzma-rs",
"md-5",
"serde",
"serde_json",
"thiserror",
@@ -145,6 +146,7 @@ dependencies = [
"bat-assetbundle",
"bat-cas-engine",
"bat-core",
"bat-patch",
"blake3",
"hex",
"libc",
+2 -2
View File
@@ -13,7 +13,7 @@ pub use adapter::{
};
pub use registry::UnityAdapterRegistry;
pub use serialized_file::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
UnityTypeTreeNode,
UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
};
pub use unity_2021_3::Unity2021_3Adapter;
+2 -2
View File
@@ -4,6 +4,6 @@
//! existing call sites can continue to import through `bat_adapters::unity`.
pub use bat_assetbundle::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
UnityTypeTreeNode,
UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
};
+3 -1
View File
@@ -7,7 +7,9 @@ pub mod translation;
pub use game_client::{ClientStatus, GameClient, GameRegion};
pub use game_version::{GameVersion, UnityVersion};
pub use resource::{crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceType};
pub use resource::{
crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceMetadata, ResourceType,
};
pub use translation::{
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
TranslationStatus,
+55
View File
@@ -43,6 +43,58 @@ pub struct ResourceEntry {
pub crc: Option<u32>,
}
/// 资源解析与发布侧元数据。
///
/// 该结构默认全空,保证旧索引和只保存基础 manifest 信息的资源仍可反序列化。
/// 官方资源导入会按 release manifest 和 parse cache 填充这些字段,供
/// `resource.index` 等只读接口暴露版本、平台、bundle、TextAsset 和 TextUnit 摘要。
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ResourceMetadata {
/// 资源所属的官方 release ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub official_release_id: Option<String>,
/// 从官方相对路径推断的平台标签,例如 `windows` 或 `android`。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub platform: Option<String>,
/// 资源本身或所在 bundle 的官方相对路径。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bundle_path: Option<String>,
/// ZIP 内被解析到的 bundle entry;直接 bundle 为空。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub archive_entries: Vec<String>,
/// parse cache 中出现过的解析状态标签。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub parse_statuses: Vec<String>,
/// 解析到的 Unity 版本集合。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unity_versions: Vec<String>,
/// UnityFS directory file 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub unityfs_file_count: u64,
/// Unity serialized file 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub serialized_file_count: u64,
/// TextAsset 对象总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_asset_count: u64,
/// TextAsset 名称集合。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub text_assets: Vec<String>,
/// TextUnit 总数。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_unit_count: u64,
/// TextUnit 格式标签集合,例如 `json`、`csv`、`tsv`、`plain`。
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub text_unit_formats: Vec<String>,
/// TextUnit 提取阶段的非致命诊断数量。
#[serde(default, skip_serializing_if = "is_zero")]
pub text_unit_error_count: u64,
}
fn is_zero(value: &u64) -> bool {
*value == 0
}
/// 已下载字节与 catalog 声明的可校验字段不一致。
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IntegrityMismatch {
@@ -133,6 +185,9 @@ pub struct Resource {
pub local_path: PathBuf,
/// 资源条目
pub entry: ResourceEntry,
/// 解析、发布和索引侧扩展元数据。
#[serde(default)]
pub metadata: ResourceMetadata,
}
#[cfg(test)]
+1
View File
@@ -13,6 +13,7 @@ serde_json.workspace = true
tracing.workspace = true
lz4 = "1.28"
lzma-rs = "0.3"
md-5 = "0.10"
[dev-dependencies]
hex = "0.4"
+14 -2
View File
@@ -9,14 +9,26 @@
pub mod error;
pub mod parser;
pub mod patch;
pub mod serialized;
pub mod text;
pub mod types;
pub use error::{AssetBundleError, Result};
pub use parser::{compression_from_flags, Parser, UnityFsParser};
pub use patch::{
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
StringFieldPatch, TextAssetPatch,
};
pub use serialized::{
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
UnityTypeTreeNode,
UnityManagedReferenceMetadata, UnityManagedReferenceRecord, UnitySerializedField,
UnitySerializedFieldReplacement, UnitySerializedFile, UnitySerializedObject,
UnitySerializedReplacementValue, UnitySerializedTextAsset, UnitySerializedType,
UnitySerializedValue, UnityTypeTreeNode,
};
pub use text::{
text_units_to_jsonl, TextUnit, TextUnitExtractionError, TextUnitExtractionReport,
TextUnitExtractor,
};
pub use types::{
AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+949
View File
@@ -0,0 +1,949 @@
//! Text extraction from parsed Unity serialized objects.
use crate::serialized::{
managed_reference_metadata_from_fields, UnityManagedReferenceMetadata,
UnityManagedReferenceRecord, UnitySerializedField, UnitySerializedValue,
};
use crate::types::ParsedAssetBundle;
use std::collections::BTreeMap;
/// One text unit used by translation, glossary and patch pipelines.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnit {
/// Original source text.
pub source_text: String,
/// Logical AssetBundle path, when provided by the caller.
pub bundle_path: Option<String>,
/// ZIP/archive entry containing the bundle, when known.
pub archive_entry: Option<String>,
/// Unity serialized file path.
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID, for example `49` for `TextAsset`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path. `TextAsset` is used for a whole TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// Unity version associated with the source.
pub version: String,
/// Stable context for format, asset name and extraction details.
pub context: BTreeMap<String, String>,
}
/// Non-fatal diagnostic generated while extracting text units.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnitExtractionError {
/// Serialized file containing the failed object.
pub serialized_file: Option<String>,
/// Object path ID, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable error.
pub error: String,
}
/// Result of extracting text units from one parsed bundle.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TextUnitExtractionReport {
/// Extracted text units in deterministic traversal order.
pub units: Vec<TextUnit>,
/// Non-fatal object-level errors.
pub errors: Vec<TextUnitExtractionError>,
/// TextAsset payloads that were binary or invalid UTF-8.
pub skipped_binary_text_assets: usize,
}
/// Extracts translation-ready text from Unity bundle data.
#[derive(Debug, Default, Clone, Copy)]
pub struct TextUnitExtractor;
impl TextUnitExtractor {
/// Creates an extractor.
pub fn new() -> Self {
Self
}
/// Extracts TextAsset and TypeTree string fields from a parsed bundle.
pub fn extract_bundle(
&self,
bundle: &ParsedAssetBundle,
bundle_path: Option<&str>,
) -> TextUnitExtractionReport {
self.extract_bundle_with_context(bundle, bundle_path, None)
}
/// Extracts text with both logical bundle and archive-entry context.
pub fn extract_bundle_with_context(
&self,
bundle: &ParsedAssetBundle,
bundle_path: Option<&str>,
archive_entry: Option<&str>,
) -> TextUnitExtractionReport {
let mut report = TextUnitExtractionReport {
units: Vec::new(),
errors: Vec::new(),
skipped_binary_text_assets: 0,
};
for asset in &bundle.text_assets {
if let Some((format, text)) = decode_text_payload(&asset.bytes) {
let mut context = BTreeMap::new();
context.insert("format".to_string(), format.to_string());
context.insert("asset_name".to_string(), asset.name.clone());
context.insert("source_kind".to_string(), "TextAsset".to_string());
report.units.push(TextUnit {
source_text: text,
bundle_path: bundle_path.map(ToOwned::to_owned),
archive_entry: archive_entry.map(ToOwned::to_owned),
serialized_file: asset.source_path.clone(),
path_id: Some(asset.path_id),
class_id: Some(49),
field_path: Some("TextAsset".to_string()),
field_offset: None,
field_byte_size: None,
version: bundle.unity_version.clone(),
context,
});
} else {
report.skipped_binary_text_assets += 1;
}
}
for serialized_file in &bundle.serialized_files {
for object in &serialized_file.objects {
if object.class_id == 49 || !serialized_file.object_has_type_tree(object) {
continue;
}
let fields = match serialized_file.fields_for_object_entry(object) {
Ok(fields) => fields,
Err(error) => {
report.errors.push(TextUnitExtractionError {
serialized_file: serialized_file.source_path.clone(),
path_id: Some(object.path_id),
class_id: Some(object.class_id),
field_path: None,
offset: None,
error: error.to_string(),
});
continue;
}
};
let context = FieldTextContext {
serialized_file_path: serialized_file.source_path.as_deref(),
path_id: object.path_id,
class_id: object.class_id,
version: &bundle.unity_version,
bundle_path,
archive_entry,
managed_reference: None,
};
for field in fields {
collect_field_text(&mut report.units, &context, &field);
}
}
}
report
}
}
/// Serializes text units as one stable JSON object per line.
pub fn text_units_to_jsonl(units: &[TextUnit]) -> Result<String, serde_json::Error> {
let mut output = String::new();
for unit in units {
output.push_str(&serde_json::to_string(unit)?);
output.push('\n');
}
Ok(output)
}
#[derive(Clone)]
struct FieldTextContext<'a> {
serialized_file_path: Option<&'a str>,
path_id: i64,
class_id: i32,
version: &'a str,
bundle_path: Option<&'a str>,
archive_entry: Option<&'a str>,
managed_reference: Option<UnityManagedReferenceMetadata>,
}
impl<'a> FieldTextContext<'a> {
fn with_managed_reference(&self, metadata: Option<&UnityManagedReferenceMetadata>) -> Self {
Self {
serialized_file_path: self.serialized_file_path,
path_id: self.path_id,
class_id: self.class_id,
version: self.version,
bundle_path: self.bundle_path,
archive_entry: self.archive_entry,
managed_reference: metadata.cloned().or_else(|| self.managed_reference.clone()),
}
}
}
fn collect_field_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
field: &'a UnitySerializedField,
) {
match &field.value {
UnitySerializedValue::String(text) if !text.is_empty() => {
let mut unit_context = BTreeMap::new();
unit_context.insert("format".to_string(), "plain".to_string());
unit_context.insert(
"source_kind".to_string(),
if context.managed_reference.is_some() {
"ManagedReferenceField"
} else {
"TypeTreeField"
}
.to_string(),
);
unit_context.insert("type_name".to_string(), field.type_name.clone());
if let Some(metadata) = &context.managed_reference {
insert_managed_reference_context(&mut unit_context, metadata);
}
units.push(TextUnit {
source_text: text.clone(),
bundle_path: context.bundle_path.map(ToOwned::to_owned),
archive_entry: context.archive_entry.map(ToOwned::to_owned),
serialized_file: context.serialized_file_path.map(ToOwned::to_owned),
path_id: Some(context.path_id),
class_id: Some(context.class_id),
field_path: Some(field.path.clone()),
field_offset: Some(field.offset),
field_byte_size: Some(field.byte_size),
version: context.version.to_string(),
context: unit_context,
});
}
UnitySerializedValue::Object(fields) => {
for field in fields {
collect_field_text(units, context, field);
}
}
UnitySerializedValue::ManagedReference {
metadata, fields, ..
} => {
let managed_context = context.with_managed_reference(metadata.as_ref());
for field in fields {
collect_managed_reference_child_text(units, &managed_context, field);
}
}
UnitySerializedValue::ManagedReferenceRegistry { references, fields } => {
if references.is_empty() {
let fallback_metadata = managed_reference_metadata_from_fields(fields);
for field in fields {
let sibling_metadata =
managed_reference_metadata_from_sibling_fields(fields, field);
let managed_context = context.with_managed_reference(
sibling_metadata.as_ref().or(fallback_metadata.as_ref()),
);
collect_managed_reference_child_text(units, &managed_context, field);
}
} else {
for reference in references {
collect_managed_reference_record_text(units, context, reference);
}
}
}
UnitySerializedValue::Array(values) | UnitySerializedValue::Map(values) => {
for field in values {
collect_field_text(units, context, field);
}
}
_ => {}
}
}
fn collect_managed_reference_record_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
reference: &'a UnityManagedReferenceRecord,
) {
let managed_context = context.with_managed_reference(Some(&reference.metadata));
for field in &reference.fields {
collect_field_text(units, &managed_context, field);
}
}
fn collect_managed_reference_child_text<'a>(
units: &mut Vec<TextUnit>,
context: &FieldTextContext<'a>,
field: &'a UnitySerializedField,
) {
let key = normalized_text_metadata_key(&field.name);
if is_managed_reference_metadata_text_key(&key) {
return;
}
if is_managed_reference_payload_text_key(&key) {
if let Some(children) = serialized_field_children(field) {
for child in children {
collect_field_text(units, context, child);
}
} else {
collect_field_text(units, context, field);
}
return;
}
if let Some(children) = serialized_field_children(field) {
if let Some(metadata) = managed_reference_metadata_from_fields(children) {
let managed_context = context.with_managed_reference(Some(&metadata));
for child in children {
collect_managed_reference_child_text(units, &managed_context, child);
}
return;
}
}
collect_field_text(units, context, field);
}
fn managed_reference_metadata_from_sibling_fields(
fields: &[UnitySerializedField],
field: &UnitySerializedField,
) -> Option<UnityManagedReferenceMetadata> {
let record_prefix = managed_reference_record_path_prefix(&field.path)?;
let grouped_fields = fields
.iter()
.filter(|candidate| {
candidate.path == record_prefix
|| candidate
.path
.strip_prefix(record_prefix)
.is_some_and(|suffix| suffix.starts_with('.'))
})
.cloned()
.collect::<Vec<_>>();
managed_reference_metadata_from_fields(&grouped_fields)
}
fn managed_reference_record_path_prefix(path: &str) -> Option<&str> {
if let Some(index_end) = path.rfind(']') {
return Some(&path[..=index_end]);
}
path.rsplit_once('.')
.map(|(parent, _)| parent)
.filter(|parent| !parent.is_empty())
}
fn insert_managed_reference_context(
context: &mut BTreeMap<String, String>,
metadata: &UnityManagedReferenceMetadata,
) {
if let Some(reference_id) = metadata.reference_id {
context.insert("managed_reference_id".to_string(), reference_id.to_string());
}
if let Some(value) = &metadata.full_type_name {
context.insert(
"managed_reference_full_type_name".to_string(),
value.clone(),
);
}
if let Some(value) = &metadata.type_name {
context.insert("managed_reference_type".to_string(), value.clone());
}
if let Some(value) = &metadata.namespace {
context.insert("managed_reference_namespace".to_string(), value.clone());
}
if let Some(value) = &metadata.assembly_name {
context.insert("managed_reference_assembly".to_string(), value.clone());
}
}
fn serialized_field_children(field: &UnitySerializedField) -> Option<&[UnitySerializedField]> {
match &field.value {
UnitySerializedValue::Object(fields)
| UnitySerializedValue::Array(fields)
| UnitySerializedValue::Map(fields)
| UnitySerializedValue::ManagedReference { fields, .. }
| UnitySerializedValue::ManagedReferenceRegistry { fields, .. } => Some(fields),
_ => None,
}
}
fn normalized_text_metadata_key(name: &str) -> String {
name.strip_prefix("m_")
.unwrap_or(name)
.chars()
.filter(|character| character.is_ascii_alphanumeric())
.flat_map(char::to_lowercase)
.collect()
}
fn is_managed_reference_metadata_text_key(key: &str) -> bool {
matches!(
key,
"rid"
| "id"
| "identifier"
| "refid"
| "referenceid"
| "managedreferenceid"
| "managedreferenceids"
| "managedreferencesid"
| "managedreferencesids"
| "serializedreferenceid"
| "serializedreferenceids"
| "refids"
| "type"
| "typeid"
| "typeinfo"
| "typename"
| "fullname"
| "fulltypename"
| "class"
| "classname"
| "managedreferenceclassname"
| "serializedreferenceclassname"
| "klass"
| "managedtype"
| "managedreferencetype"
| "managedreferencefullname"
| "managedreferencefulltypename"
| "serializedreferencetype"
| "serializedreferencefullname"
| "serializedreferencefulltypename"
| "assemblyqualifiedname"
| "ns"
| "namespace"
| "namespacename"
| "managedreferencenamespace"
| "managedreferencenamespacename"
| "serializedreferencenamespace"
| "serializedreferencenamespacename"
| "asm"
| "asmname"
| "assembly"
| "assemblyname"
| "managedreferenceassembly"
| "managedreferenceassemblyname"
| "serializedreferenceassembly"
| "serializedreferenceassemblyname"
)
}
fn is_managed_reference_payload_text_key(key: &str) -> bool {
matches!(
key,
"data"
| "payload"
| "value"
| "object"
| "instance"
| "managedreferencepayload"
| "referencepayload"
| "serializedreferencepayload"
| "managedreferencevalue"
| "referencevalue"
| "serializedreferencevalue"
| "managedreferenceobject"
| "referenceobject"
| "serializedreferenceobject"
| "managedreferencedata"
| "referencedata"
| "serializeddata"
| "serializedreferencedata"
)
}
fn decode_text_payload(bytes: &[u8]) -> Option<(&'static str, String)> {
let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
if bytes.contains(&0) {
return None;
}
let text = std::str::from_utf8(bytes).ok()?.to_string();
if text.trim().is_empty()
|| text
.chars()
.any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t'))
{
return None;
}
let trimmed = text.trim();
if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
return Some(("json", text));
}
if text.lines().any(|line| line.contains('\t')) {
return Some(("tsv", text));
}
if text.lines().count() > 1 && text.lines().any(|line| line.contains(',')) {
return Some(("csv", text));
}
Some(("plain", text))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::serialized::UnitySerializedTextAsset;
use crate::types::{
ParsedAssetBundle, UnityFsBlockInfo, UnityFsDirectoryInfo, UnityFsHeader,
UnitySerializedParseError,
};
#[test]
fn extracts_textasset_and_writes_jsonl() {
let bundle = ParsedAssetBundle {
unity_version: "2021.3.56f2".to_string(),
assets: vec!["CAB-test".to_string()],
raw_data: Vec::new(),
unityfs_header: Some(UnityFsHeader {
format_version: 8,
target_version: "5.x.x".to_string(),
unity_version: "2021.3.56f2".to_string(),
total_size: 0,
compressed_blocks_info_size: 0,
uncompressed_blocks_info_size: 0,
flags: 0,
}),
blocks: vec![UnityFsBlockInfo {
uncompressed_size: 0,
compressed_size: 0,
flags: 0,
compression: crate::types::UnityFsCompression::None,
}],
directories: vec![UnityFsDirectoryInfo {
offset: 0,
size: 0,
flags: 0,
path: "CAB-test".to_string(),
}],
files: Vec::new(),
serialized_files: Vec::new(),
text_assets: vec![UnitySerializedTextAsset {
source_path: Some("CAB-test".to_string()),
path_id: 1,
name: "dialogue.json".to_string(),
bytes: br#"{"text":"hello"}"#.to_vec(),
}],
serialized_parse_errors: Vec::<UnitySerializedParseError>::new(),
};
let report = TextUnitExtractor::new().extract_bundle_with_context(
&bundle,
Some("dialogue.bundle"),
Some("assets/dialogue.bundle"),
);
assert_eq!(report.units.len(), 1);
assert_eq!(
report.units[0].archive_entry.as_deref(),
Some("assets/dialogue.bundle")
);
assert_eq!(
report.units[0].context.get("format"),
Some(&"json".to_string())
);
assert_eq!(
text_units_to_jsonl(&report.units).unwrap(),
format!("{}\n", serde_json::to_string(&report.units[0]).unwrap())
);
}
#[test]
fn extracts_managed_reference_payload_without_metadata_strings() {
let payload_field = string_field(
"m_ManagedReferences.references[0].data.message",
"message",
"こんにちは",
);
let metadata_field = string_field(
"m_ManagedReferences.references[0].managedReferenceFullTypeName",
"managedReferenceFullTypeName",
"Game BA.Text.ScenarioLine",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferenceRegistry".to_string(),
offset: 0,
byte_size: 64,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: vec![UnityManagedReferenceRecord {
metadata: UnityManagedReferenceMetadata {
reference_id: Some(42),
full_type_name: Some("Game BA.Text.ScenarioLine".to_string()),
type_name: Some("ScenarioLine".to_string()),
namespace: Some("BA.Text".to_string()),
assembly_name: Some("Game".to_string()),
},
fields: vec![payload_field.clone()],
}],
fields: vec![metadata_field],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.references[0].data.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_id"),
Some(&"42".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_full_type_name"),
Some(&"Game BA.Text.ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_payload_alias_without_metadata_strings() {
let payload_field = string_field(
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
"message",
"こんにちは",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 96,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
string_field(
"m_ManagedReferences.RefIds[0].serializedReferenceFullTypeName",
"serializedReferenceFullTypeName",
"Game BA.Text.ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeInfo",
"typeInfo",
"Game BA.Text.ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.className",
"className",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.namespaceName",
"namespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].typeID.asmName",
"asmName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
name: "managedReferenceData".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![payload_field]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_full_type_name"),
Some(&"Game BA.Text.ScenarioLine".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_payload_family_alias_with_context() {
let payload_field = string_field(
"m_ManagedReferences.RefIds[0].serializedReferencePayload.message",
"message",
"こんにちは",
);
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 96,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].serializedReferencePayload"
.to_string(),
name: "serializedReferencePayload".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![payload_field]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 1);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].serializedReferencePayload.message")
);
assert_eq!(
units[0].context.get("source_kind"),
Some(&"ManagedReferenceField".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_namespace"),
Some(&"BA.Text".to_string())
);
assert_eq!(
units[0].context.get("managed_reference_assembly"),
Some(&"Game".to_string())
);
}
#[test]
fn extracts_fallback_managed_reference_sibling_records_with_separate_context() {
let registry_field = UnitySerializedField {
path: "m_ManagedReferences".to_string(),
name: "m_ManagedReferences".to_string(),
type_name: "ManagedReferencesRegistry".to_string(),
offset: 0,
byte_size: 160,
type_tree_node_index: None,
value: UnitySerializedValue::ManagedReferenceRegistry {
references: Vec::new(),
fields: vec![
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
"managedReferenceClassName",
"ScenarioLine",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
name: "managedReferenceData".to_string(),
type_name: "managedReference".to_string(),
offset: 64,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![string_field(
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
"message",
"こんにちは",
)]),
},
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceClassName",
"managedReferenceClassName",
"ChoiceLine",
),
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceNamespaceName",
"managedReferenceNamespaceName",
"BA.Text",
),
string_field(
"m_ManagedReferences.RefIds[1].managedReferenceAssemblyName",
"managedReferenceAssemblyName",
"Game",
),
UnitySerializedField {
path: "m_ManagedReferences.RefIds[1].referencePayload".to_string(),
name: "referencePayload".to_string(),
type_name: "managedReference".to_string(),
offset: 120,
byte_size: 24,
type_tree_node_index: None,
value: UnitySerializedValue::Object(vec![string_field(
"m_ManagedReferences.RefIds[1].referencePayload.message",
"message",
"選択肢",
)]),
},
],
},
};
let context = FieldTextContext {
serialized_file_path: Some("CAB-test"),
path_id: 7,
class_id: 114,
version: "2021.3.56f2",
bundle_path: Some("scenario.bundle"),
archive_entry: None,
managed_reference: None,
};
let mut units = Vec::new();
collect_field_text(&mut units, &context, &registry_field);
assert_eq!(units.len(), 2);
assert_eq!(
units[0].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
);
assert_eq!(units[0].source_text, "こんにちは");
assert_eq!(
units[0].context.get("managed_reference_type"),
Some(&"ScenarioLine".to_string())
);
assert_eq!(
units[1].field_path.as_deref(),
Some("m_ManagedReferences.RefIds[1].referencePayload.message")
);
assert_eq!(units[1].source_text, "選択肢");
assert_eq!(
units[1].context.get("managed_reference_type"),
Some(&"ChoiceLine".to_string())
);
}
fn string_field(path: &str, name: &str, value: &str) -> UnitySerializedField {
UnitySerializedField {
path: path.to_string(),
name: name.to_string(),
type_name: "string".to_string(),
offset: 0,
byte_size: value.len() + 4,
type_tree_node_index: None,
value: UnitySerializedValue::String(value.to_string()),
}
}
}
+1
View File
@@ -14,6 +14,7 @@ bat-core = { path = "../core" }
bat-adapters = { path = "../adapters" }
bat-assetbundle = { path = "../crates/bat-assetbundle" }
bat-cas-engine = { path = "../crates/bat-cas-engine" }
bat-patch = { path = "../crates/bat-patch" }
anyhow.workspace = true
thiserror.workspace = true
serde.workspace = true
File diff suppressed because it is too large Load Diff
+27 -2
View File
@@ -2,9 +2,10 @@
use bat_adapters::manifest::GenericManifest;
use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry};
use bat_core::domain::{Resource, ResourceEntry, ResourceType};
use bat_assetbundle::TextUnitExtractor;
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// 待导入的 AssetBundle 数据。
@@ -97,6 +98,12 @@ pub struct UnityFsImportSummary {
pub text_assets: Vec<String>,
/// 非致命 serialized-file 解析诊断数量。
pub serialized_parse_error_count: usize,
/// 从 TextAsset 和 TypeTree 字段提取出的 TextUnit 数量。
pub text_unit_count: usize,
/// TextUnit 格式标签。
pub text_unit_formats: Vec<String>,
/// TextUnit 提取阶段的非致命诊断数量。
pub text_unit_error_count: usize,
}
/// Manifest 导入报告。
@@ -208,6 +215,7 @@ impl<'a> ResourceImportService<'a> {
id: resource_id_for_path(&entry.path),
local_path: PathBuf::from(&entry.path),
entry: stored_entry,
metadata: ResourceMetadata::default(),
};
let id = self.resources.add(resource).await?;
added_resources.push(id.clone());
@@ -266,6 +274,14 @@ impl<'a> ResourceImportService<'a> {
manifest_path, error
))
})?;
let text_units = TextUnitExtractor::new().extract_bundle(&parsed, Some(manifest_path));
let text_unit_formats = text_units
.units
.iter()
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
Ok(UnityFsImportSummary {
unity_version: parsed.unity_version,
@@ -285,6 +301,9 @@ impl<'a> ResourceImportService<'a> {
.map(|asset| asset.name)
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
text_unit_error_count: text_units.errors.len(),
})
}
}
@@ -715,6 +734,9 @@ mod tests {
assert_eq!(unityfs.text_asset_count, 0);
assert!(unityfs.text_assets.is_empty());
assert_eq!(unityfs.serialized_parse_error_count, 0);
assert_eq!(unityfs.text_unit_count, 0);
assert!(unityfs.text_unit_formats.is_empty());
assert_eq!(unityfs.text_unit_error_count, 0);
assert_eq!(
report.imported[1].category,
ResourceImportCategory::TextAsset
@@ -799,6 +821,9 @@ mod tests {
assert_eq!(unityfs.text_asset_count, 1);
assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]);
assert_eq!(unityfs.serialized_parse_error_count, 0);
assert_eq!(unityfs.text_unit_count, 1);
assert_eq!(unityfs.text_unit_formats, vec!["plain".to_string()]);
assert_eq!(unityfs.text_unit_error_count, 0);
}
#[tokio::test]
+47 -3
View File
@@ -13,13 +13,18 @@
pub mod cas;
mod curl_transfer;
pub mod import;
pub mod localized_patch;
pub mod official_changes;
pub mod official_download;
pub mod official_game_main_config;
pub mod official_launcher;
pub mod official_parse;
pub mod official_pull;
pub mod official_repository;
pub mod official_sync;
pub mod official_textunit_queue;
pub mod official_update;
pub mod patch_ops;
pub mod path_security;
pub mod resources;
mod zip_validation;
@@ -32,6 +37,24 @@ pub use import::{
BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport,
ResourceImportService,
};
pub use localized_patch::{
read_localized_patch_manifest_at, read_localized_version_state, LocalizedPatchConfig,
LocalizedPatchFile, LocalizedPatchIntegrity, LocalizedPatchManifest, LocalizedPatchOperation,
LocalizedPatchReport, LocalizedPatchRollbackInfo, LocalizedPatchService,
LocalizedTextAssetPatch, LocalizedVersionState, LOCALIZED_CURRENT_LINK,
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_PATCH_MANIFEST_VERSION, LOCALIZED_STAGING_DIR,
LOCALIZED_VERSIONS_DIR, LOCALIZED_VERSION_STATE_FILE,
};
pub use official_changes::{
read_resource_change_set_at, write_crowdin_translation_handoff_at,
write_official_resource_change_handoff, write_resource_change_set_at,
CrowdinTranslationHandoff, OfficialResourceChange, OfficialResourceChangeHandoffReport,
OfficialResourceChangeKind, OfficialResourceChangeSet, OfficialResourceChangeSummary,
OfficialResourceDescriptor, TranslationHandoffProvider, TranslationHandoffResource,
TranslationHandoffStatus, CROWDIN_TRANSLATION_HANDOFF_FILE,
CROWDIN_TRANSLATION_HANDOFF_VERSION, OFFICIAL_RESOURCE_CHANGES_FILE,
OFFICIAL_RESOURCE_CHANGES_VERSION,
};
pub use official_download::{
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
@@ -47,20 +70,35 @@ pub use official_launcher::{
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
};
pub use official_parse::{
read_parse_cache_at, write_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry,
OfficialParseCacheService, OfficialParseConfig, OfficialParseReport,
query_textunit_index_errors, query_textunit_index_units, read_parse_cache_at,
read_textunit_index_at, write_parse_cache_at, write_textunit_index_at, OfficialParseCache,
OfficialParseCacheEntry, OfficialParseCacheService, OfficialParseConfig, OfficialParseReport,
OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus,
OfficialParseSummary, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION,
OfficialParseSummary, OfficialTextUnitIndex, OfficialTextUnitIndexError,
OfficialTextUnitIndexSummary, OfficialTextUnitIndexUnit, OfficialTextUnitQuery,
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION, OFFICIAL_TEXTUNIT_INDEX_FILE,
OFFICIAL_TEXTUNIT_INDEX_VERSION,
};
pub use official_pull::{
build_official_pull_plan, build_official_pull_plan_for_platform_inventory,
build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory,
OfficialResourcePullPlan,
};
pub use official_repository::{
OfficialReleaseImportConfig, OfficialReleaseImportReport, OfficialReleaseImportService,
};
pub use official_sync::{
build_official_sync_plan, changed_endpoint_urls, classify_sync_decision,
default_official_platforms, OfficialSyncDecision, OfficialSyncPlan,
};
pub use official_textunit_queue::{
read_textunit_task_queue_at, write_crowdin_textunit_queue_at, write_official_textunit_queues,
write_textunit_task_queue_at, CrowdinTextUnitQueue, CrowdinTextUnitQueueItem,
OfficialTextUnitQueueReport, OfficialTextUnitTask, OfficialTextUnitTaskQueue,
OfficialTextUnitTaskStatus, OfficialTextUnitTaskSummary, CROWDIN_TEXTUNIT_QUEUE_FILE,
CROWDIN_TEXTUNIT_QUEUE_VERSION, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE,
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
};
pub use official_update::{
cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging,
read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot,
@@ -71,6 +109,12 @@ pub use official_update::{
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap,
};
pub use patch_ops::{
apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file,
apply_unityfs_text_asset_patch_file, PatchApplyKind, PatchApplyParams, PatchApplyReport,
UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams,
UnityFsTextAssetPatchParams,
};
pub use path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
open_append_file, read_file_no_symlink, set_file_mode, validate_output_root,
+857
View File
@@ -0,0 +1,857 @@
//! Localized release publishing for verified TextAsset patches.
use bat_assetbundle::{patch_unityfs_text_asset, TextAssetPatch};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
read_file_no_symlink, write_file_atomic, STATE_FILE_MODE,
};
/// Atomic current pointer under the localized output root.
pub const LOCALIZED_CURRENT_LINK: &str = "current";
/// Staging directory under the localized output root.
pub const LOCALIZED_STAGING_DIR: &str = ".staging";
/// Version directory under the localized output root.
pub const LOCALIZED_VERSIONS_DIR: &str = "versions";
/// Persisted localized release state file name.
pub const LOCALIZED_VERSION_STATE_FILE: &str = "localized-version-state.json";
/// Per-release patch manifest file name.
pub const LOCALIZED_PATCH_MANIFEST_FILE: &str = "localized-patch-manifest.json";
/// Current localized patch manifest schema version.
pub const LOCALIZED_PATCH_MANIFEST_VERSION: u32 = 1;
/// One patch operation against a bundle in an official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedTextAssetPatch {
/// Relative path of the UnityFS bundle under the official release.
pub bundle_path: String,
/// TextAsset replacement inside the bundle.
pub text_asset: TextAssetPatch,
}
/// Configuration for one localized release publication.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedPatchConfig {
/// Immutable, verified official release root.
pub official_release_root: PathBuf,
/// Separate localized publication root.
pub localized_output_root: PathBuf,
/// Version identifier shared with the official release.
pub release_id: String,
/// Patch operations to apply.
pub patches: Vec<LocalizedTextAssetPatch>,
}
impl LocalizedPatchConfig {
/// Creates a localized patch configuration.
pub fn new(
official_release_root: impl Into<PathBuf>,
localized_output_root: impl Into<PathBuf>,
release_id: impl Into<String>,
patches: Vec<LocalizedTextAssetPatch>,
) -> Self {
Self {
official_release_root: official_release_root.into(),
localized_output_root: localized_output_root.into(),
release_id: release_id.into(),
patches,
}
}
}
/// Persisted localized release state.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedVersionState {
/// State schema version.
pub state_version: u32,
/// Official release ID used as the patch source.
pub official_release_id: String,
/// Published localized release ID.
pub current_release_id: Option<String>,
/// Stable status label.
pub status: String,
/// Last update time.
pub updated_unix_seconds: u64,
}
/// One changed file in a localized patch manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchFile {
/// Relative bundle path.
pub path: String,
/// BLAKE3 before applying the patch.
pub original_blake3: String,
/// BLAKE3 after applying the patch.
pub localized_blake3: String,
/// Original file size in bytes.
#[serde(default)]
pub original_bytes: u64,
/// Localized file size in bytes.
#[serde(default)]
pub localized_bytes: u64,
/// Localized minus original byte size.
#[serde(default)]
pub byte_delta: i64,
/// TextAsset operations applied to this file.
#[serde(default)]
pub text_asset_operations: Vec<LocalizedPatchOperation>,
}
/// One TextAsset patch operation recorded in the localized patch manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchOperation {
/// UnityFS directory path of the serialized file.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Expected TextAsset name, when provided.
pub expected_name: Option<String>,
/// Replacement payload size.
pub replacement_bytes: u64,
/// BLAKE3 of the replacement payload.
pub replacement_blake3: String,
}
/// Rollback information recorded for a localized publication.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchRollbackInfo {
/// Previous `current` symlink target before this publication.
pub previous_current_target: Option<PathBuf>,
/// Version directory that should be removed when rolling this publication back.
pub remove_version_path: PathBuf,
}
/// Integrity summary for a published localized release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchIntegrity {
/// Number of changed files verified against the manifest.
pub verified_changed_file_count: usize,
/// Number of TextAsset operations recorded in the manifest.
pub verified_text_asset_operation_count: usize,
/// Whether `current` points at this localized release.
pub current_points_to_release: bool,
}
/// Persisted manifest for one localized release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LocalizedPatchManifest {
/// Manifest schema version.
#[serde(default = "default_patch_manifest_version")]
pub manifest_version: u32,
/// Official release ID used as the patch source.
pub official_release_id: String,
/// Published localized release ID.
pub localized_release_id: String,
/// Manifest generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Changed file count.
pub file_count: usize,
/// TextAsset operation count.
pub text_asset_operation_count: usize,
/// Changed files and their before/after hashes.
pub files: Vec<LocalizedPatchFile>,
/// Rollback information for this release.
pub rollback: LocalizedPatchRollbackInfo,
}
impl LocalizedPatchManifest {
/// Converts the localized TextAsset manifest to the generic patch manifest model.
pub fn to_patch_manifest(&self) -> bat_patch::PatchManifest {
bat_patch::PatchManifest {
version: bat_patch::PATCH_MANIFEST_VERSION,
patch_id: self.localized_release_id.clone(),
source_version: self.official_release_id.clone(),
target_version: self.localized_release_id.clone(),
files: self
.files
.iter()
.map(|file| bat_patch::PatchManifestFile {
path: PathBuf::from(&file.path),
patch_kind: bat_patch::PatchKind::UnityFsTextAsset,
source_blake3: file.original_blake3.clone(),
target_blake3: file.localized_blake3.clone(),
source_size: file.original_bytes,
target_size: file.localized_bytes,
})
.collect(),
rollback: bat_patch::PatchRollback {
previous_current_target: self.rollback.previous_current_target.clone(),
remove_target_path: Some(self.rollback.remove_version_path.clone()),
},
}
}
}
/// Result of a successful localized release publication.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalizedPatchReport {
/// Published version directory.
pub version_path: PathBuf,
/// Atomic current pointer.
pub current_path: PathBuf,
/// Version state path.
pub state_path: PathBuf,
/// Patch manifest path.
pub patch_manifest_path: PathBuf,
/// Changed files.
pub files: Vec<LocalizedPatchFile>,
/// Persisted patch manifest.
pub manifest: LocalizedPatchManifest,
/// Integrity check performed after publication.
pub integrity: LocalizedPatchIntegrity,
}
/// Applies TextAsset patches and atomically publishes a localized release.
#[derive(Debug, Default, Clone, Copy)]
pub struct LocalizedPatchService;
impl LocalizedPatchService {
/// Creates the publisher.
pub fn new() -> Self {
Self
}
/// Copies the official release, applies patches in staging and publishes it.
pub fn publish(&self, config: &LocalizedPatchConfig) -> anyhow::Result<LocalizedPatchReport> {
let staging = config
.localized_output_root
.join(LOCALIZED_STAGING_DIR)
.join(&config.release_id);
let version_path = config
.localized_output_root
.join(LOCALIZED_VERSIONS_DIR)
.join(&config.release_id);
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
let previous_current_target = current_symlink_target(&current_path).ok().flatten();
let version_existed_before = version_path.exists();
match self.publish_inner(config, previous_current_target.clone()) {
Ok(report) => Ok(report),
Err(error) => {
if let Err(rollback_error) = rollback_failed_publish(
&config.localized_output_root,
&staging,
&version_path,
!version_existed_before,
&current_path,
previous_current_target.as_ref(),
) {
return Err(anyhow::anyhow!(
"{error}; rollback failed: {rollback_error}"
));
}
Err(error)
}
}
}
fn publish_inner(
&self,
config: &LocalizedPatchConfig,
previous_current_target: Option<PathBuf>,
) -> anyhow::Result<LocalizedPatchReport> {
validate_config(config).map_err(anyhow::Error::msg)?;
let staging = config
.localized_output_root
.join(LOCALIZED_STAGING_DIR)
.join(&config.release_id);
let version_path = config
.localized_output_root
.join(LOCALIZED_VERSIONS_DIR)
.join(&config.release_id);
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
let state_path = config
.localized_output_root
.join(LOCALIZED_VERSION_STATE_FILE);
let patch_manifest_path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
if version_path.exists() {
return Err(anyhow::anyhow!(
"localized release already exists: {}",
version_path.display()
));
}
remove_owned_staging(&staging)?;
fs::create_dir_all(&staging)?;
copy_tree(&config.official_release_root, &staging)?;
let mut changed_files = Vec::with_capacity(config.patches.len());
for operation in &config.patches {
let target = staging.join(Path::new(&operation.bundle_path));
ensure_path_within_root(&staging, &target).map_err(anyhow::Error::msg)?;
ensure_safe_file_target(&staging, &target, "汉化 patch 输入")
.map_err(anyhow::Error::msg)?;
let original = fs::read(&target)?;
let patched = patch_unityfs_text_asset(&original, &operation.text_asset)
.map_err(|error| anyhow::anyhow!("{}: {error}", operation.bundle_path))?;
if original == patched {
return Err(anyhow::anyhow!(
"patch produced no change: {}",
operation.bundle_path
));
}
write_file_atomic(&target, &patched, STATE_FILE_MODE, "汉化 patch 输出")
.map_err(anyhow::Error::msg)?;
let original_blake3 = blake3::hash(&original).to_hex().to_string();
let localized_blake3 = blake3::hash(&patched).to_hex().to_string();
changed_files.push(LocalizedPatchFile {
path: operation.bundle_path.clone(),
original_blake3,
localized_blake3,
original_bytes: original.len() as u64,
localized_bytes: patched.len() as u64,
byte_delta: patched.len() as i64 - original.len() as i64,
text_asset_operations: vec![LocalizedPatchOperation::from_text_asset_patch(
&operation.text_asset,
)],
});
}
let manifest = LocalizedPatchManifest {
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
official_release_id: config.release_id.clone(),
localized_release_id: config.release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
file_count: changed_files.len(),
text_asset_operation_count: changed_files
.iter()
.map(|file| file.text_asset_operations.len())
.sum(),
files: changed_files.clone(),
rollback: LocalizedPatchRollbackInfo {
previous_current_target: previous_current_target.clone(),
remove_version_path: version_path.clone(),
},
};
write_file_atomic(
&staging.join(LOCALIZED_PATCH_MANIFEST_FILE),
&serde_json::to_vec_pretty(&manifest)?,
STATE_FILE_MODE,
"汉化 patch manifest",
)
.map_err(anyhow::Error::msg)?;
verify_patch_manifest_files(&config.official_release_root, &staging, &manifest)?;
fs::create_dir_all(config.localized_output_root.join(LOCALIZED_VERSIONS_DIR))?;
fs::rename(&staging, &version_path)?;
switch_current_symlink(
&config.localized_output_root,
&current_path,
&config.release_id,
)?;
let state = LocalizedVersionState {
state_version: 1,
official_release_id: config.release_id.clone(),
current_release_id: Some(config.release_id.clone()),
status: "localized".to_string(),
updated_unix_seconds: unix_seconds_now(),
};
write_file_atomic(
&state_path,
&serde_json::to_vec_pretty(&state)?,
STATE_FILE_MODE,
"汉化版本状态",
)
.map_err(anyhow::Error::msg)?;
let integrity = verify_published_localized_release(
&config.official_release_root,
&version_path,
&current_path,
)?;
Ok(LocalizedPatchReport {
version_path,
current_path,
state_path,
patch_manifest_path,
files: changed_files,
manifest,
integrity,
})
}
}
/// Reads the localized release state without following a symlink at the file
/// path. A missing state file means no localized release has been published.
pub fn read_localized_version_state(
localized_output_root: &Path,
) -> anyhow::Result<Option<LocalizedVersionState>> {
let path = localized_output_root.join(LOCALIZED_VERSION_STATE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "汉化版本状态").map_err(anyhow::Error::msg)?
else {
return Ok(None);
};
let state: LocalizedVersionState = serde_json::from_slice(&bytes)?;
if state.state_version != 1 {
return Err(anyhow::anyhow!(
"不支持的汉化版本状态 schema:{},当前版本=1",
state.state_version
));
}
Ok(Some(state))
}
/// Reads a localized patch manifest from a published version directory.
pub fn read_localized_patch_manifest_at(
version_path: &Path,
) -> anyhow::Result<Option<LocalizedPatchManifest>> {
let path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
let Some(bytes) =
read_file_no_symlink(&path, "汉化 patch manifest").map_err(anyhow::Error::msg)?
else {
return Ok(None);
};
let manifest: LocalizedPatchManifest = serde_json::from_slice(&bytes)?;
if manifest.manifest_version != LOCALIZED_PATCH_MANIFEST_VERSION {
return Err(anyhow::anyhow!(
"不支持的汉化 patch manifest schema{},当前版本={}",
manifest.manifest_version,
LOCALIZED_PATCH_MANIFEST_VERSION
));
}
Ok(Some(manifest))
}
impl LocalizedPatchOperation {
fn from_text_asset_patch(patch: &TextAssetPatch) -> Self {
Self {
serialized_file_path: patch.serialized_file_path.clone(),
path_id: patch.path_id,
expected_name: patch.expected_name.clone(),
replacement_bytes: patch.replacement.len() as u64,
replacement_blake3: blake3::hash(&patch.replacement).to_hex().to_string(),
}
}
}
fn verify_published_localized_release(
official_release_root: &Path,
version_path: &Path,
current_path: &Path,
) -> anyhow::Result<LocalizedPatchIntegrity> {
let manifest = read_localized_patch_manifest_at(version_path)?.ok_or_else(|| {
anyhow::anyhow!(
"缺少汉化 patch manifest{}",
version_path.join(LOCALIZED_PATCH_MANIFEST_FILE).display()
)
})?;
let mut integrity =
verify_patch_manifest_files(official_release_root, version_path, &manifest)?;
integrity.current_points_to_release = current_points_to_version(current_path, version_path)?;
if !integrity.current_points_to_release {
return Err(anyhow::anyhow!(
"汉化 current 未指向发布版本:current={} version={}",
current_path.display(),
version_path.display()
));
}
Ok(integrity)
}
fn verify_patch_manifest_files(
official_release_root: &Path,
localized_release_root: &Path,
manifest: &LocalizedPatchManifest,
) -> anyhow::Result<LocalizedPatchIntegrity> {
let mut operation_count = 0usize;
for file in &manifest.files {
let relative = Path::new(&file.path);
let official_path = official_release_root.join(relative);
let localized_path = localized_release_root.join(relative);
ensure_path_within_root(official_release_root, &official_path)
.map_err(anyhow::Error::msg)?;
ensure_path_within_root(localized_release_root, &localized_path)
.map_err(anyhow::Error::msg)?;
ensure_safe_file_target(official_release_root, &official_path, "官方 patch 原文件")
.map_err(anyhow::Error::msg)?;
ensure_safe_file_target(localized_release_root, &localized_path, "汉化 patch 产物")
.map_err(anyhow::Error::msg)?;
let original = fs::read(&official_path)?;
let localized = fs::read(&localized_path)?;
let original_hash = blake3::hash(&original).to_hex().to_string();
let localized_hash = blake3::hash(&localized).to_hex().to_string();
if original_hash != file.original_blake3 || original.len() as u64 != file.original_bytes {
return Err(anyhow::anyhow!(
"汉化 manifest 原文件校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
file.path,
file.original_blake3,
file.original_bytes,
original_hash,
original.len()
));
}
if localized_hash != file.localized_blake3 || localized.len() as u64 != file.localized_bytes
{
return Err(anyhow::anyhow!(
"汉化 manifest 产物校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
file.path,
file.localized_blake3,
file.localized_bytes,
localized_hash,
localized.len()
));
}
if localized_hash == original_hash {
return Err(anyhow::anyhow!(
"汉化 manifest 文件未发生变化:{}",
file.path
));
}
operation_count += file.text_asset_operations.len();
}
if manifest.file_count != manifest.files.len() {
return Err(anyhow::anyhow!(
"汉化 manifest file_count 不一致:声明 {},实际 {}",
manifest.file_count,
manifest.files.len()
));
}
if manifest.text_asset_operation_count != operation_count {
return Err(anyhow::anyhow!(
"汉化 manifest operation_count 不一致:声明 {},实际 {}",
manifest.text_asset_operation_count,
operation_count
));
}
Ok(LocalizedPatchIntegrity {
verified_changed_file_count: manifest.files.len(),
verified_text_asset_operation_count: operation_count,
current_points_to_release: false,
})
}
fn current_symlink_target(current_path: &Path) -> anyhow::Result<Option<PathBuf>> {
match fs::symlink_metadata(current_path) {
Ok(metadata) if metadata.file_type().is_symlink() => Ok(Some(fs::read_link(current_path)?)),
Ok(_) => Err(anyhow::anyhow!(
"汉化 current 已存在但不是 symlink{}",
current_path.display()
)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error.into()),
}
}
fn current_points_to_version(current_path: &Path, version_path: &Path) -> anyhow::Result<bool> {
let metadata = fs::symlink_metadata(current_path)?;
if !metadata.file_type().is_symlink() {
return Ok(false);
}
Ok(fs::canonicalize(current_path)? == fs::canonicalize(version_path)?)
}
fn rollback_failed_publish(
localized_output_root: &Path,
staging: &Path,
version_path: &Path,
remove_version_path: bool,
current_path: &Path,
previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
remove_owned_path(staging)?;
if remove_version_path {
remove_owned_path(version_path)?;
}
remove_owned_path(&localized_output_root.join(".current.tmp"))?;
remove_owned_path(&localized_output_root.join(".current.rollback.tmp"))?;
let failed_target = version_path
.file_name()
.map(|release_id| Path::new(LOCALIZED_VERSIONS_DIR).join(release_id));
if failed_target.as_ref().is_some_and(|target| {
fs::read_link(current_path)
.map(|current_target| current_target == *target)
.unwrap_or(false)
}) {
restore_current_symlink(localized_output_root, current_path, previous_current_target)?;
}
Ok(())
}
fn remove_owned_path(path: &Path) -> anyhow::Result<()> {
if let Ok(metadata) = fs::symlink_metadata(path) {
if metadata.file_type().is_symlink() || metadata.is_file() {
fs::remove_file(path)?;
} else if metadata.is_dir() {
fs::remove_dir_all(path)?;
}
}
Ok(())
}
#[cfg(unix)]
fn restore_current_symlink(
root: &Path,
current_path: &Path,
previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
use std::os::unix::fs::symlink;
if let Some(previous_target) = previous_current_target {
let temporary = root.join(".current.rollback.tmp");
remove_owned_path(&temporary)?;
symlink(previous_target, &temporary)?;
fs::rename(temporary, current_path)?;
} else if fs::symlink_metadata(current_path)
.map(|metadata| metadata.file_type().is_symlink())
.unwrap_or(false)
{
fs::remove_file(current_path)?;
}
Ok(())
}
#[cfg(not(unix))]
fn restore_current_symlink(
_root: &Path,
_current_path: &Path,
_previous_current_target: Option<&PathBuf>,
) -> anyhow::Result<()> {
Ok(())
}
fn validate_config(config: &LocalizedPatchConfig) -> Result<(), String> {
let official = lexical_absolute(&config.official_release_root)?;
let localized = lexical_absolute(&config.localized_output_root)?;
if official == localized || official.starts_with(&localized) || localized.starts_with(&official)
{
return Err(format!(
"官方 release 与汉化输出目录不能相同或互相嵌套:官方={} 汉化={}",
official.display(),
localized.display()
));
}
if config.release_id.is_empty()
|| config.release_id.contains('/')
|| config.release_id.contains('\\')
|| config.release_id == "."
|| config.release_id == ".."
{
return Err(format!("非法汉化 release id{}", config.release_id));
}
ensure_safe_directory_path(&config.official_release_root, "官方 release")?;
ensure_safe_directory_path(&config.localized_output_root, "汉化输出目录")?;
Ok(())
}
fn copy_tree(source: &Path, destination: &Path) -> anyhow::Result<()> {
let metadata = fs::symlink_metadata(source)?;
if metadata.file_type().is_symlink() {
return Err(anyhow::anyhow!(
"官方 release 不能包含 symlink: {}",
source.display()
));
}
if metadata.is_dir() {
fs::create_dir_all(destination)?;
for entry in fs::read_dir(source)? {
let entry = entry?;
copy_tree(&entry.path(), &destination.join(entry.file_name()))?;
}
} else if metadata.is_file() {
if let Some(parent) = destination.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(source, destination)?;
} else {
return Err(anyhow::anyhow!(
"官方 release 中存在非普通文件: {}",
source.display()
));
}
Ok(())
}
fn remove_owned_staging(path: &Path) -> anyhow::Result<()> {
if let Ok(metadata) = fs::symlink_metadata(path) {
if metadata.file_type().is_symlink() {
return Err(anyhow::anyhow!(
"汉化 staging 不能是 symlink: {}",
path.display()
));
}
if metadata.is_dir() {
fs::remove_dir_all(path)?;
} else {
fs::remove_file(path)?;
}
}
Ok(())
}
#[cfg(unix)]
fn switch_current_symlink(root: &Path, current: &Path, release_id: &str) -> anyhow::Result<()> {
use std::os::unix::fs::symlink;
let temporary = root.join(".current.tmp");
if let Ok(metadata) = fs::symlink_metadata(&temporary) {
if metadata.file_type().is_symlink() || metadata.is_file() {
fs::remove_file(&temporary)?;
} else if metadata.is_dir() {
fs::remove_dir_all(&temporary)?;
}
}
symlink(
Path::new(LOCALIZED_VERSIONS_DIR).join(release_id),
&temporary,
)?;
fs::rename(temporary, current)?;
Ok(())
}
#[cfg(not(unix))]
fn switch_current_symlink(_root: &Path, _current: &Path, _release_id: &str) -> anyhow::Result<()> {
Err(anyhow::anyhow!(
"localized release publication requires a Unix symlink-capable platform"
))
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_patch_manifest_version() -> u32 {
LOCALIZED_PATCH_MANIFEST_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn localized_manifest_converts_to_generic_patch_manifest() {
let source = b"source";
let target = b"target";
let manifest = LocalizedPatchManifest {
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
official_release_id: "official-v1".to_string(),
localized_release_id: "localized-v1".to_string(),
generated_unix_seconds: 123,
file_count: 1,
text_asset_operation_count: 1,
files: vec![LocalizedPatchFile {
path: "Bundles/file.bundle".to_string(),
original_blake3: blake3::hash(source).to_hex().to_string(),
localized_blake3: blake3::hash(target).to_hex().to_string(),
original_bytes: source.len() as u64,
localized_bytes: target.len() as u64,
byte_delta: target.len() as i64 - source.len() as i64,
text_asset_operations: vec![LocalizedPatchOperation {
serialized_file_path: "CAB-asset".to_string(),
path_id: 1,
expected_name: Some("Text".to_string()),
replacement_bytes: target.len() as u64,
replacement_blake3: blake3::hash(target).to_hex().to_string(),
}],
}],
rollback: LocalizedPatchRollbackInfo {
previous_current_target: Some(PathBuf::from("versions/previous")),
remove_version_path: PathBuf::from("versions/localized-v1"),
},
};
let patch_manifest = manifest.to_patch_manifest();
assert_eq!(patch_manifest.version, bat_patch::PATCH_MANIFEST_VERSION);
assert_eq!(patch_manifest.patch_id, "localized-v1");
assert_eq!(patch_manifest.source_version, "official-v1");
assert_eq!(patch_manifest.target_version, "localized-v1");
assert_eq!(patch_manifest.files.len(), 1);
assert_eq!(
patch_manifest.files[0].patch_kind,
bat_patch::PatchKind::UnityFsTextAsset
);
assert_eq!(
patch_manifest.rollback.previous_current_target,
Some(PathBuf::from("versions/previous"))
);
assert_eq!(
patch_manifest.rollback.remove_target_path,
Some(PathBuf::from("versions/localized-v1"))
);
}
#[cfg(unix)]
#[test]
fn publishes_a_separate_localized_release_atomically() {
let temp = TempDir::new().unwrap();
let official = temp.path().join("official-release");
let localized = temp.path().join("localized");
fs::create_dir_all(official.join("TableBundles")).unwrap();
fs::write(
official.join("TableBundles/TableCatalog.bytes"),
b"official",
)
.unwrap();
let report = LocalizedPatchService::new()
.publish(&LocalizedPatchConfig::new(
&official,
&localized,
"release-1",
Vec::new(),
))
.unwrap();
assert_eq!(
fs::read(report.version_path.join("TableBundles/TableCatalog.bytes")).unwrap(),
b"official"
);
assert_eq!(
fs::read_link(report.current_path).unwrap(),
PathBuf::from("versions/release-1")
);
let state: LocalizedVersionState =
serde_json::from_slice(&fs::read(report.state_path).unwrap()).unwrap();
assert_eq!(state.status, "localized");
assert_eq!(state.current_release_id.as_deref(), Some("release-1"));
assert!(report.patch_manifest_path.is_file());
assert_eq!(report.manifest.file_count, 0);
assert_eq!(report.integrity.verified_changed_file_count, 0);
assert!(report.integrity.current_points_to_release);
let manifest = read_localized_patch_manifest_at(&report.version_path)
.unwrap()
.unwrap();
assert_eq!(manifest.localized_release_id, "release-1");
assert_eq!(manifest.rollback.previous_current_target, None);
}
#[cfg(unix)]
#[test]
fn failed_patch_publish_cleans_staging_and_unpublished_version() {
let temp = TempDir::new().unwrap();
let official = temp.path().join("official-release");
let localized = temp.path().join("localized");
fs::create_dir_all(official.join("Bundles")).unwrap();
fs::write(official.join("Bundles/bad.bundle"), b"not-unityfs").unwrap();
let error = LocalizedPatchService::new()
.publish(&LocalizedPatchConfig::new(
&official,
&localized,
"release-1",
vec![LocalizedTextAssetPatch {
bundle_path: "Bundles/bad.bundle".to_string(),
text_asset: TextAssetPatch::new("CAB-bad", 1, b"replacement".to_vec()),
}],
))
.unwrap_err();
assert!(error.to_string().contains("Bundles/bad.bundle"));
assert!(!localized
.join(LOCALIZED_STAGING_DIR)
.join("release-1")
.exists());
assert!(!localized
.join(LOCALIZED_VERSIONS_DIR)
.join("release-1")
.exists());
assert!(!localized.join(LOCALIZED_CURRENT_LINK).exists());
}
}
+653
View File
@@ -0,0 +1,653 @@
//! Official resource change sets and translation handoff files.
//!
//! This module is intentionally file-based. Official update generates the
//! durable change set after a new release has been fully downloaded and
//! verified; parser and translation modules can then consume the same immutable
//! handoff without depending on daemon internals.
use crate::official_download::{
read_download_manifest_at, OfficialDownloadManifest, OfficialDownloadManifestEntry,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
/// Current resource-change-set schema version.
pub const OFFICIAL_RESOURCE_CHANGES_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const OFFICIAL_RESOURCE_CHANGES_FILE: &str = "official-resource-changes.json";
/// Current Crowdin handoff schema version.
pub const CROWDIN_TRANSLATION_HANDOFF_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const CROWDIN_TRANSLATION_HANDOFF_FILE: &str = "crowdin-translation-handoff.json";
/// Change kind for one official resource destination.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialResourceChangeKind {
/// Destination did not exist in the previous complete release.
Added,
/// Destination exists in both releases, but verified size or BLAKE3 changed.
Modified,
/// Destination existed in the previous release but is absent from the new one.
Removed,
}
impl OfficialResourceChangeKind {
/// Returns the stable JSON/RPC label for the change kind.
pub fn as_str(self) -> &'static str {
match self {
Self::Added => "added",
Self::Modified => "modified",
Self::Removed => "removed",
}
}
/// Returns true when the changed resource should be offered to parser and
/// translation modules.
pub fn is_incremental_candidate(self) -> bool {
matches!(self, Self::Added | Self::Modified)
}
}
/// Verified manifest attributes for one official resource.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceDescriptor {
/// Official URL from the download manifest.
pub url: String,
/// Relative path under the official release root.
pub destination: String,
/// Verified byte count from the download manifest.
pub bytes: u64,
/// Verified BLAKE3 digest from the download manifest.
pub blake3: String,
}
impl From<&OfficialDownloadManifestEntry> for OfficialResourceDescriptor {
fn from(entry: &OfficialDownloadManifestEntry) -> Self {
Self {
url: entry.url.clone(),
destination: entry.destination.clone(),
bytes: entry.bytes,
blake3: entry.blake3.clone(),
}
}
}
/// One resource-level change between two complete official releases.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChange {
/// Relative path used as the stable comparison key.
pub destination: String,
/// Change kind for this destination.
pub kind: OfficialResourceChangeKind,
/// Previous release descriptor, when the destination existed before.
pub previous: Option<OfficialResourceDescriptor>,
/// Current release descriptor, when the destination exists now.
pub current: Option<OfficialResourceDescriptor>,
/// Whether parser modules should inspect this resource in incremental mode.
pub parse_candidate: bool,
/// Whether translation modules should enqueue this resource in incremental mode.
pub translation_candidate: bool,
}
impl OfficialResourceChange {
fn new(
destination: String,
kind: OfficialResourceChangeKind,
previous: Option<OfficialResourceDescriptor>,
current: Option<OfficialResourceDescriptor>,
) -> Self {
let is_candidate = kind.is_incremental_candidate();
Self {
destination,
kind,
previous,
current,
parse_candidate: is_candidate,
translation_candidate: is_candidate,
}
}
}
/// Aggregate counters for one official resource change set.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeSummary {
/// Whether a previous complete release manifest was available.
pub previous_manifest_present: bool,
/// Number of entries in the previous release manifest.
pub previous_manifest_entry_count: usize,
/// Number of entries in the current release manifest.
pub current_manifest_entry_count: usize,
/// Number of newly added destinations.
pub added_count: usize,
/// Number of destinations whose verified bytes or BLAKE3 changed.
pub modified_count: usize,
/// Number of destinations removed from the current release.
pub removed_count: usize,
/// Number of resources to offer to parser modules.
pub parse_candidate_count: usize,
/// Number of resources to offer to translation modules.
pub translation_candidate_count: usize,
}
/// Durable comparison result between a previous complete official release and
/// the newly published official release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeSet {
/// Change-set schema version.
#[serde(default = "default_resource_changes_version")]
pub change_set_version: u32,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Previous complete release root, when available.
pub previous_resource_root: Option<PathBuf>,
/// Current complete release root.
pub current_resource_root: PathBuf,
/// Aggregate counters.
pub summary: OfficialResourceChangeSummary,
/// Stable, destination-sorted list of changed resources.
pub changes: Vec<OfficialResourceChange>,
}
impl OfficialResourceChangeSet {
/// Builds a change set from already loaded manifests.
pub fn from_manifests(
official_release_id: impl Into<String>,
previous_release_id: Option<String>,
previous_resource_root: Option<PathBuf>,
current_resource_root: PathBuf,
previous_manifest: Option<&OfficialDownloadManifest>,
current_manifest: &OfficialDownloadManifest,
) -> Self {
let previous_by_destination = previous_manifest
.map(entries_by_destination)
.unwrap_or_default();
let current_by_destination = entries_by_destination(current_manifest);
let destinations = previous_by_destination
.keys()
.chain(current_by_destination.keys())
.cloned()
.collect::<BTreeSet<_>>();
let mut changes = Vec::new();
let mut summary = OfficialResourceChangeSummary {
previous_manifest_present: previous_manifest.is_some(),
previous_manifest_entry_count: previous_manifest
.map(|manifest| manifest.entries.len())
.unwrap_or(0),
current_manifest_entry_count: current_manifest.entries.len(),
..OfficialResourceChangeSummary::default()
};
for destination in destinations {
match (
previous_by_destination.get(&destination),
current_by_destination.get(&destination),
) {
(None, Some(current)) => {
summary.added_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Added,
None,
Some((*current).into()),
));
}
(Some(previous), Some(current)) if content_changed(previous, current) => {
summary.modified_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Modified,
Some((*previous).into()),
Some((*current).into()),
));
}
(Some(previous), None) => {
summary.removed_count += 1;
changes.push(OfficialResourceChange::new(
destination,
OfficialResourceChangeKind::Removed,
Some((*previous).into()),
None,
));
}
_ => {}
}
}
summary.parse_candidate_count = changes
.iter()
.filter(|change| change.parse_candidate)
.count();
summary.translation_candidate_count = changes
.iter()
.filter(|change| change.translation_candidate)
.count();
Self {
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
official_release_id: official_release_id.into(),
previous_release_id,
generated_unix_seconds: unix_seconds_now(),
previous_resource_root,
current_resource_root,
summary,
changes,
}
}
/// Returns the resources that parser modules should inspect for
/// incremental work.
pub fn parse_candidates(&self) -> Vec<&OfficialResourceChange> {
self.changes
.iter()
.filter(|change| change.parse_candidate)
.collect()
}
/// Returns the resources that translation modules should enqueue.
pub fn translation_candidates(&self) -> Vec<&OfficialResourceChange> {
self.changes
.iter()
.filter(|change| change.translation_candidate)
.collect()
}
}
/// Provider reserved for translation handoff consumers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationHandoffProvider {
/// Crowdin provider. The handoff file does not make a network request.
Crowdin,
}
impl TranslationHandoffProvider {
/// Returns the stable provider label.
pub fn as_str(self) -> &'static str {
match self {
Self::Crowdin => "crowdin",
}
}
}
/// Status of a generated translation handoff.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationHandoffStatus {
/// The handoff was written locally and is waiting for a translation worker.
QueuedOffline,
}
impl TranslationHandoffStatus {
/// Returns the stable status label.
pub fn as_str(self) -> &'static str {
match self {
Self::QueuedOffline => "queued_offline",
}
}
}
/// One resource entry queued for translation-provider processing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TranslationHandoffResource {
/// Relative path under the official release root.
pub destination: String,
/// Change kind that caused this resource to be queued.
pub kind: OfficialResourceChangeKind,
/// Current official URL.
pub url: String,
/// Verified byte count.
pub bytes: u64,
/// Verified BLAKE3 digest.
pub blake3: String,
}
/// Crowdin-ready local queue file for added or modified official resources.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTranslationHandoff {
/// Handoff schema version.
#[serde(default = "default_crowdin_handoff_version")]
pub handoff_version: u32,
/// Translation provider reserved for this queue.
pub provider: TranslationHandoffProvider,
/// Current queue status.
pub status: TranslationHandoffStatus,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Number of queued resources.
pub resource_count: usize,
/// Added or modified resources to pass into parsing/translation workers.
pub resources: Vec<TranslationHandoffResource>,
}
impl CrowdinTranslationHandoff {
/// Builds a local Crowdin handoff from a resource change set.
pub fn from_change_set(change_set: &OfficialResourceChangeSet) -> Self {
let resources = change_set
.translation_candidates()
.into_iter()
.filter_map(|change| {
let current = change.current.as_ref()?;
Some(TranslationHandoffResource {
destination: change.destination.clone(),
kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
})
})
.collect::<Vec<_>>();
Self {
handoff_version: CROWDIN_TRANSLATION_HANDOFF_VERSION,
provider: TranslationHandoffProvider::Crowdin,
status: TranslationHandoffStatus::QueuedOffline,
official_release_id: change_set.official_release_id.clone(),
previous_release_id: change_set.previous_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
resource_count: resources.len(),
resources,
}
}
}
/// Generates a change set for two complete release roots and writes both the
/// change set and the Crowdin handoff under the current release root.
pub fn write_official_resource_change_handoff(
previous_resource_root: Option<&Path>,
current_resource_root: &Path,
official_release_id: &str,
previous_release_id: Option<String>,
) -> Result<OfficialResourceChangeHandoffReport, String> {
let current_manifest = read_download_manifest_at(current_resource_root)?.ok_or_else(|| {
format!(
"缺少当前官方下载 manifest,无法生成资源变更集:{}",
current_resource_root.display()
)
})?;
let previous_manifest = match previous_resource_root {
Some(root) => read_download_manifest_at(root)?,
None => None,
};
let previous_manifest_root = previous_manifest
.as_ref()
.and(previous_resource_root)
.map(Path::to_path_buf);
let previous_release_id = previous_manifest.as_ref().and(previous_release_id);
let change_set = OfficialResourceChangeSet::from_manifests(
official_release_id,
previous_release_id,
previous_manifest_root,
current_resource_root.to_path_buf(),
previous_manifest.as_ref(),
&current_manifest,
);
write_resource_change_set_at(current_resource_root, &change_set)?;
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
write_crowdin_translation_handoff_at(current_resource_root, &handoff)?;
Ok(OfficialResourceChangeHandoffReport {
change_set_path: current_resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE),
crowdin_handoff_path: current_resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE),
summary: change_set.summary,
})
}
/// Paths and summary produced after writing a resource-change handoff.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceChangeHandoffReport {
/// Path to `official-resource-changes.json`.
pub change_set_path: PathBuf,
/// Path to `crowdin-translation-handoff.json`.
pub crowdin_handoff_path: PathBuf,
/// Aggregate change counters.
pub summary: OfficialResourceChangeSummary,
}
/// Reads a generated resource change set from a release root.
pub fn read_resource_change_set_at(
resource_root: &Path,
) -> Result<Option<OfficialResourceChangeSet>, String> {
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方资源变更集")? else {
return Ok(None);
};
let change_set: OfficialResourceChangeSet = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析官方资源变更集失败 {}{error}", path.display()))?;
if change_set.change_set_version != OFFICIAL_RESOURCE_CHANGES_VERSION {
return Err(format!(
"不支持的官方资源变更集版本 {},文件 {}",
change_set.change_set_version,
path.display()
));
}
Ok(Some(change_set))
}
/// Writes a generated resource change set under a release root.
pub fn write_resource_change_set_at(
resource_root: &Path,
change_set: &OfficialResourceChangeSet,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方资源变更集")?;
let bytes = serde_json::to_vec_pretty(change_set)
.map_err(|error| format!("序列化官方资源变更集失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方资源变更集")
}
/// Writes a generated Crowdin handoff under a release root.
pub fn write_crowdin_translation_handoff_at(
resource_root: &Path,
handoff: &CrowdinTranslationHandoff,
) -> Result<(), String> {
let path = resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "Crowdin 翻译 handoff")?;
let bytes = serde_json::to_vec_pretty(handoff)
.map_err(|error| format!("序列化 Crowdin 翻译 handoff 失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin 翻译 handoff")
}
fn entries_by_destination(
manifest: &OfficialDownloadManifest,
) -> BTreeMap<String, &OfficialDownloadManifestEntry> {
manifest
.entries
.values()
.map(|entry| (entry.destination.clone(), entry))
.collect()
}
fn content_changed(
previous: &OfficialDownloadManifestEntry,
current: &OfficialDownloadManifestEntry,
) -> bool {
previous.bytes != current.bytes || previous.blake3 != current.blake3
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_resource_changes_version() -> u32 {
OFFICIAL_RESOURCE_CHANGES_VERSION
}
fn default_crowdin_handoff_version() -> u32 {
CROWDIN_TRANSLATION_HANDOFF_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
fn manifest(entries: &[(&str, &str, &[u8])]) -> OfficialDownloadManifest {
let mut manifest = OfficialDownloadManifest::default();
for (url, destination, bytes) in entries {
manifest.entries.insert(
(*url).to_string(),
OfficialDownloadManifestEntry {
url: (*url).to_string(),
destination: (*destination).to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
);
}
manifest
}
fn write_manifest(root: &Path, manifest: &OfficialDownloadManifest) {
std::fs::create_dir_all(root).unwrap();
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(manifest).unwrap(),
)
.unwrap();
}
#[test]
fn change_set_classifies_added_modified_and_removed_resources() {
let previous = manifest(&[
("https://old/a", "TableBundles/a.bytes", b"old-a"),
("https://old/b", "TableBundles/b.bytes", b"same"),
("https://old/c", "TableBundles/c.bytes", b"removed"),
(
"https://old/u",
"TableBundles/url-only.bytes",
b"same-url-only",
),
]);
let current = manifest(&[
("https://new/a", "TableBundles/a.bytes", b"new-a"),
("https://new/b", "TableBundles/b.bytes", b"same"),
("https://new/d", "TableBundles/d.bytes", b"added"),
(
"https://changed-host/u",
"TableBundles/url-only.bytes",
b"same-url-only",
),
]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
Some("release-old".to_string()),
Some(PathBuf::from("/previous")),
PathBuf::from("/current"),
Some(&previous),
&current,
);
assert_eq!(change_set.summary.added_count, 1);
assert_eq!(change_set.summary.modified_count, 1);
assert_eq!(change_set.summary.removed_count, 1);
assert_eq!(change_set.summary.parse_candidate_count, 2);
assert_eq!(change_set.summary.translation_candidate_count, 2);
let destinations = change_set
.translation_candidates()
.into_iter()
.map(|change| change.destination.as_str())
.collect::<Vec<_>>();
assert_eq!(
destinations,
vec!["TableBundles/a.bytes", "TableBundles/d.bytes"]
);
}
#[test]
fn first_release_treats_all_current_resources_as_added() {
let current = manifest(&[
("https://new/a", "a.bundle", b"a"),
("https://new/b", "b.bundle", b"b"),
]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
None,
None,
PathBuf::from("/current"),
None,
&current,
);
assert!(!change_set.summary.previous_manifest_present);
assert_eq!(change_set.summary.added_count, 2);
assert_eq!(change_set.summary.translation_candidate_count, 2);
assert_eq!(change_set.changes.len(), 2);
}
#[test]
fn crowdin_handoff_excludes_removed_resources() {
let previous = manifest(&[("https://old/a", "a.bundle", b"a")]);
let current = manifest(&[("https://new/b", "b.bundle", b"b")]);
let change_set = OfficialResourceChangeSet::from_manifests(
"release-new",
Some("release-old".to_string()),
None,
PathBuf::from("/current"),
Some(&previous),
&current,
);
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
assert_eq!(handoff.provider, TranslationHandoffProvider::Crowdin);
assert_eq!(handoff.status, TranslationHandoffStatus::QueuedOffline);
assert_eq!(handoff.resource_count, 1);
assert_eq!(handoff.resources[0].destination, "b.bundle");
}
#[test]
fn write_handoff_persists_change_set_and_crowdin_queue() {
let temp = tempfile::TempDir::new().unwrap();
let previous_root = temp.path().join("previous");
let current_root = temp.path().join("current");
write_manifest(
&previous_root,
&manifest(&[("https://old/a", "a.bundle", b"old")]),
);
write_manifest(
&current_root,
&manifest(&[
("https://new/a", "a.bundle", b"new"),
("https://new/b", "b.bundle", b"added"),
]),
);
let report = write_official_resource_change_handoff(
Some(&previous_root),
&current_root,
"release-new",
Some("release-old".to_string()),
)
.unwrap();
assert_eq!(report.summary.modified_count, 1);
assert_eq!(report.summary.added_count, 1);
assert!(report.change_set_path.exists());
assert!(report.crowdin_handoff_path.exists());
let change_set = read_resource_change_set_at(&current_root).unwrap().unwrap();
assert_eq!(change_set.summary.translation_candidate_count, 2);
}
}
+7 -5
View File
@@ -115,8 +115,8 @@ pub struct OfficialResourcePullProgress {
///
/// `Started` events report the currently completed count before the URL
/// finishes; `Finished` events report the count after completion. This is
/// intentionally not the URL's plan position, because concurrent downloads
/// finish out of plan order and status percentages must not move backward.
/// intentionally not the URL's plan position, so status percentages stay
/// monotonic even if execution order or skip/resume mix changes.
pub index: usize,
/// Total URL count in the pull plan.
pub total: usize,
@@ -2272,14 +2272,16 @@ exit 22
}
fn write_shell_script(path: &Path, script: &str) {
fs::write(path, script).unwrap();
let temp_path = path.with_extension("tmp");
fs::write(&temp_path, script).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(path).unwrap().permissions();
let mut permissions = fs::metadata(&temp_path).unwrap().permissions();
permissions.set_mode(0o755);
fs::set_permissions(path, permissions).unwrap();
fs::set_permissions(&temp_path, permissions).unwrap();
}
fs::rename(temp_path, path).unwrap();
}
fn write_fake_curl(path: &Path) {
+101 -10
View File
@@ -9,7 +9,8 @@ use crate::official_download::DownloadError;
use crate::official_launcher::launcher_package_url;
use crate::official_launcher::OfficialLauncherBootstrapService;
use crate::official_launcher::{
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherRemoteManifest,
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl,
YostarJpLauncherRemoteManifest,
};
use crate::zip_validation::validate_zip_structure;
use bat_adapters::official::game_main_config::YostarJpGameMainConfig;
@@ -39,12 +40,44 @@ pub struct OfficialGameMainConfigBootstrap {
/// may instead point to a directory source plus per-file entries; in that
/// case this is the direct `resources.assets` URL.
pub game_zip_url: String,
/// Remote launcher manifest used for this bootstrap.
pub remote_manifest: YostarJpLauncherRemoteManifest,
/// Exact source selected to obtain `GameMainConfig`.
pub selected_source: OfficialGameMainConfigSelectedSource,
/// Number of files declared by the remote manifest.
pub manifest_file_count: usize,
/// Decrypted `GameMainConfig`.
pub game_main_config: YostarJpGameMainConfig,
}
/// Kind of official launcher package artifact selected for `GameMainConfig`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OfficialGameMainConfigSourceKind {
/// Older launcher manifests point to a single game ZIP archive.
Archive,
/// Current launcher manifests expose a directory plus per-file entries.
ManifestFile,
}
/// Exact launcher artifact selected to obtain `resources.assets`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialGameMainConfigSelectedSource {
/// Source kind.
pub kind: OfficialGameMainConfigSourceKind,
/// Official URL fetched for this source.
pub url: String,
/// Relative path under the official launcher package CDN root.
pub relative_path: String,
/// Original manifest file path when the source is a manifest file entry.
pub manifest_path: Option<String>,
/// Declared file size from the manifest, when available.
pub declared_size: Option<u64>,
/// Official launcher manifest `hash` field, when available.
pub official_hash: Option<String>,
/// Official launcher manifest per-file `vc`, when available.
pub vc: Option<String>,
}
/// Loads and decrypts the official `GameMainConfig` by following the official
/// launcher package chain.
#[derive(Debug, Clone)]
@@ -92,6 +125,18 @@ impl OfficialGameMainConfigBootstrapService {
/// launcher API, extracts `resources.assets`, and decrypts `GameMainConfig`.
pub fn fetch_bootstrap(&self) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
let (game_config, manifest_url, manifest) = self.launcher.fetch_latest_remote_manifest()?;
let cdn_config = self.launcher.fetch_cdn_config()?;
self.fetch_bootstrap_from_parts(game_config, cdn_config, manifest_url, manifest)
}
/// Extracts `GameMainConfig` from already fetched launcher metadata.
pub fn fetch_bootstrap_from_parts(
&self,
game_config: YostarJpLauncherGameConfig,
cdn_config: YostarJpLauncherCdnConfig,
manifest_url: YostarJpLauncherManifestUrl,
manifest: YostarJpLauncherRemoteManifest,
) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
let manifest_source = manifest
.source
.clone()
@@ -102,9 +147,8 @@ impl OfficialGameMainConfigBootstrapService {
"官方启动器远端 manifest 缺少 source",
)
})?;
let cdn_config = self.launcher.fetch_cdn_config()?;
let temp_dir = TempDir::new().map_err(|error| format!("创建临时目录失败:{error}"))?;
let (game_zip_url, resources_assets) = self.fetch_resources_assets(
let (game_zip_url, resources_assets, selected_source) = self.fetch_resources_assets(
&game_config,
&manifest,
&manifest_source,
@@ -113,6 +157,7 @@ impl OfficialGameMainConfigBootstrapService {
)?;
let game_main_config = YostarJpGameMainConfig::from_resources_assets(resources_assets)
.map_err(|error| DownloadError::new(ErrorCode::GAME_MAIN_CONFIG_FAILED, error))?;
let manifest_file_count = manifest.files.len();
Ok(OfficialGameMainConfigBootstrap {
game_config,
@@ -120,7 +165,9 @@ impl OfficialGameMainConfigBootstrapService {
manifest_url: manifest_url.url,
manifest_source: Some(manifest_source),
game_zip_url,
manifest_file_count: manifest.files.len(),
remote_manifest: manifest,
selected_source,
manifest_file_count,
game_main_config,
})
}
@@ -203,10 +250,12 @@ impl OfficialGameMainConfigBootstrapService {
manifest_source: &str,
cdn_config: &YostarJpLauncherCdnConfig,
temp_root: &Path,
) -> Result<(String, PathBuf), DownloadError> {
) -> Result<(String, PathBuf, OfficialGameMainConfigSelectedSource), DownloadError> {
let selected_source =
resolve_game_main_config_source(game_config, manifest, manifest_source, cdn_config)?;
match select_game_main_config_source(game_config, manifest_source, manifest)? {
GameMainConfigSource::Archive(package_path) => {
let game_zip_url = launcher_package_url(&cdn_config.primary_cdn, package_path)?;
let game_zip_url = selected_source.url.clone();
let archive_path = temp_root.join("official-game.zip");
self.download_file_with_fallback(
&game_zip_url,
@@ -227,12 +276,11 @@ impl OfficialGameMainConfigBootstrapService {
"官方启动器包内没有找到 resources.assets",
)
})?;
Ok((game_zip_url, resources_assets))
Ok((game_zip_url, resources_assets, selected_source))
}
GameMainConfigSource::ManifestFile { source_dir, file } => {
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)?;
let resources_assets_url =
launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
let resources_assets_url = selected_source.url.clone();
let resources_assets = temp_root.join("resources.assets");
self.download_file_with_fallback(
&resources_assets_url,
@@ -241,7 +289,7 @@ impl OfficialGameMainConfigBootstrapService {
&resources_assets,
)?;
verify_manifest_file_size(&resources_assets, file)?;
Ok((resources_assets_url, resources_assets))
Ok((resources_assets_url, resources_assets, selected_source))
}
}
}
@@ -283,6 +331,49 @@ impl OfficialGameMainConfigBootstrapService {
}
}
/// Resolves the exact official launcher artifact used to obtain `GameMainConfig`.
pub fn resolve_game_main_config_source(
game_config: &YostarJpLauncherGameConfig,
manifest: &YostarJpLauncherRemoteManifest,
manifest_source: &str,
cdn_config: &YostarJpLauncherCdnConfig,
) -> Result<OfficialGameMainConfigSelectedSource, DownloadError> {
match select_game_main_config_source(game_config, manifest_source, manifest)? {
GameMainConfigSource::Archive(package_path) => {
let url = launcher_package_url(&cdn_config.primary_cdn, package_path)?;
Ok(OfficialGameMainConfigSelectedSource {
kind: OfficialGameMainConfigSourceKind::Archive,
url,
relative_path: package_path.to_string(),
manifest_path: None,
declared_size: None,
official_hash: None,
vc: None,
})
}
GameMainConfigSource::ManifestFile { source_dir, file } => {
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)
.map_err(|error| DownloadError::new(ErrorCode::LAUNCHER_RESPONSE_INVALID, error))?;
let url = launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
let declared_size = file.size.parse::<u64>().map_err(|error| {
DownloadError::new(
ErrorCode::LAUNCHER_RESPONSE_INVALID,
format!("官方启动器 manifest 中 {} 的 size 无效:{error}", file.path),
)
})?;
Ok(OfficialGameMainConfigSelectedSource {
kind: OfficialGameMainConfigSourceKind::ManifestFile,
url,
relative_path,
manifest_path: Some(file.path.clone()),
declared_size: Some(declared_size),
official_hash: Some(file.hash.clone()),
vc: file.vc.clone(),
})
}
}
}
#[derive(Debug)]
enum GameMainConfigSource<'a> {
Archive(&'a str),
+585 -63
View File
@@ -10,9 +10,11 @@ use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use bat_assetbundle::{Parser, UnityFsParser};
use bat_assetbundle::{
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -20,7 +22,11 @@ use std::time::{SystemTime, UNIX_EPOCH};
/// Parse-cache file name stored under a published official resource root.
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
/// Current parse-cache schema version.
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1;
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
/// TextUnit detail index file name stored under a published official resource root.
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
/// Current TextUnit detail-index schema version.
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
/// Configuration for one official resource parse-cache refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -55,6 +61,10 @@ pub struct OfficialParseReport {
pub cache_path: PathBuf,
/// Aggregate parse-cache summary.
pub summary: OfficialParseSummary,
/// TextUnit detail-index path written by the refresh.
pub textunit_index_path: PathBuf,
/// Aggregate TextUnit detail-index summary.
pub textunit_index_summary: OfficialTextUnitIndexSummary,
}
/// Aggregate counters for a parse-cache refresh.
@@ -78,6 +88,15 @@ pub struct OfficialParseSummary {
pub failed_count: usize,
/// Total TextAsset objects found in parsed Unity serialized files.
pub text_asset_count: usize,
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
#[serde(default)]
pub text_unit_count: usize,
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Number of non-fatal TypeTree field extraction diagnostics.
#[serde(default)]
pub text_unit_error_count: usize,
}
/// Persistent parse cache for one official resource root.
@@ -127,10 +146,162 @@ pub struct OfficialParseCacheEntry {
pub text_assets: Vec<String>,
/// Non-fatal serialized-file parse diagnostic count.
pub serialized_parse_error_count: usize,
/// Number of translation-ready TextUnit entries extracted from this bundle.
#[serde(default)]
pub text_unit_count: usize,
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
#[serde(default)]
pub text_unit_formats: Vec<String>,
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Non-fatal TypeTree field extraction diagnostic count.
#[serde(default)]
pub text_unit_error_count: usize,
/// Human-readable error or skip reason.
pub error: Option<String>,
}
/// Persistent TextUnit detail index for one official resource root.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndex {
/// Index schema version.
#[serde(default = "default_textunit_index_version")]
pub version: u32,
/// Index generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Official resource root used to generate this index.
pub resource_root: PathBuf,
/// Aggregate counters for this index.
pub summary: OfficialTextUnitIndexSummary,
/// Extracted TextUnit details in deterministic order.
#[serde(default)]
pub units: Vec<OfficialTextUnitIndexUnit>,
/// Parse and extraction diagnostics in deterministic order.
#[serde(default)]
pub errors: Vec<OfficialTextUnitIndexError>,
}
/// Aggregate counters for a TextUnit detail index.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexSummary {
/// Number of TextUnit detail entries.
pub unit_count: usize,
/// Number of parse/extraction diagnostics.
pub error_count: usize,
/// Number of binary or invalid TextAsset payloads skipped by extraction.
pub skipped_binary_text_asset_count: usize,
}
/// One TextUnit detail entry.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexUnit {
/// Stable unit ID within the official release.
pub id: String,
/// Parse-cache entry key that produced this unit.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Unity editor version when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unity_version: Option<String>,
/// Original source text.
pub source_text: String,
/// Unity serialized file path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// TextUnit payload format such as json/csv/tsv/plain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
/// Extraction source kind such as TextAsset or TypeTreeField.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_source_kind: Option<String>,
/// TextAsset name when the unit came from a TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset_name: Option<String>,
/// Stable extraction context copied from the parser.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub context: BTreeMap<String, String>,
}
/// One parse or extraction diagnostic in the TextUnit detail index.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexError {
/// Stable diagnostic ID within the official release.
pub id: String,
/// Parse-cache entry key associated with this diagnostic.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Parse status associated with this diagnostic.
pub status: OfficialParseStatus,
/// Unity serialized file path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable diagnostic.
pub error: String,
}
/// Query filters for TextUnit detail entries.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialTextUnitQuery {
/// Filter by destination path.
pub destination: Option<String>,
/// Filter by destination glob pattern.
pub path_pattern: Option<String>,
/// Filter by archive entry.
pub archive_entry: Option<String>,
/// Filter by Unity object path ID.
pub path_id: Option<i64>,
/// Filter by Unity class ID.
pub class_id: Option<i32>,
/// Filter by field path.
pub field_path: Option<String>,
/// Filter by TextUnit format.
pub format: Option<String>,
}
/// Source kind for a parse-cache entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -187,21 +358,36 @@ impl OfficialParseCacheService {
)
})?;
let previous_cache = read_parse_cache_at(&config.resource_root)?;
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
let mut summary = OfficialParseSummary {
manifest_entry_count: manifest.entries.len(),
..OfficialParseSummary::default()
};
let mut entries = BTreeMap::new();
let mut units = Vec::new();
let mut errors = Vec::new();
for manifest_entry in manifest.entries.values() {
let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref());
for entry in produced {
summary.record_entry(&entry);
entries.insert(entry.key.clone(), entry);
let produced = process_manifest_entry(
config,
manifest_entry,
previous_cache.as_ref(),
previous_textunit_index.as_ref(),
);
for produced in produced {
summary.record_entry(&produced.entry);
units.extend(produced.units);
errors.extend(produced.errors);
entries.insert(produced.entry.key.clone(), produced.entry);
}
}
summary.cache_entry_count = entries.len();
let textunit_index_summary = OfficialTextUnitIndexSummary {
unit_count: units.len(),
error_count: errors.len(),
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: unix_seconds_now(),
@@ -209,15 +395,67 @@ impl OfficialParseCacheService {
entries,
};
write_parse_cache_at(&config.resource_root, &cache)?;
let textunit_index = OfficialTextUnitIndex {
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
generated_unix_seconds: cache.generated_unix_seconds,
resource_root: config.resource_root.clone(),
summary: textunit_index_summary.clone(),
units,
errors,
};
write_textunit_index_at(&config.resource_root, &textunit_index)?;
Ok(OfficialParseReport {
resource_root: config.resource_root.clone(),
cache_path: config.cache_path(),
summary,
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
textunit_index_summary,
})
}
}
struct ParseProduced {
entry: OfficialParseCacheEntry,
units: Vec<OfficialTextUnitIndexUnit>,
errors: Vec<OfficialTextUnitIndexError>,
}
impl ParseProduced {
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
let errors = error_from_status_entry(&entry).into_iter().collect();
Self {
entry,
units: Vec::new(),
errors,
}
}
fn from_cached(
mut entry: OfficialParseCacheEntry,
previous_index: Option<&OfficialTextUnitIndex>,
) -> Self {
entry.reused_from_previous_cache = true;
let units = previous_index
.into_iter()
.flat_map(|index| index.units.iter())
.filter(|unit| unit.parse_entry_key == entry.key)
.cloned()
.collect();
let errors = previous_index
.into_iter()
.flat_map(|index| index.errors.iter())
.filter(|error| error.parse_entry_key == entry.key)
.cloned()
.collect();
Self {
entry,
units,
errors,
}
}
}
impl OfficialParseSummary {
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
match entry.source_kind {
@@ -234,6 +472,9 @@ impl OfficialParseSummary {
OfficialParseStatus::Parsed => {
self.parsed_bundle_count += 1;
self.text_asset_count += entry.text_asset_count;
self.text_unit_count += entry.text_unit_count;
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
self.text_unit_error_count += entry.text_unit_error_count;
}
OfficialParseStatus::SkippedUnsupported => {
self.unsupported_count += 1;
@@ -279,21 +520,81 @@ pub fn write_parse_cache_at(
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
}
/// Reads the TextUnit detail index under a published official resource root.
pub fn read_textunit_index_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitIndex>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
return Ok(None);
};
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
return Ok(None);
};
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
return Ok(None);
}
Ok(Some(index))
}
/// Writes the TextUnit detail index under a published official resource root.
pub fn write_textunit_index_at(
resource_root: &Path,
index: &OfficialTextUnitIndex,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
let bytes = serde_json::to_vec_pretty(index)
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
}
/// Returns TextUnit detail entries matching a query.
pub fn query_textunit_index_units<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexUnit> {
index
.units
.iter()
.filter(|unit| textunit_unit_matches(unit, query))
.collect()
}
/// Returns parse/extraction diagnostics matching a query.
pub fn query_textunit_index_errors<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexError> {
index
.errors
.iter()
.filter(|error| textunit_error_matches(error, query))
.collect()
}
fn process_manifest_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
) -> Vec<OfficialParseCacheEntry> {
previous_index: Option<&OfficialTextUnitIndex>,
) -> Vec<ParseProduced> {
let fingerprint = fingerprint_for(manifest_entry);
if looks_like_zip_source(manifest_entry) {
return process_zip_entry(config, manifest_entry, previous_cache, fingerprint);
return process_zip_entry(
config,
manifest_entry,
previous_cache,
previous_index,
fingerprint,
);
}
if looks_like_direct_bundle_source(manifest_entry) {
let key = direct_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true;
return vec![cached];
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
return vec![parse_direct_bundle(
config,
@@ -304,61 +605,58 @@ fn process_manifest_entry(
}
let key = unsupported_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true;
return vec![cached];
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
vec![unsupported_entry(
vec![ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
key,
"非 UnityFS 候选资源",
)]
))]
}
fn process_zip_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
previous_index: Option<&OfficialTextUnitIndex>,
fingerprint: OfficialParseSourceFingerprint,
) -> Vec<OfficialParseCacheEntry> {
) -> Vec<ParseProduced> {
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
if !cached_entries.is_empty() {
return cached_entries
.into_iter()
.map(|mut entry| {
entry.reused_from_previous_cache = true;
entry
})
.map(|entry| ParseProduced::from_cached(entry, previous_index))
.collect();
}
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return vec![failed_entry(
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
)]
))]
}
};
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
Ok(entries) => entries,
Err(error) => {
return vec![failed_entry(
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
)]
))]
}
};
@@ -369,14 +667,14 @@ fn process_zip_entry(
{
Ok(bytes) => bytes,
Err(error) => {
produced.push(failed_entry(
produced.push(ParseProduced::from_entry(failed_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint.clone(),
key,
error,
));
)));
continue;
}
};
@@ -390,14 +688,14 @@ fn process_zip_entry(
}
if produced.is_empty() {
produced.push(unsupported_entry(
produced.push(ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
zip_list_key(&manifest_entry.url),
"ZIP 内没有可检查文件条目",
));
)));
}
produced
}
@@ -407,42 +705,42 @@ fn parse_direct_bundle(
manifest_entry: &OfficialDownloadManifestEntry,
key: String,
fingerprint: OfficialParseSourceFingerprint,
) -> OfficialParseCacheEntry {
) -> ParseProduced {
let path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return failed_entry(
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
)
))
}
};
let bytes = match read_resource_file(&path) {
Ok(bytes) => bytes,
Err(error) => {
return failed_entry(
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
)
))
}
};
if !UnityFsParser::has_unityfs_signature(&bytes) {
return unsupported_entry(
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
"文件不是 UnityFS bundle",
);
));
}
parsed_bundle_entry(
manifest_entry,
@@ -460,16 +758,16 @@ fn parse_zip_inner_file(
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> OfficialParseCacheEntry {
) -> ParseProduced {
if !UnityFsParser::has_unityfs_signature(bytes) {
return unsupported_entry(
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint,
key,
"ZIP 条目不是 UnityFS bundle",
);
));
}
parsed_bundle_entry(
manifest_entry,
@@ -488,38 +786,254 @@ fn parsed_bundle_entry(
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> OfficialParseCacheEntry {
) -> ParseProduced {
let parser = UnityFsParser::new();
match parser.parse(bytes) {
Ok(parsed) => OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some(parsed.unity_version),
file_count: parsed.files.len(),
serialized_file_count: parsed.serialized_files.len(),
text_asset_count: parsed.text_assets.len(),
text_assets: parsed
.text_assets
Ok(parsed) => {
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
&parsed,
Some(&manifest_entry.destination),
archive_entry.as_deref(),
);
let text_unit_formats = text_units
.units
.iter()
.map(|asset| asset.name.clone())
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
error: None,
},
Err(error) => failed_entry(
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let entry = OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some(parsed.unity_version),
file_count: parsed.files.len(),
serialized_file_count: parsed.serialized_files.len(),
text_asset_count: parsed.text_assets.len(),
text_assets: parsed
.text_assets
.iter()
.map(|asset| asset.name.clone())
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
text_unit_error_count: text_units.errors.len(),
error: None,
};
let units = indexed_text_units_for_entry(&entry, &text_units.units);
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
ParseProduced {
entry,
units,
errors,
}
}
Err(error) => ParseProduced::from_entry(failed_entry(
manifest_entry,
archive_entry,
source_kind,
fingerprint,
key,
error.to_string(),
),
)),
}
}
fn indexed_text_units_for_entry(
entry: &OfficialParseCacheEntry,
units: &[TextUnit],
) -> Vec<OfficialTextUnitIndexUnit> {
units
.iter()
.enumerate()
.map(|(index, unit)| OfficialTextUnitIndexUnit {
id: format!("{}#unit:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
unity_version: entry.unity_version.clone(),
source_text: unit.source_text.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
field_offset: unit.field_offset,
field_byte_size: unit.field_byte_size,
format: unit.context.get("format").cloned(),
text_source_kind: unit.context.get("source_kind").cloned(),
asset_name: unit.context.get("asset_name").cloned(),
context: unit.context.clone(),
})
.collect()
}
fn indexed_extraction_errors_for_entry(
entry: &OfficialParseCacheEntry,
errors: &[TextUnitExtractionError],
) -> Vec<OfficialTextUnitIndexError> {
errors
.iter()
.enumerate()
.map(|(index, error)| OfficialTextUnitIndexError {
id: format!("{}#extract-error:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: error.serialized_file.clone(),
path_id: error.path_id,
class_id: error.class_id,
field_path: error.field_path.clone(),
offset: error.offset,
error: error.error.clone(),
})
.collect()
}
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
let error = entry.error.as_ref()?;
Some(OfficialTextUnitIndexError {
id: format!("{}#status", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: None,
path_id: None,
class_id: None,
field_path: None,
offset: None,
error: error.clone(),
})
}
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &unit.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| unit.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| unit.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
{
return false;
}
if query
.format
.as_ref()
.is_some_and(|format| unit.format.as_ref() != Some(format))
{
return false;
}
true
}
fn textunit_error_matches(
error: &OfficialTextUnitIndexError,
query: &OfficialTextUnitQuery,
) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &error.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| error.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| error.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
{
return false;
}
true
}
fn glob_matches(pattern: &str, value: &str) -> bool {
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
}
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
match pattern.split_first() {
None => value.is_empty(),
Some((&b'*', rest)) => {
glob_matches_bytes(rest, value)
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
}
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
Some((&literal, rest)) => value
.split_first()
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
}
}
@@ -585,6 +1099,10 @@ fn status_entry(
text_asset_count: 0,
text_assets: Vec::new(),
serialized_parse_error_count: 0,
text_unit_count: 0,
text_unit_formats: Vec::new(),
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: Some(reason),
}
}
@@ -735,6 +1253,10 @@ fn default_parse_cache_version() -> u32 {
OFFICIAL_PARSE_CACHE_VERSION
}
fn default_textunit_index_version() -> u32 {
OFFICIAL_TEXTUNIT_INDEX_VERSION
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
+543
View File
@@ -0,0 +1,543 @@
//! Import of a verified official release into CAS and ResourceRepository.
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus,
OfficialParseSummary,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink,
};
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// Configuration for importing one already-published official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialReleaseImportConfig {
/// Published release root containing `official-download-manifest.json`.
pub release_root: PathBuf,
/// Official release ID associated with this root, when known.
pub official_release_id: Option<String>,
}
impl OfficialReleaseImportConfig {
/// Creates an import configuration.
pub fn new(release_root: impl Into<PathBuf>) -> Self {
Self {
release_root: release_root.into(),
official_release_id: None,
}
}
/// Attaches the official release ID that should be stored in resource metadata.
pub fn with_official_release_id(mut self, release_id: impl Into<String>) -> Self {
self.official_release_id = Some(release_id.into());
self
}
}
/// Summary returned by an official release repository import.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialReleaseImportReport {
/// Number of verified manifest entries.
pub manifest_entry_count: usize,
/// Number of new or changed repository rows.
pub imported_count: usize,
/// Number of rows already pointing at the same CAS object.
pub unchanged_count: usize,
/// Number of unchanged rows whose metadata was refreshed from parse cache.
#[serde(default)]
pub metadata_updated_count: usize,
/// Number of resources classified as AssetBundle.
pub asset_bundle_count: usize,
/// Number of resources classified as text-like payloads.
pub text_asset_count: usize,
/// Number of resources classified as tables.
pub table_count: usize,
/// Number of resources classified as media.
pub media_count: usize,
/// Parse-cache summary associated with this release, when available.
pub parse_summary: Option<OfficialParseSummary>,
/// Non-fatal cleanup warnings, such as an old CAS reference that could not
/// be decremented after a successful row replacement.
pub warnings: Vec<String>,
}
/// Imports verified official resources into CAS and the resource index.
pub struct OfficialReleaseImportService<'a> {
cas: &'a dyn CasRepository,
resources: &'a dyn ResourceRepository,
}
impl<'a> OfficialReleaseImportService<'a> {
/// Creates an import service.
pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self {
Self { cas, resources }
}
/// Imports every entry in one published release manifest.
///
/// The source files remain untouched. Every file is checked against the
/// verified download manifest before it can enter CAS. Repository IDs are
/// stable by destination, making repeated imports idempotent.
pub async fn import_release(
&self,
config: &OfficialReleaseImportConfig,
) -> bat_core::Result<OfficialReleaseImportReport> {
let manifest = read_download_manifest_at(&config.release_root)
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| {
bat_core::Error::NotFound(format!(
"官方下载 manifest 不存在:{}",
config.release_root.display()
))
})?;
let parse_cache =
read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?;
let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone());
let parse_entries_by_destination = parse_cache
.as_ref()
.map(parse_entries_by_destination)
.unwrap_or_default();
let release_id = config
.official_release_id
.clone()
.or_else(|| release_id_from_root(&config.release_root));
let mut report = OfficialReleaseImportReport {
manifest_entry_count: manifest.entries.len(),
imported_count: 0,
unchanged_count: 0,
metadata_updated_count: 0,
asset_bundle_count: 0,
text_asset_count: 0,
table_count: 0,
media_count: 0,
parse_summary,
warnings: Vec::new(),
};
for entry in manifest.entries.values() {
let parse_entries = parse_entries_by_destination
.get(&entry.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
self.import_entry(
&config.release_root,
release_id.as_deref(),
entry,
parse_entries,
&mut report,
)
.await?;
}
Ok(report)
}
async fn import_entry(
&self,
release_root: &Path,
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
report: &mut OfficialReleaseImportReport,
) -> bat_core::Result<()> {
let path = release_root.join(Path::new(&manifest_entry.destination));
ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?;
ensure_safe_file_target(release_root, &path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?;
let bytes = read_file_no_symlink(&path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?;
if bytes.len() as u64 != manifest_entry.bytes {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 size 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination,
manifest_entry.bytes,
bytes.len()
)));
}
let actual_hash = blake3::hash(&bytes).to_hex().to_string();
if actual_hash != manifest_entry.blake3 {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination, manifest_entry.blake3, actual_hash
)));
}
let resource_type = resource_type_for_path(&manifest_entry.destination);
let metadata =
metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries);
let resource_id = resource_id_for_destination(&manifest_entry.destination);
let previous = match self.resources.find_by_id(&resource_id).await {
Ok(resource) => Some(resource),
Err(bat_core::Error::NotFound(_)) => None,
Err(error) => return Err(error),
};
if previous
.as_ref()
.is_some_and(|resource| resource.entry.hash == actual_hash)
{
if let Some(mut resource) = previous {
let should_refresh_metadata = resource.metadata != metadata
|| resource.entry.resource_type != resource_type
|| resource.entry.size != manifest_entry.bytes
|| resource.local_path.as_path() != Path::new(&manifest_entry.destination);
if should_refresh_metadata {
resource.local_path = PathBuf::from(&manifest_entry.destination);
resource.entry.size = manifest_entry.bytes;
resource.entry.resource_type = resource_type;
resource.metadata = metadata;
self.resources.update(resource).await?;
report.metadata_updated_count += 1;
}
}
report.unchanged_count += 1;
count_resource_type(report, resource_type);
return Ok(());
}
let object_id = self.cas.store(&bytes).await?;
let resource = Resource {
id: resource_id,
local_path: PathBuf::from(&manifest_entry.destination),
entry: ResourceEntry {
path: manifest_entry.destination.clone(),
hash: object_id.clone(),
size: manifest_entry.bytes,
resource_type,
address: None,
dependencies: Vec::new(),
crc: None,
},
metadata,
};
if let Err(error) = self.resources.add(resource).await {
let _ = self.cas.remove_reference(&object_id).await;
return Err(error);
}
if let Some(previous) = previous {
if previous.entry.hash != object_id {
match self.cas.remove_reference(&previous.entry.hash).await {
Ok(_) => {}
Err(error) => report.warnings.push(format!(
"旧 CAS 引用清理失败 {}{}",
previous.entry.hash, error
)),
}
}
}
report.imported_count += 1;
count_resource_type(report, resource_type);
Ok(())
}
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn metadata_for_manifest_entry(
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
) -> ResourceMetadata {
let mut metadata = ResourceMetadata {
official_release_id: official_release_id.map(ToOwned::to_owned),
platform: platform_for_destination(&manifest_entry.destination),
bundle_path: Some(manifest_entry.destination.clone()),
..ResourceMetadata::default()
};
let mut archive_entries = BTreeSet::new();
let mut parse_statuses = BTreeSet::new();
let mut unity_versions = BTreeSet::new();
let mut text_assets = BTreeSet::new();
let mut text_unit_formats = BTreeSet::new();
for entry in parse_entries {
if let Some(archive_entry) = entry.archive_entry.as_ref() {
archive_entries.insert(archive_entry.clone());
}
parse_statuses.insert(parse_status_label(entry.status).to_string());
if let Some(unity_version) = entry.unity_version.as_ref() {
unity_versions.insert(unity_version.clone());
}
metadata.unityfs_file_count += entry.file_count as u64;
metadata.serialized_file_count += entry.serialized_file_count as u64;
metadata.text_asset_count += entry.text_asset_count as u64;
metadata.text_unit_count += entry.text_unit_count as u64;
metadata.text_unit_error_count += entry.text_unit_error_count as u64;
text_assets.extend(entry.text_assets.iter().cloned());
text_unit_formats.extend(entry.text_unit_formats.iter().cloned());
}
metadata.archive_entries = archive_entries.into_iter().collect();
metadata.parse_statuses = parse_statuses.into_iter().collect();
metadata.unity_versions = unity_versions.into_iter().collect();
metadata.text_assets = text_assets.into_iter().collect();
metadata.text_unit_formats = text_unit_formats.into_iter().collect();
metadata
}
fn parse_status_label(status: OfficialParseStatus) -> &'static str {
match status {
OfficialParseStatus::Parsed => "parsed",
OfficialParseStatus::SkippedUnsupported => "skipped_unsupported",
OfficialParseStatus::Failed => "failed",
}
}
fn platform_for_destination(destination: &str) -> Option<String> {
let normalized = destination.replace('\\', "/").to_ascii_lowercase();
if normalized.contains("windows") || normalized.contains("/win/") {
Some("windows".to_string())
} else if normalized.contains("android") {
Some("android".to_string())
} else {
None
}
}
fn release_id_from_root(release_root: &Path) -> Option<String> {
release_root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty() && *name != "current")
.map(ToOwned::to_owned)
}
fn resource_id_for_destination(destination: &str) -> String {
format!("official/{}", destination.replace('\\', "/"))
}
fn resource_type_for_path(path: &str) -> ResourceType {
let normalized = path.replace('\\', "/").to_ascii_lowercase();
if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") {
ResourceType::AssetBundle
} else if normalized.contains("tablebundles/") {
ResourceType::TableBundle
} else if normalized.contains("textassets/")
|| matches!(
normalized.rsplit('.').next(),
Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml")
)
{
ResourceType::TextAsset
} else if normalized.contains("mediaresources/")
|| matches!(
normalized.rsplit('.').next(),
Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp")
)
{
ResourceType::Media
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
ResourceType::Manifest
} else {
ResourceType::Other
}
}
fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) {
match resource_type {
ResourceType::AssetBundle => report.asset_bundle_count += 1,
ResourceType::TextAsset => report.text_asset_count += 1,
ResourceType::TableBundle => report.table_count += 1,
ResourceType::Media => report.media_count += 1,
ResourceType::Manifest | ResourceType::Other => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{FileSystemCasRepository, InMemoryResourceRepository};
use crate::{
OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION,
};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use std::collections::BTreeMap;
use tempfile::TempDir;
fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) {
let path = root.join(destination);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, bytes).unwrap();
let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialDownloadManifestEntry {
url: url.clone(),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
};
let manifest = serde_json::json!({
"version": 1,
"entries": BTreeMap::from([(url, entry)]),
});
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
}
fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) {
let source_url =
format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialParseCacheEntry {
key: format!("direct:{source_url}"),
source_url: source_url.clone(),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url,
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: 1,
text_assets: vec!["Scenario".to_string()],
serialized_parse_error_count: 0,
text_unit_count: 2,
text_unit_formats: vec!["json".to_string(), "plain".to_string()],
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
error: None,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 123,
summary: OfficialParseSummary {
manifest_entry_count: 1,
cache_entry_count: 1,
candidate_file_count: 1,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 0,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
},
entries: BTreeMap::from([(entry.key.clone(), entry)]),
};
std::fs::write(
root.join("official-parse-cache.json"),
serde_json::to_vec(&cache).unwrap(),
)
.unwrap();
}
#[tokio::test]
async fn imports_verified_release_idempotently() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog");
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let config = OfficialReleaseImportConfig::new(temp.path());
let first = service.import_release(&config).await.unwrap();
let second = service.import_release(&config).await.unwrap();
assert_eq!(first.imported_count, 1);
assert_eq!(second.imported_count, 0);
assert_eq!(second.unchanged_count, 1);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TableBundle))
.await
.unwrap(),
1
);
let id = resource_id_for_destination("TableBundles/TableCatalog.bytes");
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
cas.get_reference_count(&resource.entry.hash).await.unwrap(),
1
);
}
#[tokio::test]
async fn imports_parse_cache_metadata_into_resource_index() {
let temp = TempDir::new().unwrap();
let destination = "Windows/Bundles/scenario.bundle";
let bytes = b"bundle-bytes";
write_manifest(temp.path(), destination, bytes);
write_parse_cache(temp.path(), destination, bytes);
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let report = service
.import_release(
&OfficialReleaseImportConfig::new(temp.path())
.with_official_release_id("release-current"),
)
.await
.unwrap();
assert_eq!(report.imported_count, 1);
assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2);
let id = resource_id_for_destination(destination);
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
resource.metadata.official_release_id.as_deref(),
Some("release-current")
);
assert_eq!(resource.metadata.platform.as_deref(), Some("windows"));
assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination));
assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]);
assert_eq!(
resource.metadata.unity_versions,
vec!["2021.3.56f2".to_string()]
);
assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(resource.metadata.text_unit_count, 2);
assert_eq!(
resource.metadata.text_unit_formats,
vec!["json".to_string(), "plain".to_string()]
);
assert_eq!(resource.metadata.text_unit_error_count, 1);
}
#[tokio::test]
async fn rejects_manifest_hash_mismatch_before_cas_write() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original");
let path = temp.path().join("TextAssets/dialogue.txt");
std::fs::write(&path, b"tampered").unwrap();
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let error = service
.import_release(&OfficialReleaseImportConfig::new(temp.path()))
.await
.unwrap_err();
assert!(error.to_string().contains("BLAKE3"));
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
}
}
@@ -0,0 +1,834 @@
//! Incremental TextUnit task and Crowdin offline queues for official releases.
//!
//! The queue is derived only from a verified official release, its
//! `official-resource-changes.json`, and its `official-parse-cache.json`.
//! It does not call Crowdin or any network service.
use crate::official_changes::{
read_resource_change_set_at, OfficialResourceChange, OfficialResourceChangeKind,
OfficialResourceChangeSet, TranslationHandoffProvider, TranslationHandoffStatus,
};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceKind,
OfficialParseStatus,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
/// Current TextUnit task-queue schema version.
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE: &str = "official-textunit-tasks.json";
/// Current Crowdin TextUnit queue schema version.
pub const CROWDIN_TEXTUNIT_QUEUE_VERSION: u32 = 1;
/// File name stored under a published official release root.
pub const CROWDIN_TEXTUNIT_QUEUE_FILE: &str = "crowdin-textunit-queue.json";
/// Processing status for one incremental TextUnit task candidate.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialTextUnitTaskStatus {
/// The task has parse metadata and at least one TextUnit.
QueuedOffline,
/// No parse-cache entry exists for this changed resource yet.
SkippedNoParseEntry,
/// The resource parsed successfully, but did not produce TextUnits.
SkippedNoTextUnit,
/// Parse cache says the candidate failed to parse.
SkippedParseFailed,
/// Parse cache says the candidate is unsupported.
SkippedUnsupported,
}
impl OfficialTextUnitTaskStatus {
/// Returns the stable status label.
pub fn as_str(self) -> &'static str {
match self {
Self::QueuedOffline => "queued_offline",
Self::SkippedNoParseEntry => "skipped_no_parse_entry",
Self::SkippedNoTextUnit => "skipped_no_text_unit",
Self::SkippedParseFailed => "skipped_parse_failed",
Self::SkippedUnsupported => "skipped_unsupported",
}
}
}
/// One parse-cache-backed incremental task.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTask {
/// Stable task ID.
pub task_id: String,
/// Current official release ID.
pub official_release_id: String,
/// Relative resource path under the official release root.
pub destination: String,
/// Change kind that caused this task candidate.
pub change_kind: OfficialResourceChangeKind,
/// Current official URL.
pub url: String,
/// Verified byte count.
pub bytes: u64,
/// Verified BLAKE3 digest.
pub blake3: String,
/// Parse-cache entry key, when available.
pub parse_entry_key: Option<String>,
/// ZIP/archive entry containing the parsed bundle, when available.
pub archive_entry: Option<String>,
/// Parse-cache source kind, when available.
pub source_kind: Option<OfficialParseSourceKind>,
/// Parse status, when available.
pub parse_status: Option<OfficialParseStatus>,
/// Number of TextAsset objects in this parse entry.
pub text_asset_count: usize,
/// TextAsset names in this parse entry.
pub text_assets: Vec<String>,
/// Number of TextUnits in this parse entry.
pub text_unit_count: usize,
/// TextUnit format labels.
pub text_unit_formats: Vec<String>,
/// Non-fatal TextUnit extraction diagnostic count.
pub text_unit_error_count: usize,
/// Queue status.
pub status: OfficialTextUnitTaskStatus,
/// Diagnostic reason for skipped tasks.
pub reason: Option<String>,
}
/// Aggregate counters for an incremental TextUnit task queue.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTaskSummary {
/// Number of added/modified resources considered.
pub resource_candidate_count: usize,
/// Number of parse-cache entries inspected for those resources.
pub parse_entry_count: usize,
/// Number of tasks queued for offline translation.
pub queued_task_count: usize,
/// Number of changed resources with no parse-cache entry.
pub skipped_no_parse_entry_count: usize,
/// Number of parse entries with zero TextUnits.
pub skipped_no_text_unit_count: usize,
/// Number of failed parse entries.
pub skipped_parse_failed_count: usize,
/// Number of unsupported parse entries.
pub skipped_unsupported_count: usize,
/// Total queued TextUnit count.
pub text_unit_count: usize,
}
impl OfficialTextUnitTaskSummary {
fn record(&mut self, task: &OfficialTextUnitTask) {
match task.status {
OfficialTextUnitTaskStatus::QueuedOffline => {
self.queued_task_count += 1;
self.text_unit_count += task.text_unit_count;
}
OfficialTextUnitTaskStatus::SkippedNoParseEntry => {
self.skipped_no_parse_entry_count += 1;
}
OfficialTextUnitTaskStatus::SkippedNoTextUnit => {
self.skipped_no_text_unit_count += 1;
}
OfficialTextUnitTaskStatus::SkippedParseFailed => {
self.skipped_parse_failed_count += 1;
}
OfficialTextUnitTaskStatus::SkippedUnsupported => {
self.skipped_unsupported_count += 1;
}
}
}
}
/// Durable incremental TextUnit task queue for one official release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitTaskQueue {
/// Queue schema version.
#[serde(default = "default_textunit_task_queue_version")]
pub queue_version: u32,
/// Current official release ID.
pub official_release_id: String,
/// Previous official release ID, when known.
pub previous_release_id: Option<String>,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Current official release root.
pub current_resource_root: PathBuf,
/// Queue summary.
pub summary: OfficialTextUnitTaskSummary,
/// All task candidates, including skipped diagnostics.
pub tasks: Vec<OfficialTextUnitTask>,
}
impl OfficialTextUnitTaskQueue {
/// Builds a task queue from an official change set and parse cache.
pub fn from_change_set_and_parse_cache(
change_set: &OfficialResourceChangeSet,
parse_cache: &OfficialParseCache,
) -> Self {
let parse_entries = parse_entries_by_destination(parse_cache);
let mut summary = OfficialTextUnitTaskSummary {
resource_candidate_count: change_set.parse_candidates().len(),
..OfficialTextUnitTaskSummary::default()
};
let mut tasks = Vec::new();
for change in change_set.parse_candidates() {
let entries = parse_entries
.get(&change.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
if entries.is_empty() {
let task = skipped_no_parse_entry_task(change_set, change);
summary.record(&task);
tasks.push(task);
continue;
}
summary.parse_entry_count += entries.len();
for entry in entries {
let task = task_from_parse_entry(change_set, change, entry);
summary.record(&task);
tasks.push(task);
}
}
Self {
queue_version: OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
official_release_id: change_set.official_release_id.clone(),
previous_release_id: change_set.previous_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
current_resource_root: change_set.current_resource_root.clone(),
summary,
tasks,
}
}
fn matches_derivation(&self, expected: &Self) -> bool {
self.official_release_id == expected.official_release_id
&& self.previous_release_id == expected.previous_release_id
&& self.current_resource_root == expected.current_resource_root
&& self.summary == expected.summary
&& self.tasks == expected.tasks
}
}
/// One Crowdin offline queue item referencing a TextUnit task.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTextUnitQueueItem {
/// Stable TextUnit task ID.
pub task_id: String,
/// Relative resource path under the official release root.
pub destination: String,
/// ZIP/archive entry, when applicable.
pub archive_entry: Option<String>,
/// Number of queued TextUnits.
pub text_unit_count: usize,
/// TextUnit format labels.
pub text_unit_formats: Vec<String>,
}
/// Crowdin offline queue derived from queued TextUnit tasks.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CrowdinTextUnitQueue {
/// Queue schema version.
#[serde(default = "default_crowdin_textunit_queue_version")]
pub queue_version: u32,
/// Provider reserved for this queue.
pub provider: TranslationHandoffProvider,
/// Queue status.
pub status: TranslationHandoffStatus,
/// Current official release ID.
pub official_release_id: String,
/// Generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Path to the source TextUnit task queue.
pub textunit_task_queue_path: PathBuf,
/// Number of queued task items.
pub task_count: usize,
/// Total queued TextUnit count.
pub text_unit_count: usize,
/// Queued items for the future Crowdin worker.
pub items: Vec<CrowdinTextUnitQueueItem>,
}
impl CrowdinTextUnitQueue {
/// Builds a Crowdin offline queue from queued TextUnit tasks.
pub fn from_textunit_task_queue(
task_queue_path: PathBuf,
task_queue: &OfficialTextUnitTaskQueue,
) -> Self {
let items = task_queue
.tasks
.iter()
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
.map(|task| CrowdinTextUnitQueueItem {
task_id: task.task_id.clone(),
destination: task.destination.clone(),
archive_entry: task.archive_entry.clone(),
text_unit_count: task.text_unit_count,
text_unit_formats: task.text_unit_formats.clone(),
})
.collect::<Vec<_>>();
let text_unit_count = items.iter().map(|item| item.text_unit_count).sum();
Self {
queue_version: CROWDIN_TEXTUNIT_QUEUE_VERSION,
provider: TranslationHandoffProvider::Crowdin,
status: TranslationHandoffStatus::QueuedOffline,
official_release_id: task_queue.official_release_id.clone(),
generated_unix_seconds: unix_seconds_now(),
textunit_task_queue_path: task_queue_path,
task_count: items.len(),
text_unit_count,
items,
}
}
fn matches_derivation(&self, expected: &Self) -> bool {
self.provider == expected.provider
&& self.status == expected.status
&& self.official_release_id == expected.official_release_id
&& self.textunit_task_queue_path == expected.textunit_task_queue_path
&& self.task_count == expected.task_count
&& self.text_unit_count == expected.text_unit_count
&& self.items == expected.items
}
}
/// Paths and summary produced after writing incremental TextUnit queues.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitQueueReport {
/// Path to `official-textunit-tasks.json`.
pub textunit_task_queue_path: PathBuf,
/// Path to `crowdin-textunit-queue.json`.
pub crowdin_textunit_queue_path: PathBuf,
/// Aggregate queue counters.
pub summary: OfficialTextUnitTaskSummary,
}
/// Reads change/parse derived queues from a release root and writes them back.
pub fn write_official_textunit_queues(
resource_root: &Path,
) -> Result<OfficialTextUnitQueueReport, String> {
let change_set = read_resource_change_set_at(resource_root)?.ok_or_else(|| {
format!(
"缺少官方资源变更集,无法生成 TextUnit 队列:{}",
resource_root.display()
)
})?;
let parse_cache = read_parse_cache_at(resource_root)?.ok_or_else(|| {
format!(
"缺少官方解析缓存,无法生成 TextUnit 队列:{}",
resource_root.display()
)
})?;
let task_queue =
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
write_textunit_task_queue_at(resource_root, &task_queue)?;
let task_queue_path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
let crowdin_queue =
CrowdinTextUnitQueue::from_textunit_task_queue(task_queue_path.clone(), &task_queue);
write_crowdin_textunit_queue_at(resource_root, &crowdin_queue)?;
Ok(OfficialTextUnitQueueReport {
textunit_task_queue_path: task_queue_path,
crowdin_textunit_queue_path: resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE),
summary: task_queue.summary,
})
}
/// Reads a generated TextUnit task queue from a release root.
pub fn read_textunit_task_queue_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitTaskQueue>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 任务队列")? else {
return Ok(None);
};
let queue: OfficialTextUnitTaskQueue = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析官方 TextUnit 任务队列失败 {}{error}", path.display()))?;
if queue.queue_version != OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION {
return Err(format!(
"不支持的官方 TextUnit 任务队列版本 {},文件 {}",
queue.queue_version,
path.display()
));
}
Ok(Some(queue))
}
/// Returns whether the persisted TextUnit queue still matches current inputs.
pub fn is_textunit_task_queue_current(
resource_root: &Path,
queue: &OfficialTextUnitTaskQueue,
) -> Result<bool, String> {
let Some(change_set) = read_resource_change_set_at(resource_root)? else {
return Ok(false);
};
let Some(parse_cache) = read_parse_cache_at(resource_root)? else {
return Ok(false);
};
let expected =
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
Ok(queue.matches_derivation(&expected))
}
/// Writes a TextUnit task queue under a release root.
pub fn write_textunit_task_queue_at(
resource_root: &Path,
queue: &OfficialTextUnitTaskQueue,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 任务队列")?;
let bytes = serde_json::to_vec_pretty(queue)
.map_err(|error| format!("序列化官方 TextUnit 任务队列失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 任务队列")
}
/// Reads a generated Crowdin offline TextUnit queue from a release root.
pub fn read_crowdin_textunit_queue_at(
resource_root: &Path,
) -> Result<Option<CrowdinTextUnitQueue>, String> {
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
let Some(bytes) = read_file_no_symlink(&path, "Crowdin TextUnit 离线队列")? else {
return Ok(None);
};
let queue: CrowdinTextUnitQueue = serde_json::from_slice(&bytes).map_err(|error| {
format!(
"解析 Crowdin TextUnit 离线队列失败 {}{error}",
path.display()
)
})?;
if queue.queue_version != CROWDIN_TEXTUNIT_QUEUE_VERSION {
return Err(format!(
"不支持的 Crowdin TextUnit 离线队列版本 {},文件 {}",
queue.queue_version,
path.display()
));
}
Ok(Some(queue))
}
/// Returns whether the persisted Crowdin queue still matches a TextUnit queue.
pub fn is_crowdin_textunit_queue_current(
resource_root: &Path,
task_queue: &OfficialTextUnitTaskQueue,
) -> Result<bool, String> {
let Some(queue) = read_crowdin_textunit_queue_at(resource_root)? else {
return Ok(false);
};
let expected = CrowdinTextUnitQueue::from_textunit_task_queue(
resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
task_queue,
);
Ok(queue.matches_derivation(&expected))
}
/// Writes a Crowdin offline TextUnit queue under a release root.
pub fn write_crowdin_textunit_queue_at(
resource_root: &Path,
queue: &CrowdinTextUnitQueue,
) -> Result<(), String> {
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "Crowdin TextUnit 离线队列")?;
let bytes = serde_json::to_vec_pretty(queue)
.map_err(|error| format!("序列化 Crowdin TextUnit 离线队列失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin TextUnit 离线队列")
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn skipped_no_parse_entry_task(
change_set: &OfficialResourceChangeSet,
change: &OfficialResourceChange,
) -> OfficialTextUnitTask {
let current = change
.current
.as_ref()
.expect("parse candidate has current");
OfficialTextUnitTask {
task_id: task_id_for(&change_set.official_release_id, &change.destination, None),
official_release_id: change_set.official_release_id.clone(),
destination: change.destination.clone(),
change_kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
parse_entry_key: None,
archive_entry: None,
source_kind: None,
parse_status: None,
text_asset_count: 0,
text_assets: Vec::new(),
text_unit_count: 0,
text_unit_formats: Vec::new(),
text_unit_error_count: 0,
status: OfficialTextUnitTaskStatus::SkippedNoParseEntry,
reason: Some("parse cache entry not found for changed resource".to_string()),
}
}
fn task_from_parse_entry(
change_set: &OfficialResourceChangeSet,
change: &OfficialResourceChange,
entry: &OfficialParseCacheEntry,
) -> OfficialTextUnitTask {
let current = change
.current
.as_ref()
.expect("parse candidate has current");
let (status, reason) = status_for_parse_entry(entry);
OfficialTextUnitTask {
task_id: task_id_for(
&change_set.official_release_id,
&change.destination,
entry.archive_entry.as_deref(),
),
official_release_id: change_set.official_release_id.clone(),
destination: change.destination.clone(),
change_kind: change.kind,
url: current.url.clone(),
bytes: current.bytes,
blake3: current.blake3.clone(),
parse_entry_key: Some(entry.key.clone()),
archive_entry: entry.archive_entry.clone(),
source_kind: Some(entry.source_kind),
parse_status: Some(entry.status),
text_asset_count: entry.text_asset_count,
text_assets: entry.text_assets.clone(),
text_unit_count: entry.text_unit_count,
text_unit_formats: entry.text_unit_formats.clone(),
text_unit_error_count: entry.text_unit_error_count,
status,
reason,
}
}
fn status_for_parse_entry(
entry: &OfficialParseCacheEntry,
) -> (OfficialTextUnitTaskStatus, Option<String>) {
match entry.status {
OfficialParseStatus::Parsed if entry.text_unit_count > 0 => {
(OfficialTextUnitTaskStatus::QueuedOffline, None)
}
OfficialParseStatus::Parsed => (
OfficialTextUnitTaskStatus::SkippedNoTextUnit,
Some("parsed resource has no TextUnit".to_string()),
),
OfficialParseStatus::Failed => (
OfficialTextUnitTaskStatus::SkippedParseFailed,
entry.error.clone(),
),
OfficialParseStatus::SkippedUnsupported => (
OfficialTextUnitTaskStatus::SkippedUnsupported,
entry.error.clone(),
),
}
}
fn task_id_for(
official_release_id: &str,
destination: &str,
archive_entry: Option<&str>,
) -> String {
match archive_entry {
Some(entry) => format!("textunit/{official_release_id}/{destination}#{entry}"),
None => format!("textunit/{official_release_id}/{destination}"),
}
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn default_textunit_task_queue_version() -> u32 {
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION
}
fn default_crowdin_textunit_queue_version() -> u32 {
CROWDIN_TEXTUNIT_QUEUE_VERSION
}
#[cfg(test)]
mod tests {
use super::*;
use crate::official_changes::{
OfficialResourceChangeSet, OfficialResourceDescriptor, OFFICIAL_RESOURCE_CHANGES_VERSION,
};
use crate::official_parse::{
OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceFingerprint,
OfficialParseSourceKind, OfficialParseSummary, OFFICIAL_PARSE_CACHE_VERSION,
};
fn change_set(root: &Path) -> OfficialResourceChangeSet {
OfficialResourceChangeSet {
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
official_release_id: "release-new".to_string(),
previous_release_id: Some("release-old".to_string()),
generated_unix_seconds: 123,
previous_resource_root: None,
current_resource_root: root.to_path_buf(),
summary: crate::OfficialResourceChangeSummary {
previous_manifest_present: true,
previous_manifest_entry_count: 2,
current_manifest_entry_count: 3,
added_count: 2,
modified_count: 1,
removed_count: 1,
parse_candidate_count: 3,
translation_candidate_count: 3,
},
changes: vec![
change("Bundles/a.bundle", OfficialResourceChangeKind::Added, b"a"),
change(
"Bundles/b.bundle",
OfficialResourceChangeKind::Modified,
b"b",
),
change("Bundles/c.bundle", OfficialResourceChangeKind::Added, b"c"),
crate::OfficialResourceChange {
destination: "Bundles/removed.bundle".to_string(),
kind: OfficialResourceChangeKind::Removed,
previous: Some(OfficialResourceDescriptor {
url: "https://old/removed".to_string(),
destination: "Bundles/removed.bundle".to_string(),
bytes: 1,
blake3: "old".to_string(),
}),
current: None,
parse_candidate: false,
translation_candidate: false,
},
],
}
}
fn change(
destination: &str,
kind: OfficialResourceChangeKind,
bytes: &[u8],
) -> crate::OfficialResourceChange {
crate::OfficialResourceChange {
destination: destination.to_string(),
kind,
previous: None,
current: Some(OfficialResourceDescriptor {
url: format!("https://new/{destination}"),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
}),
parse_candidate: true,
translation_candidate: true,
}
}
fn parse_cache() -> OfficialParseCache {
OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 124,
summary: OfficialParseSummary {
manifest_entry_count: 3,
cache_entry_count: 2,
candidate_file_count: 2,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 1,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
},
entries: BTreeMap::from([
(
"direct:a".to_string(),
parse_entry(
"direct:a",
"Bundles/a.bundle",
OfficialParseStatus::Parsed,
2,
),
),
(
"direct:b".to_string(),
parse_entry(
"direct:b",
"Bundles/b.bundle",
OfficialParseStatus::SkippedUnsupported,
0,
),
),
]),
}
}
fn parse_entry(
key: &str,
destination: &str,
status: OfficialParseStatus,
text_unit_count: usize,
) -> OfficialParseCacheEntry {
OfficialParseCacheEntry {
key: key.to_string(),
source_url: format!("https://new/{destination}"),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url: format!("https://new/{destination}"),
destination: destination.to_string(),
bytes: 1,
blake3: "hash".to_string(),
},
status,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: usize::from(text_unit_count > 0),
text_assets: if text_unit_count > 0 {
vec!["Scenario".to_string()]
} else {
Vec::new()
},
serialized_parse_error_count: 0,
text_unit_count,
text_unit_formats: if text_unit_count > 0 {
vec!["plain".to_string()]
} else {
Vec::new()
},
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: (status != OfficialParseStatus::Parsed).then(|| "unsupported".to_string()),
}
}
#[test]
fn textunit_queue_uses_only_added_and_modified_parse_candidates() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
assert_eq!(queue.summary.resource_candidate_count, 3);
assert_eq!(queue.summary.parse_entry_count, 2);
assert_eq!(queue.summary.queued_task_count, 1);
assert_eq!(queue.summary.skipped_unsupported_count, 1);
assert_eq!(queue.summary.skipped_no_parse_entry_count, 1);
assert_eq!(queue.summary.text_unit_count, 2);
assert_eq!(
queue
.tasks
.iter()
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
.map(|task| task.destination.as_str())
.collect::<Vec<_>>(),
vec!["Bundles/a.bundle"]
);
}
#[test]
fn crowdin_queue_contains_only_queued_textunit_tasks() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
&queue,
);
assert_eq!(crowdin.provider, TranslationHandoffProvider::Crowdin);
assert_eq!(crowdin.status, TranslationHandoffStatus::QueuedOffline);
assert_eq!(crowdin.task_count, 1);
assert_eq!(crowdin.text_unit_count, 2);
assert_eq!(crowdin.items[0].destination, "Bundles/a.bundle");
}
#[test]
fn write_textunit_queues_persists_files() {
let temp = tempfile::TempDir::new().unwrap();
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
let report = write_official_textunit_queues(temp.path()).unwrap();
assert_eq!(report.summary.queued_task_count, 1);
assert!(report.textunit_task_queue_path.is_file());
assert!(report.crowdin_textunit_queue_path.is_file());
let persisted = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
assert_eq!(persisted.summary.text_unit_count, 2);
}
#[test]
fn textunit_queue_current_detects_parse_cache_changes() {
let temp = tempfile::TempDir::new().unwrap();
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
write_official_textunit_queues(temp.path()).unwrap();
let queue = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
assert!(is_textunit_task_queue_current(temp.path(), &queue).unwrap());
let mut changed_cache = parse_cache();
changed_cache.entries.insert(
"direct:c".to_string(),
parse_entry(
"direct:c",
"Bundles/c.bundle",
OfficialParseStatus::Parsed,
1,
),
);
crate::write_parse_cache_at(temp.path(), &changed_cache).unwrap();
assert!(!is_textunit_task_queue_current(temp.path(), &queue).unwrap());
}
#[test]
fn crowdin_queue_current_requires_matching_file() {
let temp = tempfile::TempDir::new().unwrap();
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
&change_set(temp.path()),
&parse_cache(),
);
write_textunit_task_queue_at(temp.path(), &queue).unwrap();
assert!(!is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
&queue,
);
write_crowdin_textunit_queue_at(temp.path(), &crowdin).unwrap();
assert!(is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
}
}
File diff suppressed because it is too large Load Diff
+424
View File
@@ -0,0 +1,424 @@
//! File-level Patch and UnityFS write operations.
use std::path::{Path, PathBuf};
use bat_assetbundle::{
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
StringFieldPatch, TextAssetPatch, UnitySerializedReplacementValue,
};
use serde::{Deserialize, Serialize};
use crate::path_security::{
ensure_safe_file_target, lexical_absolute, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
/// File patch algorithm selected by `patch.apply`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PatchApplyKind {
/// Deterministic binary hunk patch.
Binary,
/// RFC 6902 JSON Patch.
Json,
/// UTF-8 Text Patch.
Text,
}
impl PatchApplyKind {
/// Stable RPC/CLI label.
pub fn as_str(self) -> &'static str {
match self {
Self::Binary => "binary",
Self::Json => "json",
Self::Text => "text",
}
}
}
/// Parameters for applying one file-level patch.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PatchApplyParams {
/// Patch algorithm.
pub kind: PatchApplyKind,
/// Source file path.
pub source_path: PathBuf,
/// Patch document path.
pub patch_path: PathBuf,
/// Target file path written atomically.
pub target_path: PathBuf,
}
/// Parameters for patching one UnityFS TextAsset object.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsTextAssetPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Replacement raw bytes path.
pub replacement_path: PathBuf,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected TextAsset name.
#[serde(default)]
pub expected_name: Option<String>,
}
/// Parameters for patching one TypeTree string field inside a UnityFS bundle.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsStringFieldPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Stable TypeTree field path.
pub field_path: String,
/// Replacement string supplied inline.
#[serde(default)]
pub replacement_text: Option<String>,
/// Replacement UTF-8 file path.
#[serde(default)]
pub replacement_path: Option<PathBuf>,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected source string.
#[serde(default)]
pub expected_value: Option<String>,
}
/// Parameters for patching one semantic TypeTree field inside a UnityFS bundle.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnityFsFieldPatchParams {
/// Source UnityFS bundle file.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Stable TypeTree field path.
pub field_path: String,
/// Replacement value encoded according to the current TypeTree field type.
pub replacement: UnitySerializedReplacementValue,
/// Target bundle file path written atomically.
pub target_path: PathBuf,
/// Optional expected source value.
#[serde(default)]
pub expected_value: Option<UnitySerializedReplacementValue>,
}
/// Result of a file-level patch operation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct PatchApplyReport {
/// RPC/CLI command name.
pub command: &'static str,
/// Operation status.
pub status: &'static str,
/// Human-readable summary.
pub message: &'static str,
/// Patch algorithm.
pub kind: PatchApplyKind,
/// Absolute source path.
pub source_path: PathBuf,
/// Absolute patch path.
pub patch_path: PathBuf,
/// Absolute target path.
pub target_path: PathBuf,
/// Source byte length.
pub source_size: u64,
/// Patch document byte length.
pub patch_size: u64,
/// Target byte length.
pub target_size: u64,
/// Source BLAKE3 hash.
pub source_blake3: String,
/// Patch document BLAKE3 hash.
pub patch_blake3: String,
/// Target BLAKE3 hash.
pub target_blake3: String,
}
/// Result of a UnityFS write operation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct UnityFsPatchReport {
/// RPC/CLI command name.
pub command: &'static str,
/// Operation status.
pub status: &'static str,
/// Human-readable summary.
pub message: &'static str,
/// Absolute source UnityFS bundle path.
pub bundle_path: PathBuf,
/// Serialized file path inside the UnityFS directory table.
pub serialized_file_path: String,
/// Unity object path ID.
pub path_id: i64,
/// Optional TypeTree field path for string-field patches.
#[serde(skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Absolute target bundle path.
pub target_path: PathBuf,
/// Source bundle byte length.
pub source_size: u64,
/// Replacement byte length.
pub replacement_size: u64,
/// Target bundle byte length.
pub target_size: u64,
/// Source bundle BLAKE3 hash.
pub source_blake3: String,
/// Replacement BLAKE3 hash.
pub replacement_blake3: String,
/// Target bundle BLAKE3 hash.
pub target_blake3: String,
}
/// Applies a Binary/JSON/Text patch document to one source file.
pub fn apply_patch_file(params: &PatchApplyParams) -> anyhow::Result<PatchApplyReport> {
let source_path = lexical_absolute(&params.source_path).map_err(anyhow::Error::msg)?;
let patch_path = lexical_absolute(&params.patch_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&source_path, &patch_path])?;
let source = read_required_file(&source_path, "patch source")?;
let patch = read_required_file(&patch_path, "patch document")?;
let target = match params.kind {
PatchApplyKind::Binary => bat_patch::binary::apply_patch(&source, &patch)?,
PatchApplyKind::Json => {
let source = std::str::from_utf8(&source)
.map_err(|error| anyhow::anyhow!("JSON patch source 不是 UTF-8{error}"))?;
let patch = std::str::from_utf8(&patch)
.map_err(|error| anyhow::anyhow!("JSON patch document 不是 UTF-8{error}"))?;
bat_patch::json::apply_json_patch(source, patch)?.into_bytes()
}
PatchApplyKind::Text => bat_patch::text::apply_patch_bytes(&source, &patch)?,
};
write_output_file(&target_path, &target, "patch target")?;
Ok(PatchApplyReport {
command: "patch.apply",
status: "patched",
message: "patch 已应用并原子写入目标文件",
kind: params.kind,
source_path,
patch_path,
target_path,
source_size: source.len() as u64,
patch_size: patch.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&source),
patch_blake3: blake3_hex(&patch),
target_blake3: blake3_hex(&target),
})
}
/// Patches one UnityFS TextAsset and writes the rebuilt bundle atomically.
pub fn apply_unityfs_text_asset_patch_file(
params: &UnityFsTextAssetPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let replacement_path =
lexical_absolute(&params.replacement_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&bundle_path, &replacement_path])?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let replacement = read_required_file(&replacement_path, "TextAsset replacement")?;
let mut patch = TextAssetPatch::new(
params.serialized_file_path.clone(),
params.path_id,
replacement.clone(),
);
patch.expected_name = params.expected_name.clone();
let target = patch_unityfs_text_asset(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_text_asset",
status: "patched",
message: "UnityFS TextAsset patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: None,
target_path,
source_size: bundle.len() as u64,
replacement_size: replacement.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(&replacement),
target_blake3: blake3_hex(&target),
})
}
/// Patches one TypeTree string field and writes the rebuilt UnityFS bundle atomically.
pub fn apply_unityfs_string_field_patch_file(
params: &UnityFsStringFieldPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
let replacement = replacement_text(params)?;
let extra_inputs = params
.replacement_path
.as_ref()
.map(|path| lexical_absolute(path).map_err(anyhow::Error::msg))
.transpose()?;
let mut inputs = vec![bundle_path.as_path()];
if let Some(path) = extra_inputs.as_ref() {
inputs.push(path.as_path());
}
ensure_target_is_not_input(&target_path, &inputs)?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let patch = StringFieldPatch {
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: params.field_path.clone(),
expected_value: params.expected_value.clone(),
replacement: replacement.clone(),
};
let target = patch_unityfs_string_field(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_string_field",
status: "patched",
message: "UnityFS TypeTree string field patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: Some(params.field_path.clone()),
target_path,
source_size: bundle.len() as u64,
replacement_size: replacement.len() as u64,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(replacement.as_bytes()),
target_blake3: blake3_hex(&target),
})
}
/// Patches one semantic TypeTree field and writes the rebuilt UnityFS bundle atomically.
pub fn apply_unityfs_field_patch_file(
params: &UnityFsFieldPatchParams,
) -> anyhow::Result<UnityFsPatchReport> {
let bundle_path = lexical_absolute(&params.bundle_path).map_err(anyhow::Error::msg)?;
let target_path = lexical_absolute(&params.target_path).map_err(anyhow::Error::msg)?;
ensure_target_is_not_input(&target_path, &[&bundle_path])?;
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
let replacement_size = serde_json::to_vec(&params.replacement)
.map_err(anyhow::Error::from)?
.len() as u64;
let patch = FieldPatch {
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: params.field_path.clone(),
expected_value: params.expected_value.clone(),
replacement: params.replacement.clone(),
};
let target = patch_unityfs_field(&bundle, &patch)?;
write_output_file(&target_path, &target, "UnityFS target")?;
Ok(UnityFsPatchReport {
command: "unityfs.patch_field",
status: "patched",
message: "UnityFS TypeTree field patch 已应用并原子写入目标 bundle",
bundle_path,
serialized_file_path: params.serialized_file_path.clone(),
path_id: params.path_id,
field_path: Some(params.field_path.clone()),
target_path,
source_size: bundle.len() as u64,
replacement_size,
target_size: target.len() as u64,
source_blake3: blake3_hex(&bundle),
replacement_blake3: blake3_hex(&serde_json::to_vec(&params.replacement)?),
target_blake3: blake3_hex(&target),
})
}
fn replacement_text(params: &UnityFsStringFieldPatchParams) -> anyhow::Result<String> {
match (&params.replacement_text, &params.replacement_path) {
(Some(_), Some(_)) => Err(anyhow::anyhow!(
"replacement_text 和 replacement_path 只能指定一个"
)),
(Some(text), None) => Ok(text.clone()),
(None, Some(path)) => {
let path = lexical_absolute(path).map_err(anyhow::Error::msg)?;
let bytes = read_required_file(&path, "string replacement")?;
String::from_utf8(bytes)
.map_err(|error| anyhow::anyhow!("string replacement 不是 UTF-8{error}"))
}
(None, None) => Err(anyhow::anyhow!(
"必须指定 replacement_text 或 replacement_path"
)),
}
}
fn read_required_file(path: &Path, label: &str) -> anyhow::Result<Vec<u8>> {
read_file_no_symlink(path, label)
.map_err(anyhow::Error::msg)?
.ok_or_else(|| anyhow::anyhow!("{label} 不存在:{}", path.display()))
}
fn write_output_file(path: &Path, bytes: &[u8], label: &str) -> anyhow::Result<()> {
let parent = path
.parent()
.ok_or_else(|| anyhow::anyhow!("{label} 缺少父目录:{}", path.display()))?;
ensure_safe_file_target(parent, path, label).map_err(anyhow::Error::msg)?;
write_file_atomic(path, bytes, STATE_FILE_MODE, label).map_err(anyhow::Error::msg)
}
fn ensure_target_is_not_input(target: &Path, inputs: &[&Path]) -> anyhow::Result<()> {
for input in inputs {
if target == *input {
return Err(anyhow::anyhow!(
"target_path 不能与输入文件相同:{}",
target.display()
));
}
}
Ok(())
}
fn blake3_hex(bytes: &[u8]) -> String {
blake3::hash(bytes).to_hex().to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn patch_apply_rejects_in_place_target() {
let path = PathBuf::from("/tmp/source.bin");
let params = PatchApplyParams {
kind: PatchApplyKind::Binary,
source_path: path.clone(),
patch_path: PathBuf::from("/tmp/patch.json"),
target_path: path,
};
let error = apply_patch_file(&params).unwrap_err().to_string();
assert!(error.contains("target_path 不能与输入文件相同"));
}
#[test]
fn string_field_params_require_one_replacement_source() {
let params = UnityFsStringFieldPatchParams {
bundle_path: PathBuf::from("/tmp/source.bundle"),
serialized_file_path: "CAB".to_string(),
path_id: 1,
field_path: "message".to_string(),
replacement_text: Some("a".to_string()),
replacement_path: Some(PathBuf::from("/tmp/replacement.txt")),
target_path: PathBuf::from("/tmp/target.bundle"),
expected_value: None,
};
let error = replacement_text(&params).unwrap_err().to_string();
assert!(error.contains("只能指定一个"));
}
}
+59 -11
View File
@@ -1,7 +1,7 @@
//! 内存资源仓储实现。
use async_trait::async_trait;
use bat_core::domain::{Resource, ResourceEntry, ResourceType};
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult};
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
@@ -130,7 +130,8 @@ impl SqliteResourceRepository {
local_path TEXT NOT NULL,
address TEXT,
dependencies_json TEXT NOT NULL DEFAULT '[]',
crc INTEGER
crc INTEGER,
metadata_json TEXT NOT NULL DEFAULT '{}'
)
"#,
),
@@ -140,6 +141,13 @@ impl SqliteResourceRepository {
// 向后兼容:早于 crc 列的旧库缺少该列,按需补加(新建库已含该列,
// pragma 检查后不会重复 ALTER)。
Self::ensure_column(&self.pool, "resources", "crc", "INTEGER").await?;
Self::ensure_column(
&self.pool,
"resources",
"metadata_json",
"TEXT NOT NULL DEFAULT '{}'",
)
.await?;
Self::execute_query(
&self.pool,
@@ -250,9 +258,32 @@ impl SqliteResourceRepository {
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
}
fn metadata_to_json(metadata: &ResourceMetadata) -> bat_core::Result<String> {
serde_json::to_string(metadata)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
}
fn metadata_from_json(value: &str) -> bat_core::Result<ResourceMetadata> {
if value.trim().is_empty() {
return Ok(ResourceMetadata::default());
}
serde_json::from_str(value)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
}
fn resource_from_row(row: ResourceRow) -> bat_core::Result<Resource> {
let (id, path, hash, size, resource_type, local_path, address, dependencies_json, crc) =
row;
let (
id,
path,
hash,
size,
resource_type,
local_path,
address,
dependencies_json,
crc,
metadata_json,
) = row;
Ok(Resource {
id,
local_path: PathBuf::from(local_path),
@@ -265,6 +296,7 @@ impl SqliteResourceRepository {
dependencies: Self::dependencies_from_json(&dependencies_json)?,
crc: crc.and_then(|value| u32::try_from(value).ok()),
},
metadata: Self::metadata_from_json(&metadata_json)?,
})
}
@@ -303,7 +335,7 @@ impl SqliteResourceRepository {
limit: Option<usize>,
) -> bat_core::Result<Vec<Resource>> {
let mut builder = QueryBuilder::<Sqlite>::new(
"SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc FROM resources",
"SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json FROM resources",
);
Self::apply_filters(&mut builder, query)?;
builder.push(" ORDER BY id");
@@ -337,14 +369,15 @@ impl SqliteResourceRepository {
impl ResourceRepository for SqliteResourceRepository {
async fn add(&self, resource: Resource) -> bat_core::Result<String> {
let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?;
let metadata = Self::metadata_to_json(&resource.metadata)?;
Self::execute_query(
&self.pool,
sqlx::query(
r#"
INSERT INTO resources (
id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
ON CONFLICT(id) DO UPDATE SET
path = excluded.path,
hash = excluded.hash,
@@ -353,7 +386,8 @@ impl ResourceRepository for SqliteResourceRepository {
local_path = excluded.local_path,
address = excluded.address,
dependencies_json = excluded.dependencies_json,
crc = excluded.crc
crc = excluded.crc,
metadata_json = excluded.metadata_json
"#,
)
.bind(resource.id.clone())
@@ -364,7 +398,8 @@ impl ResourceRepository for SqliteResourceRepository {
.bind(resource.local_path.to_string_lossy().to_string())
.bind(resource.entry.address.clone())
.bind(dependencies)
.bind(resource.entry.crc.map(i64::from)),
.bind(resource.entry.crc.map(i64::from))
.bind(metadata),
)
.await?;
@@ -374,7 +409,7 @@ impl ResourceRepository for SqliteResourceRepository {
async fn find_by_id(&self, id: &str) -> bat_core::Result<Resource> {
let row: Option<ResourceRow> = sqlx::query_as(
r#"
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
FROM resources
WHERE id = ?1
"#,
@@ -392,7 +427,7 @@ impl ResourceRepository for SqliteResourceRepository {
async fn find_by_hash(&self, hash: &str) -> bat_core::Result<Resource> {
let row: Option<ResourceRow> = sqlx::query_as(
r#"
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
FROM resources
WHERE hash = ?1
ORDER BY id
@@ -455,6 +490,7 @@ type ResourceRow = (
Option<String>,
String,
Option<i64>,
String,
);
fn glob_to_like(pattern: &str) -> String {
@@ -544,6 +580,7 @@ mod tests {
dependencies: Vec::new(),
crc: None,
},
metadata: ResourceMetadata::default(),
}
}
@@ -638,6 +675,10 @@ mod tests {
.entry
.dependencies
.push("assets/shared.bundle".to_string());
resource.metadata.official_release_id = Some("release-1".to_string());
resource.metadata.platform = Some("windows".to_string());
resource.metadata.text_assets = vec!["Scenario".to_string()];
resource.metadata.text_unit_count = 3;
repository.add(resource.clone()).await.unwrap();
@@ -647,6 +688,13 @@ mod tests {
by_id.entry.dependencies,
vec!["assets/shared.bundle".to_string()]
);
assert_eq!(
by_id.metadata.official_release_id.as_deref(),
Some("release-1")
);
assert_eq!(by_id.metadata.platform.as_deref(), Some("windows"));
assert_eq!(by_id.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(by_id.metadata.text_unit_count, 3);
assert_eq!(
repository.find_by_hash("hash-sqlite-a").await.unwrap().id,
resource.id
@@ -367,14 +367,16 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
let curl_failure_state = harness.temp.path().join("curl-failure.state");
write_executable(
&harness.curl_script,
&official_curl_script(
&harness.curl_log,
&zip_fixture,
&resources_assets_path,
fs::metadata(&resources_assets_path).unwrap().len() as usize,
TEST_LAUNCHER_MANIFEST_SOURCE,
Some(&curl_failure_state),
),
&official_curl_script(OfficialCurlScriptFixture {
curl_log: &harness.curl_log,
zip_fixture: &zip_fixture,
resources_assets_fixture: &resources_assets_path,
resources_assets_size: fs::metadata(&resources_assets_path).unwrap().len() as usize,
manifest_source: TEST_LAUNCHER_MANIFEST_SOURCE,
failure_state: Some(&curl_failure_state),
clientpatch_unavailable: false,
seed_unavailable_only: false,
}),
);
let config = harness.sync_config("failed-staging-output");
@@ -399,6 +401,77 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
assert!(version_state.current_completed_version.is_some());
}
#[test]
fn official_update_waits_when_client_patch_markers_are_not_ready() {
let harness = TestHarness::new_with_clientpatch_unavailable(false);
let config = harness.sync_config("clientpatch-marker-wait-output");
let mut events = Vec::<OfficialUpdateProgress>::new();
let report = OfficialUpdateService::new()
.run_with_progress(&config, |event| events.push(event))
.unwrap();
assert_eq!(
report.update_status,
OfficialUpdateStatus::WaitingForOfficialResources
);
assert!(report.waiting_for_official_resources);
assert!(!report.should_download);
assert!(report
.unavailable_endpoints
.iter()
.any(|endpoint| endpoint.url.ends_with("/TableBundles/TableCatalog.hash")));
assert!(report
.unavailable_endpoints
.iter()
.all(|endpoint| endpoint.error_kind == "http_forbidden"));
assert!(report.staging_path.is_none());
assert!(report.published_version_path.is_none());
assert!(!config.output_root.join("current").exists());
assert!(!config.output_root.join(".staging").exists());
assert!(read_version_state(&report.version_state_path)
.unwrap()
.is_none());
assert!(events
.iter()
.any(|event| event.stage == "upstream" && event.message.contains("尚未开放")));
}
#[test]
fn official_update_waits_when_required_seed_catalog_is_not_ready() {
let harness = TestHarness::new_with_clientpatch_unavailable(true);
let config = harness.sync_config("clientpatch-seed-wait-output");
let mut events = Vec::<OfficialUpdateProgress>::new();
let report = OfficialUpdateService::new()
.run_with_progress(&config, |event| events.push(event))
.unwrap();
assert_eq!(
report.update_status,
OfficialUpdateStatus::WaitingForOfficialResources
);
assert!(report.waiting_for_official_resources);
assert_eq!(report.unavailable_endpoints.len(), 1);
assert!(report.unavailable_endpoints[0]
.url
.ends_with("/TableBundles/TableCatalog.bytes"));
assert_eq!(report.unavailable_endpoints[0].http_status, Some(403));
assert!(report.staging_path.is_none());
assert!(report.published_version_path.is_none());
assert!(!config.output_root.join("current").exists());
assert!(!config.output_root.join(".staging").exists());
assert!(read_version_state(&report.version_state_path)
.unwrap()
.is_none());
assert!(events
.iter()
.any(|event| event.stage == "catalog" && event.message.contains("TableCatalog.bytes")));
assert!(events
.iter()
.any(|event| event.stage == "upstream" && event.message.contains("必需资源")));
}
struct TestHarness {
temp: TempDir,
curl_script: std::path::PathBuf,
@@ -412,6 +485,18 @@ impl TestHarness {
}
fn new_with_manifest_source(manifest_source: &str) -> Self {
Self::new_with_options(manifest_source, false, false)
}
fn new_with_clientpatch_unavailable(seed_only: bool) -> Self {
Self::new_with_options(TEST_LAUNCHER_MANIFEST_SOURCE, true, seed_only)
}
fn new_with_options(
manifest_source: &str,
clientpatch_unavailable: bool,
seed_unavailable_only: bool,
) -> Self {
let temp = TempDir::new().unwrap();
let resources_assets = synthetic_resources_assets();
@@ -425,14 +510,16 @@ impl TestHarness {
let curl_script = temp.path().join("fake-curl");
write_executable(
&curl_script,
&official_curl_script(
&curl_log,
&zip_fixture,
&resources_assets_path,
resources_assets.len(),
&official_curl_script(OfficialCurlScriptFixture {
curl_log: &curl_log,
zip_fixture: &zip_fixture,
resources_assets_fixture: &resources_assets_path,
resources_assets_size: resources_assets.len(),
manifest_source,
None,
),
failure_state: None,
clientpatch_unavailable,
seed_unavailable_only,
}),
);
let unzip_script = temp.path().join("fake-unzip");
@@ -542,15 +629,20 @@ fn one_file_zip(name: &[u8], data: &[u8]) -> Vec<u8> {
bytes
}
fn official_curl_script(
curl_log: &Path,
zip_fixture: &Path,
resources_assets_fixture: &Path,
struct OfficialCurlScriptFixture<'a> {
curl_log: &'a Path,
zip_fixture: &'a Path,
resources_assets_fixture: &'a Path,
resources_assets_size: usize,
manifest_source: &str,
failure_state: Option<&Path>,
) -> String {
let failure_state = failure_state
manifest_source: &'a str,
failure_state: Option<&'a Path>,
clientpatch_unavailable: bool,
seed_unavailable_only: bool,
}
fn official_curl_script(fixture: OfficialCurlScriptFixture<'_>) -> String {
let failure_state = fixture
.failure_state
.map(shell_quote)
.unwrap_or_else(|| "''".to_string());
format!(
@@ -614,6 +706,31 @@ maybe_fail_once() {{
fi
}}
clientpatch_is_unavailable() {{
if [[ "{clientpatch_unavailable}" != "true" ]]; then
return 1
fi
if [[ "$url" != "{addressables_root}/"* ]]; then
return 1
fi
if [[ "{seed_unavailable_only}" == "true" ]]; then
case "$url" in
*/TableBundles/TableCatalog.bytes|*/BundlePackingInfo.bytes|*/Catalog/MediaCatalog.bytes)
return 0
;;
*)
return 1
;;
esac
fi
return 0
}}
if clientpatch_is_unavailable; then
echo "curl: (22) The requested URL returned error: 403" >&2
exit 22
fi
if [[ "$url" == "https://api-launcher-jp.yo-star.com/api/launcher/game/config" ]]; then
cat <<'JSON'
{{"code":200,"message":"ok","data":{{"game_latest_version":"{launcher_latest_version}","game_latest_file_path":"{launcher_latest_file_path}"}}}}
@@ -679,16 +796,16 @@ else
exit 1
fi
"#,
shell_quote(curl_log),
shell_quote(zip_fixture),
shell_quote(resources_assets_fixture),
shell_quote(fixture.curl_log),
shell_quote(fixture.zip_fixture),
shell_quote(fixture.resources_assets_fixture),
launcher_latest_version = TEST_LAUNCHER_LATEST_VERSION,
launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH,
launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL,
launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL,
launcher_manifest_source = manifest_source,
launcher_manifest_source = fixture.manifest_source,
launcher_resources_assets_url = TEST_LAUNCHER_RESOURCES_ASSETS_URL,
resources_assets_size = resources_assets_size,
resources_assets_size = fixture.resources_assets_size,
server_info_url = TEST_SERVER_INFO_URL,
connection_group = TEST_CONNECTION_GROUP,
addressables_root = TEST_ADDRESSABLES_ROOT,
@@ -698,6 +815,8 @@ fi
android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"),
android_media_catalog_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"),
failure_state = failure_state,
clientpatch_unavailable = fixture.clientpatch_unavailable,
seed_unavailable_only = fixture.seed_unavailable_only,
)
}