mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 07:24:55 +08:00
feat(sync): 接入解析缓存与汉化发布前置
补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。 同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。 验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
Generated
+2
@@ -88,6 +88,7 @@ dependencies = [
|
|||||||
"hex",
|
"hex",
|
||||||
"lz4",
|
"lz4",
|
||||||
"lzma-rs",
|
"lzma-rs",
|
||||||
|
"md-5",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
@@ -145,6 +146,7 @@ dependencies = [
|
|||||||
"bat-assetbundle",
|
"bat-assetbundle",
|
||||||
"bat-cas-engine",
|
"bat-cas-engine",
|
||||||
"bat-core",
|
"bat-core",
|
||||||
|
"bat-patch",
|
||||||
"blake3",
|
"blake3",
|
||||||
"hex",
|
"hex",
|
||||||
"libc",
|
"libc",
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ pub use adapter::{
|
|||||||
};
|
};
|
||||||
pub use registry::UnityAdapterRegistry;
|
pub use registry::UnityAdapterRegistry;
|
||||||
pub use serialized_file::{
|
pub use serialized_file::{
|
||||||
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
|
UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
|
||||||
UnityTypeTreeNode,
|
UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
|
||||||
};
|
};
|
||||||
pub use unity_2021_3::Unity2021_3Adapter;
|
pub use unity_2021_3::Unity2021_3Adapter;
|
||||||
|
|||||||
@@ -4,6 +4,6 @@
|
|||||||
//! existing call sites can continue to import through `bat_adapters::unity`.
|
//! existing call sites can continue to import through `bat_adapters::unity`.
|
||||||
|
|
||||||
pub use bat_assetbundle::{
|
pub use bat_assetbundle::{
|
||||||
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
|
UnitySerializedField, UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset,
|
||||||
UnityTypeTreeNode,
|
UnitySerializedType, UnitySerializedValue, UnityTypeTreeNode,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ pub mod translation;
|
|||||||
|
|
||||||
pub use game_client::{ClientStatus, GameClient, GameRegion};
|
pub use game_client::{ClientStatus, GameClient, GameRegion};
|
||||||
pub use game_version::{GameVersion, UnityVersion};
|
pub use game_version::{GameVersion, UnityVersion};
|
||||||
pub use resource::{crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceType};
|
pub use resource::{
|
||||||
|
crc32_ieee, IntegrityMismatch, Resource, ResourceEntry, ResourceMetadata, ResourceType,
|
||||||
|
};
|
||||||
pub use translation::{
|
pub use translation::{
|
||||||
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
|
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
|
||||||
TranslationStatus,
|
TranslationStatus,
|
||||||
|
|||||||
@@ -43,6 +43,58 @@ pub struct ResourceEntry {
|
|||||||
pub crc: Option<u32>,
|
pub crc: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 资源解析与发布侧元数据。
|
||||||
|
///
|
||||||
|
/// 该结构默认全空,保证旧索引和只保存基础 manifest 信息的资源仍可反序列化。
|
||||||
|
/// 官方资源导入会按 release manifest 和 parse cache 填充这些字段,供
|
||||||
|
/// `resource.index` 等只读接口暴露版本、平台、bundle、TextAsset 和 TextUnit 摘要。
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct ResourceMetadata {
|
||||||
|
/// 资源所属的官方 release ID。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub official_release_id: Option<String>,
|
||||||
|
/// 从官方相对路径推断的平台标签,例如 `windows` 或 `android`。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub platform: Option<String>,
|
||||||
|
/// 资源本身或所在 bundle 的官方相对路径。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub bundle_path: Option<String>,
|
||||||
|
/// ZIP 内被解析到的 bundle entry;直接 bundle 为空。
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub archive_entries: Vec<String>,
|
||||||
|
/// parse cache 中出现过的解析状态标签。
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub parse_statuses: Vec<String>,
|
||||||
|
/// 解析到的 Unity 版本集合。
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub unity_versions: Vec<String>,
|
||||||
|
/// UnityFS directory file 总数。
|
||||||
|
#[serde(default, skip_serializing_if = "is_zero")]
|
||||||
|
pub unityfs_file_count: u64,
|
||||||
|
/// Unity serialized file 总数。
|
||||||
|
#[serde(default, skip_serializing_if = "is_zero")]
|
||||||
|
pub serialized_file_count: u64,
|
||||||
|
/// TextAsset 对象总数。
|
||||||
|
#[serde(default, skip_serializing_if = "is_zero")]
|
||||||
|
pub text_asset_count: u64,
|
||||||
|
/// TextAsset 名称集合。
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub text_assets: Vec<String>,
|
||||||
|
/// TextUnit 总数。
|
||||||
|
#[serde(default, skip_serializing_if = "is_zero")]
|
||||||
|
pub text_unit_count: u64,
|
||||||
|
/// TextUnit 格式标签集合,例如 `json`、`csv`、`tsv`、`plain`。
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub text_unit_formats: Vec<String>,
|
||||||
|
/// TextUnit 提取阶段的非致命诊断数量。
|
||||||
|
#[serde(default, skip_serializing_if = "is_zero")]
|
||||||
|
pub text_unit_error_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_zero(value: &u64) -> bool {
|
||||||
|
*value == 0
|
||||||
|
}
|
||||||
|
|
||||||
/// 已下载字节与 catalog 声明的可校验字段不一致。
|
/// 已下载字节与 catalog 声明的可校验字段不一致。
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum IntegrityMismatch {
|
pub enum IntegrityMismatch {
|
||||||
@@ -133,6 +185,9 @@ pub struct Resource {
|
|||||||
pub local_path: PathBuf,
|
pub local_path: PathBuf,
|
||||||
/// 资源条目
|
/// 资源条目
|
||||||
pub entry: ResourceEntry,
|
pub entry: ResourceEntry,
|
||||||
|
/// 解析、发布和索引侧扩展元数据。
|
||||||
|
#[serde(default)]
|
||||||
|
pub metadata: ResourceMetadata,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ serde_json.workspace = true
|
|||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
lz4 = "1.28"
|
lz4 = "1.28"
|
||||||
lzma-rs = "0.3"
|
lzma-rs = "0.3"
|
||||||
|
md-5 = "0.10"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
|
|||||||
@@ -9,14 +9,26 @@
|
|||||||
|
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod parser;
|
pub mod parser;
|
||||||
|
pub mod patch;
|
||||||
pub mod serialized;
|
pub mod serialized;
|
||||||
|
pub mod text;
|
||||||
pub mod types;
|
pub mod types;
|
||||||
|
|
||||||
pub use error::{AssetBundleError, Result};
|
pub use error::{AssetBundleError, Result};
|
||||||
pub use parser::{compression_from_flags, Parser, UnityFsParser};
|
pub use parser::{compression_from_flags, Parser, UnityFsParser};
|
||||||
|
pub use patch::{
|
||||||
|
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
|
||||||
|
StringFieldPatch, TextAssetPatch,
|
||||||
|
};
|
||||||
pub use serialized::{
|
pub use serialized::{
|
||||||
UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType,
|
UnityManagedReferenceMetadata, UnityManagedReferenceRecord, UnitySerializedField,
|
||||||
UnityTypeTreeNode,
|
UnitySerializedFieldReplacement, UnitySerializedFile, UnitySerializedObject,
|
||||||
|
UnitySerializedReplacementValue, UnitySerializedTextAsset, UnitySerializedType,
|
||||||
|
UnitySerializedValue, UnityTypeTreeNode,
|
||||||
|
};
|
||||||
|
pub use text::{
|
||||||
|
text_units_to_jsonl, TextUnit, TextUnitExtractionError, TextUnitExtractionReport,
|
||||||
|
TextUnitExtractor,
|
||||||
};
|
};
|
||||||
pub use types::{
|
pub use types::{
|
||||||
AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle,
|
AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,949 @@
|
|||||||
|
//! Text extraction from parsed Unity serialized objects.
|
||||||
|
|
||||||
|
use crate::serialized::{
|
||||||
|
managed_reference_metadata_from_fields, UnityManagedReferenceMetadata,
|
||||||
|
UnityManagedReferenceRecord, UnitySerializedField, UnitySerializedValue,
|
||||||
|
};
|
||||||
|
use crate::types::ParsedAssetBundle;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
/// One text unit used by translation, glossary and patch pipelines.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TextUnit {
|
||||||
|
/// Original source text.
|
||||||
|
pub source_text: String,
|
||||||
|
/// Logical AssetBundle path, when provided by the caller.
|
||||||
|
pub bundle_path: Option<String>,
|
||||||
|
/// ZIP/archive entry containing the bundle, when known.
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Unity serialized file path.
|
||||||
|
pub serialized_file: Option<String>,
|
||||||
|
/// Unity object path ID.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Unity class ID, for example `49` for `TextAsset`.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// TypeTree field path. `TextAsset` is used for a whole TextAsset payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Byte offset relative to the beginning of the Unity object payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_offset: Option<usize>,
|
||||||
|
/// Number of bytes consumed by this field, including alignment padding.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_byte_size: Option<usize>,
|
||||||
|
/// Unity version associated with the source.
|
||||||
|
pub version: String,
|
||||||
|
/// Stable context for format, asset name and extraction details.
|
||||||
|
pub context: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Non-fatal diagnostic generated while extracting text units.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TextUnitExtractionError {
|
||||||
|
/// Serialized file containing the failed object.
|
||||||
|
pub serialized_file: Option<String>,
|
||||||
|
/// Object path ID, when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Unity class ID, when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// TypeTree field path, when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Byte offset relative to the beginning of the Unity object payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub offset: Option<usize>,
|
||||||
|
/// Human-readable error.
|
||||||
|
pub error: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of extracting text units from one parsed bundle.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TextUnitExtractionReport {
|
||||||
|
/// Extracted text units in deterministic traversal order.
|
||||||
|
pub units: Vec<TextUnit>,
|
||||||
|
/// Non-fatal object-level errors.
|
||||||
|
pub errors: Vec<TextUnitExtractionError>,
|
||||||
|
/// TextAsset payloads that were binary or invalid UTF-8.
|
||||||
|
pub skipped_binary_text_assets: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts translation-ready text from Unity bundle data.
|
||||||
|
#[derive(Debug, Default, Clone, Copy)]
|
||||||
|
pub struct TextUnitExtractor;
|
||||||
|
|
||||||
|
impl TextUnitExtractor {
|
||||||
|
/// Creates an extractor.
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts TextAsset and TypeTree string fields from a parsed bundle.
|
||||||
|
pub fn extract_bundle(
|
||||||
|
&self,
|
||||||
|
bundle: &ParsedAssetBundle,
|
||||||
|
bundle_path: Option<&str>,
|
||||||
|
) -> TextUnitExtractionReport {
|
||||||
|
self.extract_bundle_with_context(bundle, bundle_path, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts text with both logical bundle and archive-entry context.
|
||||||
|
pub fn extract_bundle_with_context(
|
||||||
|
&self,
|
||||||
|
bundle: &ParsedAssetBundle,
|
||||||
|
bundle_path: Option<&str>,
|
||||||
|
archive_entry: Option<&str>,
|
||||||
|
) -> TextUnitExtractionReport {
|
||||||
|
let mut report = TextUnitExtractionReport {
|
||||||
|
units: Vec::new(),
|
||||||
|
errors: Vec::new(),
|
||||||
|
skipped_binary_text_assets: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
for asset in &bundle.text_assets {
|
||||||
|
if let Some((format, text)) = decode_text_payload(&asset.bytes) {
|
||||||
|
let mut context = BTreeMap::new();
|
||||||
|
context.insert("format".to_string(), format.to_string());
|
||||||
|
context.insert("asset_name".to_string(), asset.name.clone());
|
||||||
|
context.insert("source_kind".to_string(), "TextAsset".to_string());
|
||||||
|
report.units.push(TextUnit {
|
||||||
|
source_text: text,
|
||||||
|
bundle_path: bundle_path.map(ToOwned::to_owned),
|
||||||
|
archive_entry: archive_entry.map(ToOwned::to_owned),
|
||||||
|
serialized_file: asset.source_path.clone(),
|
||||||
|
path_id: Some(asset.path_id),
|
||||||
|
class_id: Some(49),
|
||||||
|
field_path: Some("TextAsset".to_string()),
|
||||||
|
field_offset: None,
|
||||||
|
field_byte_size: None,
|
||||||
|
version: bundle.unity_version.clone(),
|
||||||
|
context,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
report.skipped_binary_text_assets += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for serialized_file in &bundle.serialized_files {
|
||||||
|
for object in &serialized_file.objects {
|
||||||
|
if object.class_id == 49 || !serialized_file.object_has_type_tree(object) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let fields = match serialized_file.fields_for_object_entry(object) {
|
||||||
|
Ok(fields) => fields,
|
||||||
|
Err(error) => {
|
||||||
|
report.errors.push(TextUnitExtractionError {
|
||||||
|
serialized_file: serialized_file.source_path.clone(),
|
||||||
|
path_id: Some(object.path_id),
|
||||||
|
class_id: Some(object.class_id),
|
||||||
|
field_path: None,
|
||||||
|
offset: None,
|
||||||
|
error: error.to_string(),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let context = FieldTextContext {
|
||||||
|
serialized_file_path: serialized_file.source_path.as_deref(),
|
||||||
|
path_id: object.path_id,
|
||||||
|
class_id: object.class_id,
|
||||||
|
version: &bundle.unity_version,
|
||||||
|
bundle_path,
|
||||||
|
archive_entry,
|
||||||
|
managed_reference: None,
|
||||||
|
};
|
||||||
|
for field in fields {
|
||||||
|
collect_field_text(&mut report.units, &context, &field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
report
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serializes text units as one stable JSON object per line.
|
||||||
|
pub fn text_units_to_jsonl(units: &[TextUnit]) -> Result<String, serde_json::Error> {
|
||||||
|
let mut output = String::new();
|
||||||
|
for unit in units {
|
||||||
|
output.push_str(&serde_json::to_string(unit)?);
|
||||||
|
output.push('\n');
|
||||||
|
}
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FieldTextContext<'a> {
|
||||||
|
serialized_file_path: Option<&'a str>,
|
||||||
|
path_id: i64,
|
||||||
|
class_id: i32,
|
||||||
|
version: &'a str,
|
||||||
|
bundle_path: Option<&'a str>,
|
||||||
|
archive_entry: Option<&'a str>,
|
||||||
|
managed_reference: Option<UnityManagedReferenceMetadata>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> FieldTextContext<'a> {
|
||||||
|
fn with_managed_reference(&self, metadata: Option<&UnityManagedReferenceMetadata>) -> Self {
|
||||||
|
Self {
|
||||||
|
serialized_file_path: self.serialized_file_path,
|
||||||
|
path_id: self.path_id,
|
||||||
|
class_id: self.class_id,
|
||||||
|
version: self.version,
|
||||||
|
bundle_path: self.bundle_path,
|
||||||
|
archive_entry: self.archive_entry,
|
||||||
|
managed_reference: metadata.cloned().or_else(|| self.managed_reference.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_field_text<'a>(
|
||||||
|
units: &mut Vec<TextUnit>,
|
||||||
|
context: &FieldTextContext<'a>,
|
||||||
|
field: &'a UnitySerializedField,
|
||||||
|
) {
|
||||||
|
match &field.value {
|
||||||
|
UnitySerializedValue::String(text) if !text.is_empty() => {
|
||||||
|
let mut unit_context = BTreeMap::new();
|
||||||
|
unit_context.insert("format".to_string(), "plain".to_string());
|
||||||
|
unit_context.insert(
|
||||||
|
"source_kind".to_string(),
|
||||||
|
if context.managed_reference.is_some() {
|
||||||
|
"ManagedReferenceField"
|
||||||
|
} else {
|
||||||
|
"TypeTreeField"
|
||||||
|
}
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
unit_context.insert("type_name".to_string(), field.type_name.clone());
|
||||||
|
if let Some(metadata) = &context.managed_reference {
|
||||||
|
insert_managed_reference_context(&mut unit_context, metadata);
|
||||||
|
}
|
||||||
|
units.push(TextUnit {
|
||||||
|
source_text: text.clone(),
|
||||||
|
bundle_path: context.bundle_path.map(ToOwned::to_owned),
|
||||||
|
archive_entry: context.archive_entry.map(ToOwned::to_owned),
|
||||||
|
serialized_file: context.serialized_file_path.map(ToOwned::to_owned),
|
||||||
|
path_id: Some(context.path_id),
|
||||||
|
class_id: Some(context.class_id),
|
||||||
|
field_path: Some(field.path.clone()),
|
||||||
|
field_offset: Some(field.offset),
|
||||||
|
field_byte_size: Some(field.byte_size),
|
||||||
|
version: context.version.to_string(),
|
||||||
|
context: unit_context,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
UnitySerializedValue::Object(fields) => {
|
||||||
|
for field in fields {
|
||||||
|
collect_field_text(units, context, field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
UnitySerializedValue::ManagedReference {
|
||||||
|
metadata, fields, ..
|
||||||
|
} => {
|
||||||
|
let managed_context = context.with_managed_reference(metadata.as_ref());
|
||||||
|
for field in fields {
|
||||||
|
collect_managed_reference_child_text(units, &managed_context, field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
UnitySerializedValue::ManagedReferenceRegistry { references, fields } => {
|
||||||
|
if references.is_empty() {
|
||||||
|
let fallback_metadata = managed_reference_metadata_from_fields(fields);
|
||||||
|
for field in fields {
|
||||||
|
let sibling_metadata =
|
||||||
|
managed_reference_metadata_from_sibling_fields(fields, field);
|
||||||
|
let managed_context = context.with_managed_reference(
|
||||||
|
sibling_metadata.as_ref().or(fallback_metadata.as_ref()),
|
||||||
|
);
|
||||||
|
collect_managed_reference_child_text(units, &managed_context, field);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for reference in references {
|
||||||
|
collect_managed_reference_record_text(units, context, reference);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
UnitySerializedValue::Array(values) | UnitySerializedValue::Map(values) => {
|
||||||
|
for field in values {
|
||||||
|
collect_field_text(units, context, field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_managed_reference_record_text<'a>(
|
||||||
|
units: &mut Vec<TextUnit>,
|
||||||
|
context: &FieldTextContext<'a>,
|
||||||
|
reference: &'a UnityManagedReferenceRecord,
|
||||||
|
) {
|
||||||
|
let managed_context = context.with_managed_reference(Some(&reference.metadata));
|
||||||
|
for field in &reference.fields {
|
||||||
|
collect_field_text(units, &managed_context, field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_managed_reference_child_text<'a>(
|
||||||
|
units: &mut Vec<TextUnit>,
|
||||||
|
context: &FieldTextContext<'a>,
|
||||||
|
field: &'a UnitySerializedField,
|
||||||
|
) {
|
||||||
|
let key = normalized_text_metadata_key(&field.name);
|
||||||
|
if is_managed_reference_metadata_text_key(&key) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if is_managed_reference_payload_text_key(&key) {
|
||||||
|
if let Some(children) = serialized_field_children(field) {
|
||||||
|
for child in children {
|
||||||
|
collect_field_text(units, context, child);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
collect_field_text(units, context, field);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(children) = serialized_field_children(field) {
|
||||||
|
if let Some(metadata) = managed_reference_metadata_from_fields(children) {
|
||||||
|
let managed_context = context.with_managed_reference(Some(&metadata));
|
||||||
|
for child in children {
|
||||||
|
collect_managed_reference_child_text(units, &managed_context, child);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
collect_field_text(units, context, field);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn managed_reference_metadata_from_sibling_fields(
|
||||||
|
fields: &[UnitySerializedField],
|
||||||
|
field: &UnitySerializedField,
|
||||||
|
) -> Option<UnityManagedReferenceMetadata> {
|
||||||
|
let record_prefix = managed_reference_record_path_prefix(&field.path)?;
|
||||||
|
let grouped_fields = fields
|
||||||
|
.iter()
|
||||||
|
.filter(|candidate| {
|
||||||
|
candidate.path == record_prefix
|
||||||
|
|| candidate
|
||||||
|
.path
|
||||||
|
.strip_prefix(record_prefix)
|
||||||
|
.is_some_and(|suffix| suffix.starts_with('.'))
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
managed_reference_metadata_from_fields(&grouped_fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn managed_reference_record_path_prefix(path: &str) -> Option<&str> {
|
||||||
|
if let Some(index_end) = path.rfind(']') {
|
||||||
|
return Some(&path[..=index_end]);
|
||||||
|
}
|
||||||
|
path.rsplit_once('.')
|
||||||
|
.map(|(parent, _)| parent)
|
||||||
|
.filter(|parent| !parent.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_managed_reference_context(
|
||||||
|
context: &mut BTreeMap<String, String>,
|
||||||
|
metadata: &UnityManagedReferenceMetadata,
|
||||||
|
) {
|
||||||
|
if let Some(reference_id) = metadata.reference_id {
|
||||||
|
context.insert("managed_reference_id".to_string(), reference_id.to_string());
|
||||||
|
}
|
||||||
|
if let Some(value) = &metadata.full_type_name {
|
||||||
|
context.insert(
|
||||||
|
"managed_reference_full_type_name".to_string(),
|
||||||
|
value.clone(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(value) = &metadata.type_name {
|
||||||
|
context.insert("managed_reference_type".to_string(), value.clone());
|
||||||
|
}
|
||||||
|
if let Some(value) = &metadata.namespace {
|
||||||
|
context.insert("managed_reference_namespace".to_string(), value.clone());
|
||||||
|
}
|
||||||
|
if let Some(value) = &metadata.assembly_name {
|
||||||
|
context.insert("managed_reference_assembly".to_string(), value.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn serialized_field_children(field: &UnitySerializedField) -> Option<&[UnitySerializedField]> {
|
||||||
|
match &field.value {
|
||||||
|
UnitySerializedValue::Object(fields)
|
||||||
|
| UnitySerializedValue::Array(fields)
|
||||||
|
| UnitySerializedValue::Map(fields)
|
||||||
|
| UnitySerializedValue::ManagedReference { fields, .. }
|
||||||
|
| UnitySerializedValue::ManagedReferenceRegistry { fields, .. } => Some(fields),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_text_metadata_key(name: &str) -> String {
|
||||||
|
name.strip_prefix("m_")
|
||||||
|
.unwrap_or(name)
|
||||||
|
.chars()
|
||||||
|
.filter(|character| character.is_ascii_alphanumeric())
|
||||||
|
.flat_map(char::to_lowercase)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_managed_reference_metadata_text_key(key: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
key,
|
||||||
|
"rid"
|
||||||
|
| "id"
|
||||||
|
| "identifier"
|
||||||
|
| "refid"
|
||||||
|
| "referenceid"
|
||||||
|
| "managedreferenceid"
|
||||||
|
| "managedreferenceids"
|
||||||
|
| "managedreferencesid"
|
||||||
|
| "managedreferencesids"
|
||||||
|
| "serializedreferenceid"
|
||||||
|
| "serializedreferenceids"
|
||||||
|
| "refids"
|
||||||
|
| "type"
|
||||||
|
| "typeid"
|
||||||
|
| "typeinfo"
|
||||||
|
| "typename"
|
||||||
|
| "fullname"
|
||||||
|
| "fulltypename"
|
||||||
|
| "class"
|
||||||
|
| "classname"
|
||||||
|
| "managedreferenceclassname"
|
||||||
|
| "serializedreferenceclassname"
|
||||||
|
| "klass"
|
||||||
|
| "managedtype"
|
||||||
|
| "managedreferencetype"
|
||||||
|
| "managedreferencefullname"
|
||||||
|
| "managedreferencefulltypename"
|
||||||
|
| "serializedreferencetype"
|
||||||
|
| "serializedreferencefullname"
|
||||||
|
| "serializedreferencefulltypename"
|
||||||
|
| "assemblyqualifiedname"
|
||||||
|
| "ns"
|
||||||
|
| "namespace"
|
||||||
|
| "namespacename"
|
||||||
|
| "managedreferencenamespace"
|
||||||
|
| "managedreferencenamespacename"
|
||||||
|
| "serializedreferencenamespace"
|
||||||
|
| "serializedreferencenamespacename"
|
||||||
|
| "asm"
|
||||||
|
| "asmname"
|
||||||
|
| "assembly"
|
||||||
|
| "assemblyname"
|
||||||
|
| "managedreferenceassembly"
|
||||||
|
| "managedreferenceassemblyname"
|
||||||
|
| "serializedreferenceassembly"
|
||||||
|
| "serializedreferenceassemblyname"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_managed_reference_payload_text_key(key: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
key,
|
||||||
|
"data"
|
||||||
|
| "payload"
|
||||||
|
| "value"
|
||||||
|
| "object"
|
||||||
|
| "instance"
|
||||||
|
| "managedreferencepayload"
|
||||||
|
| "referencepayload"
|
||||||
|
| "serializedreferencepayload"
|
||||||
|
| "managedreferencevalue"
|
||||||
|
| "referencevalue"
|
||||||
|
| "serializedreferencevalue"
|
||||||
|
| "managedreferenceobject"
|
||||||
|
| "referenceobject"
|
||||||
|
| "serializedreferenceobject"
|
||||||
|
| "managedreferencedata"
|
||||||
|
| "referencedata"
|
||||||
|
| "serializeddata"
|
||||||
|
| "serializedreferencedata"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_text_payload(bytes: &[u8]) -> Option<(&'static str, String)> {
|
||||||
|
let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
|
||||||
|
if bytes.contains(&0) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let text = std::str::from_utf8(bytes).ok()?.to_string();
|
||||||
|
if text.trim().is_empty()
|
||||||
|
|| text
|
||||||
|
.chars()
|
||||||
|
.any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t'))
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let trimmed = text.trim();
|
||||||
|
if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
|
||||||
|
return Some(("json", text));
|
||||||
|
}
|
||||||
|
if text.lines().any(|line| line.contains('\t')) {
|
||||||
|
return Some(("tsv", text));
|
||||||
|
}
|
||||||
|
if text.lines().count() > 1 && text.lines().any(|line| line.contains(',')) {
|
||||||
|
return Some(("csv", text));
|
||||||
|
}
|
||||||
|
Some(("plain", text))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::serialized::UnitySerializedTextAsset;
|
||||||
|
use crate::types::{
|
||||||
|
ParsedAssetBundle, UnityFsBlockInfo, UnityFsDirectoryInfo, UnityFsHeader,
|
||||||
|
UnitySerializedParseError,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_textasset_and_writes_jsonl() {
|
||||||
|
let bundle = ParsedAssetBundle {
|
||||||
|
unity_version: "2021.3.56f2".to_string(),
|
||||||
|
assets: vec!["CAB-test".to_string()],
|
||||||
|
raw_data: Vec::new(),
|
||||||
|
unityfs_header: Some(UnityFsHeader {
|
||||||
|
format_version: 8,
|
||||||
|
target_version: "5.x.x".to_string(),
|
||||||
|
unity_version: "2021.3.56f2".to_string(),
|
||||||
|
total_size: 0,
|
||||||
|
compressed_blocks_info_size: 0,
|
||||||
|
uncompressed_blocks_info_size: 0,
|
||||||
|
flags: 0,
|
||||||
|
}),
|
||||||
|
blocks: vec![UnityFsBlockInfo {
|
||||||
|
uncompressed_size: 0,
|
||||||
|
compressed_size: 0,
|
||||||
|
flags: 0,
|
||||||
|
compression: crate::types::UnityFsCompression::None,
|
||||||
|
}],
|
||||||
|
directories: vec![UnityFsDirectoryInfo {
|
||||||
|
offset: 0,
|
||||||
|
size: 0,
|
||||||
|
flags: 0,
|
||||||
|
path: "CAB-test".to_string(),
|
||||||
|
}],
|
||||||
|
files: Vec::new(),
|
||||||
|
serialized_files: Vec::new(),
|
||||||
|
text_assets: vec![UnitySerializedTextAsset {
|
||||||
|
source_path: Some("CAB-test".to_string()),
|
||||||
|
path_id: 1,
|
||||||
|
name: "dialogue.json".to_string(),
|
||||||
|
bytes: br#"{"text":"hello"}"#.to_vec(),
|
||||||
|
}],
|
||||||
|
serialized_parse_errors: Vec::<UnitySerializedParseError>::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let report = TextUnitExtractor::new().extract_bundle_with_context(
|
||||||
|
&bundle,
|
||||||
|
Some("dialogue.bundle"),
|
||||||
|
Some("assets/dialogue.bundle"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(report.units.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
report.units[0].archive_entry.as_deref(),
|
||||||
|
Some("assets/dialogue.bundle")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
report.units[0].context.get("format"),
|
||||||
|
Some(&"json".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
text_units_to_jsonl(&report.units).unwrap(),
|
||||||
|
format!("{}\n", serde_json::to_string(&report.units[0]).unwrap())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_managed_reference_payload_without_metadata_strings() {
|
||||||
|
let payload_field = string_field(
|
||||||
|
"m_ManagedReferences.references[0].data.message",
|
||||||
|
"message",
|
||||||
|
"こんにちは",
|
||||||
|
);
|
||||||
|
let metadata_field = string_field(
|
||||||
|
"m_ManagedReferences.references[0].managedReferenceFullTypeName",
|
||||||
|
"managedReferenceFullTypeName",
|
||||||
|
"Game BA.Text.ScenarioLine",
|
||||||
|
);
|
||||||
|
let registry_field = UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences".to_string(),
|
||||||
|
name: "m_ManagedReferences".to_string(),
|
||||||
|
type_name: "ManagedReferenceRegistry".to_string(),
|
||||||
|
offset: 0,
|
||||||
|
byte_size: 64,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::ManagedReferenceRegistry {
|
||||||
|
references: vec![UnityManagedReferenceRecord {
|
||||||
|
metadata: UnityManagedReferenceMetadata {
|
||||||
|
reference_id: Some(42),
|
||||||
|
full_type_name: Some("Game BA.Text.ScenarioLine".to_string()),
|
||||||
|
type_name: Some("ScenarioLine".to_string()),
|
||||||
|
namespace: Some("BA.Text".to_string()),
|
||||||
|
assembly_name: Some("Game".to_string()),
|
||||||
|
},
|
||||||
|
fields: vec![payload_field.clone()],
|
||||||
|
}],
|
||||||
|
fields: vec![metadata_field],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let context = FieldTextContext {
|
||||||
|
serialized_file_path: Some("CAB-test"),
|
||||||
|
path_id: 7,
|
||||||
|
class_id: 114,
|
||||||
|
version: "2021.3.56f2",
|
||||||
|
bundle_path: Some("scenario.bundle"),
|
||||||
|
archive_entry: None,
|
||||||
|
managed_reference: None,
|
||||||
|
};
|
||||||
|
let mut units = Vec::new();
|
||||||
|
|
||||||
|
collect_field_text(&mut units, &context, ®istry_field);
|
||||||
|
|
||||||
|
assert_eq!(units.len(), 1);
|
||||||
|
assert_eq!(units[0].source_text, "こんにちは");
|
||||||
|
assert_eq!(
|
||||||
|
units[0].field_path.as_deref(),
|
||||||
|
Some("m_ManagedReferences.references[0].data.message")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("source_kind"),
|
||||||
|
Some(&"ManagedReferenceField".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_id"),
|
||||||
|
Some(&"42".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_full_type_name"),
|
||||||
|
Some(&"Game BA.Text.ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_type"),
|
||||||
|
Some(&"ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_namespace"),
|
||||||
|
Some(&"BA.Text".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_assembly"),
|
||||||
|
Some(&"Game".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_fallback_managed_reference_payload_alias_without_metadata_strings() {
|
||||||
|
let payload_field = string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
|
||||||
|
"message",
|
||||||
|
"こんにちは",
|
||||||
|
);
|
||||||
|
let registry_field = UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences".to_string(),
|
||||||
|
name: "m_ManagedReferences".to_string(),
|
||||||
|
type_name: "ManagedReferencesRegistry".to_string(),
|
||||||
|
offset: 0,
|
||||||
|
byte_size: 96,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::ManagedReferenceRegistry {
|
||||||
|
references: Vec::new(),
|
||||||
|
fields: vec![
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
|
||||||
|
"managedReferenceClassName",
|
||||||
|
"ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
|
||||||
|
"managedReferenceNamespaceName",
|
||||||
|
"BA.Text",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
|
||||||
|
"managedReferenceAssemblyName",
|
||||||
|
"Game",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].serializedReferenceFullTypeName",
|
||||||
|
"serializedReferenceFullTypeName",
|
||||||
|
"Game BA.Text.ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].typeInfo",
|
||||||
|
"typeInfo",
|
||||||
|
"Game BA.Text.ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].typeID.className",
|
||||||
|
"className",
|
||||||
|
"ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].typeID.namespaceName",
|
||||||
|
"namespaceName",
|
||||||
|
"BA.Text",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].typeID.asmName",
|
||||||
|
"asmName",
|
||||||
|
"Game",
|
||||||
|
),
|
||||||
|
UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
|
||||||
|
name: "managedReferenceData".to_string(),
|
||||||
|
type_name: "managedReference".to_string(),
|
||||||
|
offset: 64,
|
||||||
|
byte_size: 24,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::Object(vec![payload_field]),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let context = FieldTextContext {
|
||||||
|
serialized_file_path: Some("CAB-test"),
|
||||||
|
path_id: 7,
|
||||||
|
class_id: 114,
|
||||||
|
version: "2021.3.56f2",
|
||||||
|
bundle_path: Some("scenario.bundle"),
|
||||||
|
archive_entry: None,
|
||||||
|
managed_reference: None,
|
||||||
|
};
|
||||||
|
let mut units = Vec::new();
|
||||||
|
|
||||||
|
collect_field_text(&mut units, &context, ®istry_field);
|
||||||
|
|
||||||
|
assert_eq!(units.len(), 1);
|
||||||
|
assert_eq!(units[0].source_text, "こんにちは");
|
||||||
|
assert_eq!(
|
||||||
|
units[0].field_path.as_deref(),
|
||||||
|
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("source_kind"),
|
||||||
|
Some(&"ManagedReferenceField".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_type"),
|
||||||
|
Some(&"ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_namespace"),
|
||||||
|
Some(&"BA.Text".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_assembly"),
|
||||||
|
Some(&"Game".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_full_type_name"),
|
||||||
|
Some(&"Game BA.Text.ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_fallback_managed_reference_payload_family_alias_with_context() {
|
||||||
|
let payload_field = string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].serializedReferencePayload.message",
|
||||||
|
"message",
|
||||||
|
"こんにちは",
|
||||||
|
);
|
||||||
|
let registry_field = UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences".to_string(),
|
||||||
|
name: "m_ManagedReferences".to_string(),
|
||||||
|
type_name: "ManagedReferencesRegistry".to_string(),
|
||||||
|
offset: 0,
|
||||||
|
byte_size: 96,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::ManagedReferenceRegistry {
|
||||||
|
references: Vec::new(),
|
||||||
|
fields: vec![
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
|
||||||
|
"managedReferenceClassName",
|
||||||
|
"ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
|
||||||
|
"managedReferenceNamespaceName",
|
||||||
|
"BA.Text",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
|
||||||
|
"managedReferenceAssemblyName",
|
||||||
|
"Game",
|
||||||
|
),
|
||||||
|
UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences.RefIds[0].serializedReferencePayload"
|
||||||
|
.to_string(),
|
||||||
|
name: "serializedReferencePayload".to_string(),
|
||||||
|
type_name: "managedReference".to_string(),
|
||||||
|
offset: 64,
|
||||||
|
byte_size: 24,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::Object(vec![payload_field]),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let context = FieldTextContext {
|
||||||
|
serialized_file_path: Some("CAB-test"),
|
||||||
|
path_id: 7,
|
||||||
|
class_id: 114,
|
||||||
|
version: "2021.3.56f2",
|
||||||
|
bundle_path: Some("scenario.bundle"),
|
||||||
|
archive_entry: None,
|
||||||
|
managed_reference: None,
|
||||||
|
};
|
||||||
|
let mut units = Vec::new();
|
||||||
|
|
||||||
|
collect_field_text(&mut units, &context, ®istry_field);
|
||||||
|
|
||||||
|
assert_eq!(units.len(), 1);
|
||||||
|
assert_eq!(units[0].source_text, "こんにちは");
|
||||||
|
assert_eq!(
|
||||||
|
units[0].field_path.as_deref(),
|
||||||
|
Some("m_ManagedReferences.RefIds[0].serializedReferencePayload.message")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("source_kind"),
|
||||||
|
Some(&"ManagedReferenceField".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_type"),
|
||||||
|
Some(&"ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_namespace"),
|
||||||
|
Some(&"BA.Text".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_assembly"),
|
||||||
|
Some(&"Game".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_fallback_managed_reference_sibling_records_with_separate_context() {
|
||||||
|
let registry_field = UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences".to_string(),
|
||||||
|
name: "m_ManagedReferences".to_string(),
|
||||||
|
type_name: "ManagedReferencesRegistry".to_string(),
|
||||||
|
offset: 0,
|
||||||
|
byte_size: 160,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::ManagedReferenceRegistry {
|
||||||
|
references: Vec::new(),
|
||||||
|
fields: vec![
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceClassName",
|
||||||
|
"managedReferenceClassName",
|
||||||
|
"ScenarioLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceNamespaceName",
|
||||||
|
"managedReferenceNamespaceName",
|
||||||
|
"BA.Text",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceAssemblyName",
|
||||||
|
"managedReferenceAssemblyName",
|
||||||
|
"Game",
|
||||||
|
),
|
||||||
|
UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences.RefIds[0].managedReferenceData".to_string(),
|
||||||
|
name: "managedReferenceData".to_string(),
|
||||||
|
type_name: "managedReference".to_string(),
|
||||||
|
offset: 64,
|
||||||
|
byte_size: 24,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::Object(vec![string_field(
|
||||||
|
"m_ManagedReferences.RefIds[0].managedReferenceData.message",
|
||||||
|
"message",
|
||||||
|
"こんにちは",
|
||||||
|
)]),
|
||||||
|
},
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[1].managedReferenceClassName",
|
||||||
|
"managedReferenceClassName",
|
||||||
|
"ChoiceLine",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[1].managedReferenceNamespaceName",
|
||||||
|
"managedReferenceNamespaceName",
|
||||||
|
"BA.Text",
|
||||||
|
),
|
||||||
|
string_field(
|
||||||
|
"m_ManagedReferences.RefIds[1].managedReferenceAssemblyName",
|
||||||
|
"managedReferenceAssemblyName",
|
||||||
|
"Game",
|
||||||
|
),
|
||||||
|
UnitySerializedField {
|
||||||
|
path: "m_ManagedReferences.RefIds[1].referencePayload".to_string(),
|
||||||
|
name: "referencePayload".to_string(),
|
||||||
|
type_name: "managedReference".to_string(),
|
||||||
|
offset: 120,
|
||||||
|
byte_size: 24,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::Object(vec![string_field(
|
||||||
|
"m_ManagedReferences.RefIds[1].referencePayload.message",
|
||||||
|
"message",
|
||||||
|
"選択肢",
|
||||||
|
)]),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let context = FieldTextContext {
|
||||||
|
serialized_file_path: Some("CAB-test"),
|
||||||
|
path_id: 7,
|
||||||
|
class_id: 114,
|
||||||
|
version: "2021.3.56f2",
|
||||||
|
bundle_path: Some("scenario.bundle"),
|
||||||
|
archive_entry: None,
|
||||||
|
managed_reference: None,
|
||||||
|
};
|
||||||
|
let mut units = Vec::new();
|
||||||
|
|
||||||
|
collect_field_text(&mut units, &context, ®istry_field);
|
||||||
|
|
||||||
|
assert_eq!(units.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
units[0].field_path.as_deref(),
|
||||||
|
Some("m_ManagedReferences.RefIds[0].managedReferenceData.message")
|
||||||
|
);
|
||||||
|
assert_eq!(units[0].source_text, "こんにちは");
|
||||||
|
assert_eq!(
|
||||||
|
units[0].context.get("managed_reference_type"),
|
||||||
|
Some(&"ScenarioLine".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
units[1].field_path.as_deref(),
|
||||||
|
Some("m_ManagedReferences.RefIds[1].referencePayload.message")
|
||||||
|
);
|
||||||
|
assert_eq!(units[1].source_text, "選択肢");
|
||||||
|
assert_eq!(
|
||||||
|
units[1].context.get("managed_reference_type"),
|
||||||
|
Some(&"ChoiceLine".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn string_field(path: &str, name: &str, value: &str) -> UnitySerializedField {
|
||||||
|
UnitySerializedField {
|
||||||
|
path: path.to_string(),
|
||||||
|
name: name.to_string(),
|
||||||
|
type_name: "string".to_string(),
|
||||||
|
offset: 0,
|
||||||
|
byte_size: value.len() + 4,
|
||||||
|
type_tree_node_index: None,
|
||||||
|
value: UnitySerializedValue::String(value.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ bat-core = { path = "../core" }
|
|||||||
bat-adapters = { path = "../adapters" }
|
bat-adapters = { path = "../adapters" }
|
||||||
bat-assetbundle = { path = "../crates/bat-assetbundle" }
|
bat-assetbundle = { path = "../crates/bat-assetbundle" }
|
||||||
bat-cas-engine = { path = "../crates/bat-cas-engine" }
|
bat-cas-engine = { path = "../crates/bat-cas-engine" }
|
||||||
|
bat-patch = { path = "../crates/bat-patch" }
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
use bat_adapters::manifest::GenericManifest;
|
use bat_adapters::manifest::GenericManifest;
|
||||||
use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry};
|
use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry};
|
||||||
use bat_core::domain::{Resource, ResourceEntry, ResourceType};
|
use bat_assetbundle::TextUnitExtractor;
|
||||||
|
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
|
||||||
use bat_core::repositories::{CasRepository, ResourceRepository};
|
use bat_core::repositories::{CasRepository, ResourceRepository};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// 待导入的 AssetBundle 数据。
|
/// 待导入的 AssetBundle 数据。
|
||||||
@@ -97,6 +98,12 @@ pub struct UnityFsImportSummary {
|
|||||||
pub text_assets: Vec<String>,
|
pub text_assets: Vec<String>,
|
||||||
/// 非致命 serialized-file 解析诊断数量。
|
/// 非致命 serialized-file 解析诊断数量。
|
||||||
pub serialized_parse_error_count: usize,
|
pub serialized_parse_error_count: usize,
|
||||||
|
/// 从 TextAsset 和 TypeTree 字段提取出的 TextUnit 数量。
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// TextUnit 格式标签。
|
||||||
|
pub text_unit_formats: Vec<String>,
|
||||||
|
/// TextUnit 提取阶段的非致命诊断数量。
|
||||||
|
pub text_unit_error_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Manifest 导入报告。
|
/// Manifest 导入报告。
|
||||||
@@ -208,6 +215,7 @@ impl<'a> ResourceImportService<'a> {
|
|||||||
id: resource_id_for_path(&entry.path),
|
id: resource_id_for_path(&entry.path),
|
||||||
local_path: PathBuf::from(&entry.path),
|
local_path: PathBuf::from(&entry.path),
|
||||||
entry: stored_entry,
|
entry: stored_entry,
|
||||||
|
metadata: ResourceMetadata::default(),
|
||||||
};
|
};
|
||||||
let id = self.resources.add(resource).await?;
|
let id = self.resources.add(resource).await?;
|
||||||
added_resources.push(id.clone());
|
added_resources.push(id.clone());
|
||||||
@@ -266,6 +274,14 @@ impl<'a> ResourceImportService<'a> {
|
|||||||
manifest_path, error
|
manifest_path, error
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
|
let text_units = TextUnitExtractor::new().extract_bundle(&parsed, Some(manifest_path));
|
||||||
|
let text_unit_formats = text_units
|
||||||
|
.units
|
||||||
|
.iter()
|
||||||
|
.filter_map(|unit| unit.context.get("format").cloned())
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
|
|
||||||
Ok(UnityFsImportSummary {
|
Ok(UnityFsImportSummary {
|
||||||
unity_version: parsed.unity_version,
|
unity_version: parsed.unity_version,
|
||||||
@@ -285,6 +301,9 @@ impl<'a> ResourceImportService<'a> {
|
|||||||
.map(|asset| asset.name)
|
.map(|asset| asset.name)
|
||||||
.collect(),
|
.collect(),
|
||||||
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
||||||
|
text_unit_count: text_units.units.len(),
|
||||||
|
text_unit_formats,
|
||||||
|
text_unit_error_count: text_units.errors.len(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -715,6 +734,9 @@ mod tests {
|
|||||||
assert_eq!(unityfs.text_asset_count, 0);
|
assert_eq!(unityfs.text_asset_count, 0);
|
||||||
assert!(unityfs.text_assets.is_empty());
|
assert!(unityfs.text_assets.is_empty());
|
||||||
assert_eq!(unityfs.serialized_parse_error_count, 0);
|
assert_eq!(unityfs.serialized_parse_error_count, 0);
|
||||||
|
assert_eq!(unityfs.text_unit_count, 0);
|
||||||
|
assert!(unityfs.text_unit_formats.is_empty());
|
||||||
|
assert_eq!(unityfs.text_unit_error_count, 0);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report.imported[1].category,
|
report.imported[1].category,
|
||||||
ResourceImportCategory::TextAsset
|
ResourceImportCategory::TextAsset
|
||||||
@@ -799,6 +821,9 @@ mod tests {
|
|||||||
assert_eq!(unityfs.text_asset_count, 1);
|
assert_eq!(unityfs.text_asset_count, 1);
|
||||||
assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]);
|
assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]);
|
||||||
assert_eq!(unityfs.serialized_parse_error_count, 0);
|
assert_eq!(unityfs.serialized_parse_error_count, 0);
|
||||||
|
assert_eq!(unityfs.text_unit_count, 1);
|
||||||
|
assert_eq!(unityfs.text_unit_formats, vec!["plain".to_string()]);
|
||||||
|
assert_eq!(unityfs.text_unit_error_count, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -13,13 +13,18 @@
|
|||||||
pub mod cas;
|
pub mod cas;
|
||||||
mod curl_transfer;
|
mod curl_transfer;
|
||||||
pub mod import;
|
pub mod import;
|
||||||
|
pub mod localized_patch;
|
||||||
|
pub mod official_changes;
|
||||||
pub mod official_download;
|
pub mod official_download;
|
||||||
pub mod official_game_main_config;
|
pub mod official_game_main_config;
|
||||||
pub mod official_launcher;
|
pub mod official_launcher;
|
||||||
pub mod official_parse;
|
pub mod official_parse;
|
||||||
pub mod official_pull;
|
pub mod official_pull;
|
||||||
|
pub mod official_repository;
|
||||||
pub mod official_sync;
|
pub mod official_sync;
|
||||||
|
pub mod official_textunit_queue;
|
||||||
pub mod official_update;
|
pub mod official_update;
|
||||||
|
pub mod patch_ops;
|
||||||
pub mod path_security;
|
pub mod path_security;
|
||||||
pub mod resources;
|
pub mod resources;
|
||||||
mod zip_validation;
|
mod zip_validation;
|
||||||
@@ -32,6 +37,24 @@ pub use import::{
|
|||||||
BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport,
|
BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport,
|
||||||
ResourceImportService,
|
ResourceImportService,
|
||||||
};
|
};
|
||||||
|
pub use localized_patch::{
|
||||||
|
read_localized_patch_manifest_at, read_localized_version_state, LocalizedPatchConfig,
|
||||||
|
LocalizedPatchFile, LocalizedPatchIntegrity, LocalizedPatchManifest, LocalizedPatchOperation,
|
||||||
|
LocalizedPatchReport, LocalizedPatchRollbackInfo, LocalizedPatchService,
|
||||||
|
LocalizedTextAssetPatch, LocalizedVersionState, LOCALIZED_CURRENT_LINK,
|
||||||
|
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_PATCH_MANIFEST_VERSION, LOCALIZED_STAGING_DIR,
|
||||||
|
LOCALIZED_VERSIONS_DIR, LOCALIZED_VERSION_STATE_FILE,
|
||||||
|
};
|
||||||
|
pub use official_changes::{
|
||||||
|
read_resource_change_set_at, write_crowdin_translation_handoff_at,
|
||||||
|
write_official_resource_change_handoff, write_resource_change_set_at,
|
||||||
|
CrowdinTranslationHandoff, OfficialResourceChange, OfficialResourceChangeHandoffReport,
|
||||||
|
OfficialResourceChangeKind, OfficialResourceChangeSet, OfficialResourceChangeSummary,
|
||||||
|
OfficialResourceDescriptor, TranslationHandoffProvider, TranslationHandoffResource,
|
||||||
|
TranslationHandoffStatus, CROWDIN_TRANSLATION_HANDOFF_FILE,
|
||||||
|
CROWDIN_TRANSLATION_HANDOFF_VERSION, OFFICIAL_RESOURCE_CHANGES_FILE,
|
||||||
|
OFFICIAL_RESOURCE_CHANGES_VERSION,
|
||||||
|
};
|
||||||
pub use official_download::{
|
pub use official_download::{
|
||||||
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
|
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
|
||||||
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
|
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
|
||||||
@@ -47,20 +70,35 @@ pub use official_launcher::{
|
|||||||
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
|
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
|
||||||
};
|
};
|
||||||
pub use official_parse::{
|
pub use official_parse::{
|
||||||
read_parse_cache_at, write_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry,
|
query_textunit_index_errors, query_textunit_index_units, read_parse_cache_at,
|
||||||
OfficialParseCacheService, OfficialParseConfig, OfficialParseReport,
|
read_textunit_index_at, write_parse_cache_at, write_textunit_index_at, OfficialParseCache,
|
||||||
|
OfficialParseCacheEntry, OfficialParseCacheService, OfficialParseConfig, OfficialParseReport,
|
||||||
OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus,
|
OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus,
|
||||||
OfficialParseSummary, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION,
|
OfficialParseSummary, OfficialTextUnitIndex, OfficialTextUnitIndexError,
|
||||||
|
OfficialTextUnitIndexSummary, OfficialTextUnitIndexUnit, OfficialTextUnitQuery,
|
||||||
|
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION, OFFICIAL_TEXTUNIT_INDEX_FILE,
|
||||||
|
OFFICIAL_TEXTUNIT_INDEX_VERSION,
|
||||||
};
|
};
|
||||||
pub use official_pull::{
|
pub use official_pull::{
|
||||||
build_official_pull_plan, build_official_pull_plan_for_platform_inventory,
|
build_official_pull_plan, build_official_pull_plan_for_platform_inventory,
|
||||||
build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory,
|
build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory,
|
||||||
OfficialResourcePullPlan,
|
OfficialResourcePullPlan,
|
||||||
};
|
};
|
||||||
|
pub use official_repository::{
|
||||||
|
OfficialReleaseImportConfig, OfficialReleaseImportReport, OfficialReleaseImportService,
|
||||||
|
};
|
||||||
pub use official_sync::{
|
pub use official_sync::{
|
||||||
build_official_sync_plan, changed_endpoint_urls, classify_sync_decision,
|
build_official_sync_plan, changed_endpoint_urls, classify_sync_decision,
|
||||||
default_official_platforms, OfficialSyncDecision, OfficialSyncPlan,
|
default_official_platforms, OfficialSyncDecision, OfficialSyncPlan,
|
||||||
};
|
};
|
||||||
|
pub use official_textunit_queue::{
|
||||||
|
read_textunit_task_queue_at, write_crowdin_textunit_queue_at, write_official_textunit_queues,
|
||||||
|
write_textunit_task_queue_at, CrowdinTextUnitQueue, CrowdinTextUnitQueueItem,
|
||||||
|
OfficialTextUnitQueueReport, OfficialTextUnitTask, OfficialTextUnitTaskQueue,
|
||||||
|
OfficialTextUnitTaskStatus, OfficialTextUnitTaskSummary, CROWDIN_TEXTUNIT_QUEUE_FILE,
|
||||||
|
CROWDIN_TEXTUNIT_QUEUE_VERSION, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE,
|
||||||
|
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
|
||||||
|
};
|
||||||
pub use official_update::{
|
pub use official_update::{
|
||||||
cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging,
|
cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging,
|
||||||
read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot,
|
read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot,
|
||||||
@@ -71,6 +109,12 @@ pub use official_update::{
|
|||||||
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
|
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
|
||||||
OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap,
|
OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap,
|
||||||
};
|
};
|
||||||
|
pub use patch_ops::{
|
||||||
|
apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file,
|
||||||
|
apply_unityfs_text_asset_patch_file, PatchApplyKind, PatchApplyParams, PatchApplyReport,
|
||||||
|
UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams,
|
||||||
|
UnityFsTextAssetPatchParams,
|
||||||
|
};
|
||||||
pub use path_security::{
|
pub use path_security::{
|
||||||
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
|
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
|
||||||
open_append_file, read_file_no_symlink, set_file_mode, validate_output_root,
|
open_append_file, read_file_no_symlink, set_file_mode, validate_output_root,
|
||||||
|
|||||||
@@ -0,0 +1,857 @@
|
|||||||
|
//! Localized release publishing for verified TextAsset patches.
|
||||||
|
|
||||||
|
use bat_assetbundle::{patch_unityfs_text_asset, TextAssetPatch};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use crate::path_security::{
|
||||||
|
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute,
|
||||||
|
read_file_no_symlink, write_file_atomic, STATE_FILE_MODE,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Atomic current pointer under the localized output root.
|
||||||
|
pub const LOCALIZED_CURRENT_LINK: &str = "current";
|
||||||
|
/// Staging directory under the localized output root.
|
||||||
|
pub const LOCALIZED_STAGING_DIR: &str = ".staging";
|
||||||
|
/// Version directory under the localized output root.
|
||||||
|
pub const LOCALIZED_VERSIONS_DIR: &str = "versions";
|
||||||
|
/// Persisted localized release state file name.
|
||||||
|
pub const LOCALIZED_VERSION_STATE_FILE: &str = "localized-version-state.json";
|
||||||
|
/// Per-release patch manifest file name.
|
||||||
|
pub const LOCALIZED_PATCH_MANIFEST_FILE: &str = "localized-patch-manifest.json";
|
||||||
|
/// Current localized patch manifest schema version.
|
||||||
|
pub const LOCALIZED_PATCH_MANIFEST_VERSION: u32 = 1;
|
||||||
|
|
||||||
|
/// One patch operation against a bundle in an official release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct LocalizedTextAssetPatch {
|
||||||
|
/// Relative path of the UnityFS bundle under the official release.
|
||||||
|
pub bundle_path: String,
|
||||||
|
/// TextAsset replacement inside the bundle.
|
||||||
|
pub text_asset: TextAssetPatch,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Configuration for one localized release publication.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct LocalizedPatchConfig {
|
||||||
|
/// Immutable, verified official release root.
|
||||||
|
pub official_release_root: PathBuf,
|
||||||
|
/// Separate localized publication root.
|
||||||
|
pub localized_output_root: PathBuf,
|
||||||
|
/// Version identifier shared with the official release.
|
||||||
|
pub release_id: String,
|
||||||
|
/// Patch operations to apply.
|
||||||
|
pub patches: Vec<LocalizedTextAssetPatch>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalizedPatchConfig {
|
||||||
|
/// Creates a localized patch configuration.
|
||||||
|
pub fn new(
|
||||||
|
official_release_root: impl Into<PathBuf>,
|
||||||
|
localized_output_root: impl Into<PathBuf>,
|
||||||
|
release_id: impl Into<String>,
|
||||||
|
patches: Vec<LocalizedTextAssetPatch>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
official_release_root: official_release_root.into(),
|
||||||
|
localized_output_root: localized_output_root.into(),
|
||||||
|
release_id: release_id.into(),
|
||||||
|
patches,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted localized release state.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedVersionState {
|
||||||
|
/// State schema version.
|
||||||
|
pub state_version: u32,
|
||||||
|
/// Official release ID used as the patch source.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Published localized release ID.
|
||||||
|
pub current_release_id: Option<String>,
|
||||||
|
/// Stable status label.
|
||||||
|
pub status: String,
|
||||||
|
/// Last update time.
|
||||||
|
pub updated_unix_seconds: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One changed file in a localized patch manifest.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedPatchFile {
|
||||||
|
/// Relative bundle path.
|
||||||
|
pub path: String,
|
||||||
|
/// BLAKE3 before applying the patch.
|
||||||
|
pub original_blake3: String,
|
||||||
|
/// BLAKE3 after applying the patch.
|
||||||
|
pub localized_blake3: String,
|
||||||
|
/// Original file size in bytes.
|
||||||
|
#[serde(default)]
|
||||||
|
pub original_bytes: u64,
|
||||||
|
/// Localized file size in bytes.
|
||||||
|
#[serde(default)]
|
||||||
|
pub localized_bytes: u64,
|
||||||
|
/// Localized minus original byte size.
|
||||||
|
#[serde(default)]
|
||||||
|
pub byte_delta: i64,
|
||||||
|
/// TextAsset operations applied to this file.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_asset_operations: Vec<LocalizedPatchOperation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One TextAsset patch operation recorded in the localized patch manifest.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedPatchOperation {
|
||||||
|
/// UnityFS directory path of the serialized file.
|
||||||
|
pub serialized_file_path: String,
|
||||||
|
/// Unity object path ID.
|
||||||
|
pub path_id: i64,
|
||||||
|
/// Expected TextAsset name, when provided.
|
||||||
|
pub expected_name: Option<String>,
|
||||||
|
/// Replacement payload size.
|
||||||
|
pub replacement_bytes: u64,
|
||||||
|
/// BLAKE3 of the replacement payload.
|
||||||
|
pub replacement_blake3: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rollback information recorded for a localized publication.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedPatchRollbackInfo {
|
||||||
|
/// Previous `current` symlink target before this publication.
|
||||||
|
pub previous_current_target: Option<PathBuf>,
|
||||||
|
/// Version directory that should be removed when rolling this publication back.
|
||||||
|
pub remove_version_path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Integrity summary for a published localized release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedPatchIntegrity {
|
||||||
|
/// Number of changed files verified against the manifest.
|
||||||
|
pub verified_changed_file_count: usize,
|
||||||
|
/// Number of TextAsset operations recorded in the manifest.
|
||||||
|
pub verified_text_asset_operation_count: usize,
|
||||||
|
/// Whether `current` points at this localized release.
|
||||||
|
pub current_points_to_release: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted manifest for one localized release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LocalizedPatchManifest {
|
||||||
|
/// Manifest schema version.
|
||||||
|
#[serde(default = "default_patch_manifest_version")]
|
||||||
|
pub manifest_version: u32,
|
||||||
|
/// Official release ID used as the patch source.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Published localized release ID.
|
||||||
|
pub localized_release_id: String,
|
||||||
|
/// Manifest generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Changed file count.
|
||||||
|
pub file_count: usize,
|
||||||
|
/// TextAsset operation count.
|
||||||
|
pub text_asset_operation_count: usize,
|
||||||
|
/// Changed files and their before/after hashes.
|
||||||
|
pub files: Vec<LocalizedPatchFile>,
|
||||||
|
/// Rollback information for this release.
|
||||||
|
pub rollback: LocalizedPatchRollbackInfo,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalizedPatchManifest {
|
||||||
|
/// Converts the localized TextAsset manifest to the generic patch manifest model.
|
||||||
|
pub fn to_patch_manifest(&self) -> bat_patch::PatchManifest {
|
||||||
|
bat_patch::PatchManifest {
|
||||||
|
version: bat_patch::PATCH_MANIFEST_VERSION,
|
||||||
|
patch_id: self.localized_release_id.clone(),
|
||||||
|
source_version: self.official_release_id.clone(),
|
||||||
|
target_version: self.localized_release_id.clone(),
|
||||||
|
files: self
|
||||||
|
.files
|
||||||
|
.iter()
|
||||||
|
.map(|file| bat_patch::PatchManifestFile {
|
||||||
|
path: PathBuf::from(&file.path),
|
||||||
|
patch_kind: bat_patch::PatchKind::UnityFsTextAsset,
|
||||||
|
source_blake3: file.original_blake3.clone(),
|
||||||
|
target_blake3: file.localized_blake3.clone(),
|
||||||
|
source_size: file.original_bytes,
|
||||||
|
target_size: file.localized_bytes,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
rollback: bat_patch::PatchRollback {
|
||||||
|
previous_current_target: self.rollback.previous_current_target.clone(),
|
||||||
|
remove_target_path: Some(self.rollback.remove_version_path.clone()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of a successful localized release publication.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct LocalizedPatchReport {
|
||||||
|
/// Published version directory.
|
||||||
|
pub version_path: PathBuf,
|
||||||
|
/// Atomic current pointer.
|
||||||
|
pub current_path: PathBuf,
|
||||||
|
/// Version state path.
|
||||||
|
pub state_path: PathBuf,
|
||||||
|
/// Patch manifest path.
|
||||||
|
pub patch_manifest_path: PathBuf,
|
||||||
|
/// Changed files.
|
||||||
|
pub files: Vec<LocalizedPatchFile>,
|
||||||
|
/// Persisted patch manifest.
|
||||||
|
pub manifest: LocalizedPatchManifest,
|
||||||
|
/// Integrity check performed after publication.
|
||||||
|
pub integrity: LocalizedPatchIntegrity,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies TextAsset patches and atomically publishes a localized release.
|
||||||
|
#[derive(Debug, Default, Clone, Copy)]
|
||||||
|
pub struct LocalizedPatchService;
|
||||||
|
|
||||||
|
impl LocalizedPatchService {
|
||||||
|
/// Creates the publisher.
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copies the official release, applies patches in staging and publishes it.
|
||||||
|
pub fn publish(&self, config: &LocalizedPatchConfig) -> anyhow::Result<LocalizedPatchReport> {
|
||||||
|
let staging = config
|
||||||
|
.localized_output_root
|
||||||
|
.join(LOCALIZED_STAGING_DIR)
|
||||||
|
.join(&config.release_id);
|
||||||
|
let version_path = config
|
||||||
|
.localized_output_root
|
||||||
|
.join(LOCALIZED_VERSIONS_DIR)
|
||||||
|
.join(&config.release_id);
|
||||||
|
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
|
||||||
|
let previous_current_target = current_symlink_target(¤t_path).ok().flatten();
|
||||||
|
let version_existed_before = version_path.exists();
|
||||||
|
match self.publish_inner(config, previous_current_target.clone()) {
|
||||||
|
Ok(report) => Ok(report),
|
||||||
|
Err(error) => {
|
||||||
|
if let Err(rollback_error) = rollback_failed_publish(
|
||||||
|
&config.localized_output_root,
|
||||||
|
&staging,
|
||||||
|
&version_path,
|
||||||
|
!version_existed_before,
|
||||||
|
¤t_path,
|
||||||
|
previous_current_target.as_ref(),
|
||||||
|
) {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"{error}; rollback failed: {rollback_error}"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn publish_inner(
|
||||||
|
&self,
|
||||||
|
config: &LocalizedPatchConfig,
|
||||||
|
previous_current_target: Option<PathBuf>,
|
||||||
|
) -> anyhow::Result<LocalizedPatchReport> {
|
||||||
|
validate_config(config).map_err(anyhow::Error::msg)?;
|
||||||
|
let staging = config
|
||||||
|
.localized_output_root
|
||||||
|
.join(LOCALIZED_STAGING_DIR)
|
||||||
|
.join(&config.release_id);
|
||||||
|
let version_path = config
|
||||||
|
.localized_output_root
|
||||||
|
.join(LOCALIZED_VERSIONS_DIR)
|
||||||
|
.join(&config.release_id);
|
||||||
|
let current_path = config.localized_output_root.join(LOCALIZED_CURRENT_LINK);
|
||||||
|
let state_path = config
|
||||||
|
.localized_output_root
|
||||||
|
.join(LOCALIZED_VERSION_STATE_FILE);
|
||||||
|
let patch_manifest_path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
|
||||||
|
|
||||||
|
if version_path.exists() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"localized release already exists: {}",
|
||||||
|
version_path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
remove_owned_staging(&staging)?;
|
||||||
|
fs::create_dir_all(&staging)?;
|
||||||
|
copy_tree(&config.official_release_root, &staging)?;
|
||||||
|
|
||||||
|
let mut changed_files = Vec::with_capacity(config.patches.len());
|
||||||
|
for operation in &config.patches {
|
||||||
|
let target = staging.join(Path::new(&operation.bundle_path));
|
||||||
|
ensure_path_within_root(&staging, &target).map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_safe_file_target(&staging, &target, "汉化 patch 输入")
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
let original = fs::read(&target)?;
|
||||||
|
let patched = patch_unityfs_text_asset(&original, &operation.text_asset)
|
||||||
|
.map_err(|error| anyhow::anyhow!("{}: {error}", operation.bundle_path))?;
|
||||||
|
if original == patched {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"patch produced no change: {}",
|
||||||
|
operation.bundle_path
|
||||||
|
));
|
||||||
|
}
|
||||||
|
write_file_atomic(&target, &patched, STATE_FILE_MODE, "汉化 patch 输出")
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
let original_blake3 = blake3::hash(&original).to_hex().to_string();
|
||||||
|
let localized_blake3 = blake3::hash(&patched).to_hex().to_string();
|
||||||
|
changed_files.push(LocalizedPatchFile {
|
||||||
|
path: operation.bundle_path.clone(),
|
||||||
|
original_blake3,
|
||||||
|
localized_blake3,
|
||||||
|
original_bytes: original.len() as u64,
|
||||||
|
localized_bytes: patched.len() as u64,
|
||||||
|
byte_delta: patched.len() as i64 - original.len() as i64,
|
||||||
|
text_asset_operations: vec![LocalizedPatchOperation::from_text_asset_patch(
|
||||||
|
&operation.text_asset,
|
||||||
|
)],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let manifest = LocalizedPatchManifest {
|
||||||
|
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
|
||||||
|
official_release_id: config.release_id.clone(),
|
||||||
|
localized_release_id: config.release_id.clone(),
|
||||||
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
|
file_count: changed_files.len(),
|
||||||
|
text_asset_operation_count: changed_files
|
||||||
|
.iter()
|
||||||
|
.map(|file| file.text_asset_operations.len())
|
||||||
|
.sum(),
|
||||||
|
files: changed_files.clone(),
|
||||||
|
rollback: LocalizedPatchRollbackInfo {
|
||||||
|
previous_current_target: previous_current_target.clone(),
|
||||||
|
remove_version_path: version_path.clone(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
write_file_atomic(
|
||||||
|
&staging.join(LOCALIZED_PATCH_MANIFEST_FILE),
|
||||||
|
&serde_json::to_vec_pretty(&manifest)?,
|
||||||
|
STATE_FILE_MODE,
|
||||||
|
"汉化 patch manifest",
|
||||||
|
)
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
verify_patch_manifest_files(&config.official_release_root, &staging, &manifest)?;
|
||||||
|
fs::create_dir_all(config.localized_output_root.join(LOCALIZED_VERSIONS_DIR))?;
|
||||||
|
fs::rename(&staging, &version_path)?;
|
||||||
|
switch_current_symlink(
|
||||||
|
&config.localized_output_root,
|
||||||
|
¤t_path,
|
||||||
|
&config.release_id,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let state = LocalizedVersionState {
|
||||||
|
state_version: 1,
|
||||||
|
official_release_id: config.release_id.clone(),
|
||||||
|
current_release_id: Some(config.release_id.clone()),
|
||||||
|
status: "localized".to_string(),
|
||||||
|
updated_unix_seconds: unix_seconds_now(),
|
||||||
|
};
|
||||||
|
write_file_atomic(
|
||||||
|
&state_path,
|
||||||
|
&serde_json::to_vec_pretty(&state)?,
|
||||||
|
STATE_FILE_MODE,
|
||||||
|
"汉化版本状态",
|
||||||
|
)
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
let integrity = verify_published_localized_release(
|
||||||
|
&config.official_release_root,
|
||||||
|
&version_path,
|
||||||
|
¤t_path,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(LocalizedPatchReport {
|
||||||
|
version_path,
|
||||||
|
current_path,
|
||||||
|
state_path,
|
||||||
|
patch_manifest_path,
|
||||||
|
files: changed_files,
|
||||||
|
manifest,
|
||||||
|
integrity,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads the localized release state without following a symlink at the file
|
||||||
|
/// path. A missing state file means no localized release has been published.
|
||||||
|
pub fn read_localized_version_state(
|
||||||
|
localized_output_root: &Path,
|
||||||
|
) -> anyhow::Result<Option<LocalizedVersionState>> {
|
||||||
|
let path = localized_output_root.join(LOCALIZED_VERSION_STATE_FILE);
|
||||||
|
let Some(bytes) = read_file_no_symlink(&path, "汉化版本状态").map_err(anyhow::Error::msg)?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let state: LocalizedVersionState = serde_json::from_slice(&bytes)?;
|
||||||
|
if state.state_version != 1 {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"不支持的汉化版本状态 schema:{},当前版本=1",
|
||||||
|
state.state_version
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Some(state))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads a localized patch manifest from a published version directory.
|
||||||
|
pub fn read_localized_patch_manifest_at(
|
||||||
|
version_path: &Path,
|
||||||
|
) -> anyhow::Result<Option<LocalizedPatchManifest>> {
|
||||||
|
let path = version_path.join(LOCALIZED_PATCH_MANIFEST_FILE);
|
||||||
|
let Some(bytes) =
|
||||||
|
read_file_no_symlink(&path, "汉化 patch manifest").map_err(anyhow::Error::msg)?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let manifest: LocalizedPatchManifest = serde_json::from_slice(&bytes)?;
|
||||||
|
if manifest.manifest_version != LOCALIZED_PATCH_MANIFEST_VERSION {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"不支持的汉化 patch manifest schema:{},当前版本={}",
|
||||||
|
manifest.manifest_version,
|
||||||
|
LOCALIZED_PATCH_MANIFEST_VERSION
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Some(manifest))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalizedPatchOperation {
|
||||||
|
fn from_text_asset_patch(patch: &TextAssetPatch) -> Self {
|
||||||
|
Self {
|
||||||
|
serialized_file_path: patch.serialized_file_path.clone(),
|
||||||
|
path_id: patch.path_id,
|
||||||
|
expected_name: patch.expected_name.clone(),
|
||||||
|
replacement_bytes: patch.replacement.len() as u64,
|
||||||
|
replacement_blake3: blake3::hash(&patch.replacement).to_hex().to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_published_localized_release(
|
||||||
|
official_release_root: &Path,
|
||||||
|
version_path: &Path,
|
||||||
|
current_path: &Path,
|
||||||
|
) -> anyhow::Result<LocalizedPatchIntegrity> {
|
||||||
|
let manifest = read_localized_patch_manifest_at(version_path)?.ok_or_else(|| {
|
||||||
|
anyhow::anyhow!(
|
||||||
|
"缺少汉化 patch manifest:{}",
|
||||||
|
version_path.join(LOCALIZED_PATCH_MANIFEST_FILE).display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let mut integrity =
|
||||||
|
verify_patch_manifest_files(official_release_root, version_path, &manifest)?;
|
||||||
|
integrity.current_points_to_release = current_points_to_version(current_path, version_path)?;
|
||||||
|
if !integrity.current_points_to_release {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 current 未指向发布版本:current={} version={}",
|
||||||
|
current_path.display(),
|
||||||
|
version_path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(integrity)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_patch_manifest_files(
|
||||||
|
official_release_root: &Path,
|
||||||
|
localized_release_root: &Path,
|
||||||
|
manifest: &LocalizedPatchManifest,
|
||||||
|
) -> anyhow::Result<LocalizedPatchIntegrity> {
|
||||||
|
let mut operation_count = 0usize;
|
||||||
|
for file in &manifest.files {
|
||||||
|
let relative = Path::new(&file.path);
|
||||||
|
let official_path = official_release_root.join(relative);
|
||||||
|
let localized_path = localized_release_root.join(relative);
|
||||||
|
ensure_path_within_root(official_release_root, &official_path)
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_path_within_root(localized_release_root, &localized_path)
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_safe_file_target(official_release_root, &official_path, "官方 patch 原文件")
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_safe_file_target(localized_release_root, &localized_path, "汉化 patch 产物")
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
let original = fs::read(&official_path)?;
|
||||||
|
let localized = fs::read(&localized_path)?;
|
||||||
|
let original_hash = blake3::hash(&original).to_hex().to_string();
|
||||||
|
let localized_hash = blake3::hash(&localized).to_hex().to_string();
|
||||||
|
if original_hash != file.original_blake3 || original.len() as u64 != file.original_bytes {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 manifest 原文件校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
|
||||||
|
file.path,
|
||||||
|
file.original_blake3,
|
||||||
|
file.original_bytes,
|
||||||
|
original_hash,
|
||||||
|
original.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if localized_hash != file.localized_blake3 || localized.len() as u64 != file.localized_bytes
|
||||||
|
{
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 manifest 产物校验失败 {}:期望 hash={} bytes={},实际 hash={} bytes={}",
|
||||||
|
file.path,
|
||||||
|
file.localized_blake3,
|
||||||
|
file.localized_bytes,
|
||||||
|
localized_hash,
|
||||||
|
localized.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if localized_hash == original_hash {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 manifest 文件未发生变化:{}",
|
||||||
|
file.path
|
||||||
|
));
|
||||||
|
}
|
||||||
|
operation_count += file.text_asset_operations.len();
|
||||||
|
}
|
||||||
|
if manifest.file_count != manifest.files.len() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 manifest file_count 不一致:声明 {},实际 {}",
|
||||||
|
manifest.file_count,
|
||||||
|
manifest.files.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if manifest.text_asset_operation_count != operation_count {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 manifest operation_count 不一致:声明 {},实际 {}",
|
||||||
|
manifest.text_asset_operation_count,
|
||||||
|
operation_count
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(LocalizedPatchIntegrity {
|
||||||
|
verified_changed_file_count: manifest.files.len(),
|
||||||
|
verified_text_asset_operation_count: operation_count,
|
||||||
|
current_points_to_release: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_symlink_target(current_path: &Path) -> anyhow::Result<Option<PathBuf>> {
|
||||||
|
match fs::symlink_metadata(current_path) {
|
||||||
|
Ok(metadata) if metadata.file_type().is_symlink() => Ok(Some(fs::read_link(current_path)?)),
|
||||||
|
Ok(_) => Err(anyhow::anyhow!(
|
||||||
|
"汉化 current 已存在但不是 symlink:{}",
|
||||||
|
current_path.display()
|
||||||
|
)),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||||
|
Err(error) => Err(error.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_points_to_version(current_path: &Path, version_path: &Path) -> anyhow::Result<bool> {
|
||||||
|
let metadata = fs::symlink_metadata(current_path)?;
|
||||||
|
if !metadata.file_type().is_symlink() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
Ok(fs::canonicalize(current_path)? == fs::canonicalize(version_path)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rollback_failed_publish(
|
||||||
|
localized_output_root: &Path,
|
||||||
|
staging: &Path,
|
||||||
|
version_path: &Path,
|
||||||
|
remove_version_path: bool,
|
||||||
|
current_path: &Path,
|
||||||
|
previous_current_target: Option<&PathBuf>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
remove_owned_path(staging)?;
|
||||||
|
if remove_version_path {
|
||||||
|
remove_owned_path(version_path)?;
|
||||||
|
}
|
||||||
|
remove_owned_path(&localized_output_root.join(".current.tmp"))?;
|
||||||
|
remove_owned_path(&localized_output_root.join(".current.rollback.tmp"))?;
|
||||||
|
let failed_target = version_path
|
||||||
|
.file_name()
|
||||||
|
.map(|release_id| Path::new(LOCALIZED_VERSIONS_DIR).join(release_id));
|
||||||
|
if failed_target.as_ref().is_some_and(|target| {
|
||||||
|
fs::read_link(current_path)
|
||||||
|
.map(|current_target| current_target == *target)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}) {
|
||||||
|
restore_current_symlink(localized_output_root, current_path, previous_current_target)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_owned_path(path: &Path) -> anyhow::Result<()> {
|
||||||
|
if let Ok(metadata) = fs::symlink_metadata(path) {
|
||||||
|
if metadata.file_type().is_symlink() || metadata.is_file() {
|
||||||
|
fs::remove_file(path)?;
|
||||||
|
} else if metadata.is_dir() {
|
||||||
|
fs::remove_dir_all(path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn restore_current_symlink(
|
||||||
|
root: &Path,
|
||||||
|
current_path: &Path,
|
||||||
|
previous_current_target: Option<&PathBuf>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
if let Some(previous_target) = previous_current_target {
|
||||||
|
let temporary = root.join(".current.rollback.tmp");
|
||||||
|
remove_owned_path(&temporary)?;
|
||||||
|
symlink(previous_target, &temporary)?;
|
||||||
|
fs::rename(temporary, current_path)?;
|
||||||
|
} else if fs::symlink_metadata(current_path)
|
||||||
|
.map(|metadata| metadata.file_type().is_symlink())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
fs::remove_file(current_path)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn restore_current_symlink(
|
||||||
|
_root: &Path,
|
||||||
|
_current_path: &Path,
|
||||||
|
_previous_current_target: Option<&PathBuf>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_config(config: &LocalizedPatchConfig) -> Result<(), String> {
|
||||||
|
let official = lexical_absolute(&config.official_release_root)?;
|
||||||
|
let localized = lexical_absolute(&config.localized_output_root)?;
|
||||||
|
if official == localized || official.starts_with(&localized) || localized.starts_with(&official)
|
||||||
|
{
|
||||||
|
return Err(format!(
|
||||||
|
"官方 release 与汉化输出目录不能相同或互相嵌套:官方={} 汉化={}",
|
||||||
|
official.display(),
|
||||||
|
localized.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if config.release_id.is_empty()
|
||||||
|
|| config.release_id.contains('/')
|
||||||
|
|| config.release_id.contains('\\')
|
||||||
|
|| config.release_id == "."
|
||||||
|
|| config.release_id == ".."
|
||||||
|
{
|
||||||
|
return Err(format!("非法汉化 release id:{}", config.release_id));
|
||||||
|
}
|
||||||
|
ensure_safe_directory_path(&config.official_release_root, "官方 release")?;
|
||||||
|
ensure_safe_directory_path(&config.localized_output_root, "汉化输出目录")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_tree(source: &Path, destination: &Path) -> anyhow::Result<()> {
|
||||||
|
let metadata = fs::symlink_metadata(source)?;
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"官方 release 不能包含 symlink: {}",
|
||||||
|
source.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if metadata.is_dir() {
|
||||||
|
fs::create_dir_all(destination)?;
|
||||||
|
for entry in fs::read_dir(source)? {
|
||||||
|
let entry = entry?;
|
||||||
|
copy_tree(&entry.path(), &destination.join(entry.file_name()))?;
|
||||||
|
}
|
||||||
|
} else if metadata.is_file() {
|
||||||
|
if let Some(parent) = destination.parent() {
|
||||||
|
fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
fs::copy(source, destination)?;
|
||||||
|
} else {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"官方 release 中存在非普通文件: {}",
|
||||||
|
source.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_owned_staging(path: &Path) -> anyhow::Result<()> {
|
||||||
|
if let Ok(metadata) = fs::symlink_metadata(path) {
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"汉化 staging 不能是 symlink: {}",
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if metadata.is_dir() {
|
||||||
|
fs::remove_dir_all(path)?;
|
||||||
|
} else {
|
||||||
|
fs::remove_file(path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn switch_current_symlink(root: &Path, current: &Path, release_id: &str) -> anyhow::Result<()> {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let temporary = root.join(".current.tmp");
|
||||||
|
if let Ok(metadata) = fs::symlink_metadata(&temporary) {
|
||||||
|
if metadata.file_type().is_symlink() || metadata.is_file() {
|
||||||
|
fs::remove_file(&temporary)?;
|
||||||
|
} else if metadata.is_dir() {
|
||||||
|
fs::remove_dir_all(&temporary)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
symlink(
|
||||||
|
Path::new(LOCALIZED_VERSIONS_DIR).join(release_id),
|
||||||
|
&temporary,
|
||||||
|
)?;
|
||||||
|
fs::rename(temporary, current)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn switch_current_symlink(_root: &Path, _current: &Path, _release_id: &str) -> anyhow::Result<()> {
|
||||||
|
Err(anyhow::anyhow!(
|
||||||
|
"localized release publication requires a Unix symlink-capable platform"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_seconds_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_patch_manifest_version() -> u32 {
|
||||||
|
LOCALIZED_PATCH_MANIFEST_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn localized_manifest_converts_to_generic_patch_manifest() {
|
||||||
|
let source = b"source";
|
||||||
|
let target = b"target";
|
||||||
|
let manifest = LocalizedPatchManifest {
|
||||||
|
manifest_version: LOCALIZED_PATCH_MANIFEST_VERSION,
|
||||||
|
official_release_id: "official-v1".to_string(),
|
||||||
|
localized_release_id: "localized-v1".to_string(),
|
||||||
|
generated_unix_seconds: 123,
|
||||||
|
file_count: 1,
|
||||||
|
text_asset_operation_count: 1,
|
||||||
|
files: vec![LocalizedPatchFile {
|
||||||
|
path: "Bundles/file.bundle".to_string(),
|
||||||
|
original_blake3: blake3::hash(source).to_hex().to_string(),
|
||||||
|
localized_blake3: blake3::hash(target).to_hex().to_string(),
|
||||||
|
original_bytes: source.len() as u64,
|
||||||
|
localized_bytes: target.len() as u64,
|
||||||
|
byte_delta: target.len() as i64 - source.len() as i64,
|
||||||
|
text_asset_operations: vec![LocalizedPatchOperation {
|
||||||
|
serialized_file_path: "CAB-asset".to_string(),
|
||||||
|
path_id: 1,
|
||||||
|
expected_name: Some("Text".to_string()),
|
||||||
|
replacement_bytes: target.len() as u64,
|
||||||
|
replacement_blake3: blake3::hash(target).to_hex().to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
rollback: LocalizedPatchRollbackInfo {
|
||||||
|
previous_current_target: Some(PathBuf::from("versions/previous")),
|
||||||
|
remove_version_path: PathBuf::from("versions/localized-v1"),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let patch_manifest = manifest.to_patch_manifest();
|
||||||
|
|
||||||
|
assert_eq!(patch_manifest.version, bat_patch::PATCH_MANIFEST_VERSION);
|
||||||
|
assert_eq!(patch_manifest.patch_id, "localized-v1");
|
||||||
|
assert_eq!(patch_manifest.source_version, "official-v1");
|
||||||
|
assert_eq!(patch_manifest.target_version, "localized-v1");
|
||||||
|
assert_eq!(patch_manifest.files.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
patch_manifest.files[0].patch_kind,
|
||||||
|
bat_patch::PatchKind::UnityFsTextAsset
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_manifest.rollback.previous_current_target,
|
||||||
|
Some(PathBuf::from("versions/previous"))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
patch_manifest.rollback.remove_target_path,
|
||||||
|
Some(PathBuf::from("versions/localized-v1"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn publishes_a_separate_localized_release_atomically() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let official = temp.path().join("official-release");
|
||||||
|
let localized = temp.path().join("localized");
|
||||||
|
fs::create_dir_all(official.join("TableBundles")).unwrap();
|
||||||
|
fs::write(
|
||||||
|
official.join("TableBundles/TableCatalog.bytes"),
|
||||||
|
b"official",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let report = LocalizedPatchService::new()
|
||||||
|
.publish(&LocalizedPatchConfig::new(
|
||||||
|
&official,
|
||||||
|
&localized,
|
||||||
|
"release-1",
|
||||||
|
Vec::new(),
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(report.version_path.join("TableBundles/TableCatalog.bytes")).unwrap(),
|
||||||
|
b"official"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_link(report.current_path).unwrap(),
|
||||||
|
PathBuf::from("versions/release-1")
|
||||||
|
);
|
||||||
|
let state: LocalizedVersionState =
|
||||||
|
serde_json::from_slice(&fs::read(report.state_path).unwrap()).unwrap();
|
||||||
|
assert_eq!(state.status, "localized");
|
||||||
|
assert_eq!(state.current_release_id.as_deref(), Some("release-1"));
|
||||||
|
assert!(report.patch_manifest_path.is_file());
|
||||||
|
assert_eq!(report.manifest.file_count, 0);
|
||||||
|
assert_eq!(report.integrity.verified_changed_file_count, 0);
|
||||||
|
assert!(report.integrity.current_points_to_release);
|
||||||
|
let manifest = read_localized_patch_manifest_at(&report.version_path)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(manifest.localized_release_id, "release-1");
|
||||||
|
assert_eq!(manifest.rollback.previous_current_target, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn failed_patch_publish_cleans_staging_and_unpublished_version() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let official = temp.path().join("official-release");
|
||||||
|
let localized = temp.path().join("localized");
|
||||||
|
fs::create_dir_all(official.join("Bundles")).unwrap();
|
||||||
|
fs::write(official.join("Bundles/bad.bundle"), b"not-unityfs").unwrap();
|
||||||
|
|
||||||
|
let error = LocalizedPatchService::new()
|
||||||
|
.publish(&LocalizedPatchConfig::new(
|
||||||
|
&official,
|
||||||
|
&localized,
|
||||||
|
"release-1",
|
||||||
|
vec![LocalizedTextAssetPatch {
|
||||||
|
bundle_path: "Bundles/bad.bundle".to_string(),
|
||||||
|
text_asset: TextAssetPatch::new("CAB-bad", 1, b"replacement".to_vec()),
|
||||||
|
}],
|
||||||
|
))
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("Bundles/bad.bundle"));
|
||||||
|
assert!(!localized
|
||||||
|
.join(LOCALIZED_STAGING_DIR)
|
||||||
|
.join("release-1")
|
||||||
|
.exists());
|
||||||
|
assert!(!localized
|
||||||
|
.join(LOCALIZED_VERSIONS_DIR)
|
||||||
|
.join("release-1")
|
||||||
|
.exists());
|
||||||
|
assert!(!localized.join(LOCALIZED_CURRENT_LINK).exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,653 @@
|
|||||||
|
//! Official resource change sets and translation handoff files.
|
||||||
|
//!
|
||||||
|
//! This module is intentionally file-based. Official update generates the
|
||||||
|
//! durable change set after a new release has been fully downloaded and
|
||||||
|
//! verified; parser and translation modules can then consume the same immutable
|
||||||
|
//! handoff without depending on daemon internals.
|
||||||
|
|
||||||
|
use crate::official_download::{
|
||||||
|
read_download_manifest_at, OfficialDownloadManifest, OfficialDownloadManifestEntry,
|
||||||
|
};
|
||||||
|
use crate::path_security::{
|
||||||
|
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
||||||
|
STATE_FILE_MODE,
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
/// Current resource-change-set schema version.
|
||||||
|
pub const OFFICIAL_RESOURCE_CHANGES_VERSION: u32 = 1;
|
||||||
|
/// File name stored under a published official release root.
|
||||||
|
pub const OFFICIAL_RESOURCE_CHANGES_FILE: &str = "official-resource-changes.json";
|
||||||
|
/// Current Crowdin handoff schema version.
|
||||||
|
pub const CROWDIN_TRANSLATION_HANDOFF_VERSION: u32 = 1;
|
||||||
|
/// File name stored under a published official release root.
|
||||||
|
pub const CROWDIN_TRANSLATION_HANDOFF_FILE: &str = "crowdin-translation-handoff.json";
|
||||||
|
|
||||||
|
/// Change kind for one official resource destination.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum OfficialResourceChangeKind {
|
||||||
|
/// Destination did not exist in the previous complete release.
|
||||||
|
Added,
|
||||||
|
/// Destination exists in both releases, but verified size or BLAKE3 changed.
|
||||||
|
Modified,
|
||||||
|
/// Destination existed in the previous release but is absent from the new one.
|
||||||
|
Removed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialResourceChangeKind {
|
||||||
|
/// Returns the stable JSON/RPC label for the change kind.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Added => "added",
|
||||||
|
Self::Modified => "modified",
|
||||||
|
Self::Removed => "removed",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns true when the changed resource should be offered to parser and
|
||||||
|
/// translation modules.
|
||||||
|
pub fn is_incremental_candidate(self) -> bool {
|
||||||
|
matches!(self, Self::Added | Self::Modified)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verified manifest attributes for one official resource.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceDescriptor {
|
||||||
|
/// Official URL from the download manifest.
|
||||||
|
pub url: String,
|
||||||
|
/// Relative path under the official release root.
|
||||||
|
pub destination: String,
|
||||||
|
/// Verified byte count from the download manifest.
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Verified BLAKE3 digest from the download manifest.
|
||||||
|
pub blake3: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&OfficialDownloadManifestEntry> for OfficialResourceDescriptor {
|
||||||
|
fn from(entry: &OfficialDownloadManifestEntry) -> Self {
|
||||||
|
Self {
|
||||||
|
url: entry.url.clone(),
|
||||||
|
destination: entry.destination.clone(),
|
||||||
|
bytes: entry.bytes,
|
||||||
|
blake3: entry.blake3.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One resource-level change between two complete official releases.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceChange {
|
||||||
|
/// Relative path used as the stable comparison key.
|
||||||
|
pub destination: String,
|
||||||
|
/// Change kind for this destination.
|
||||||
|
pub kind: OfficialResourceChangeKind,
|
||||||
|
/// Previous release descriptor, when the destination existed before.
|
||||||
|
pub previous: Option<OfficialResourceDescriptor>,
|
||||||
|
/// Current release descriptor, when the destination exists now.
|
||||||
|
pub current: Option<OfficialResourceDescriptor>,
|
||||||
|
/// Whether parser modules should inspect this resource in incremental mode.
|
||||||
|
pub parse_candidate: bool,
|
||||||
|
/// Whether translation modules should enqueue this resource in incremental mode.
|
||||||
|
pub translation_candidate: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialResourceChange {
|
||||||
|
fn new(
|
||||||
|
destination: String,
|
||||||
|
kind: OfficialResourceChangeKind,
|
||||||
|
previous: Option<OfficialResourceDescriptor>,
|
||||||
|
current: Option<OfficialResourceDescriptor>,
|
||||||
|
) -> Self {
|
||||||
|
let is_candidate = kind.is_incremental_candidate();
|
||||||
|
Self {
|
||||||
|
destination,
|
||||||
|
kind,
|
||||||
|
previous,
|
||||||
|
current,
|
||||||
|
parse_candidate: is_candidate,
|
||||||
|
translation_candidate: is_candidate,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate counters for one official resource change set.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceChangeSummary {
|
||||||
|
/// Whether a previous complete release manifest was available.
|
||||||
|
pub previous_manifest_present: bool,
|
||||||
|
/// Number of entries in the previous release manifest.
|
||||||
|
pub previous_manifest_entry_count: usize,
|
||||||
|
/// Number of entries in the current release manifest.
|
||||||
|
pub current_manifest_entry_count: usize,
|
||||||
|
/// Number of newly added destinations.
|
||||||
|
pub added_count: usize,
|
||||||
|
/// Number of destinations whose verified bytes or BLAKE3 changed.
|
||||||
|
pub modified_count: usize,
|
||||||
|
/// Number of destinations removed from the current release.
|
||||||
|
pub removed_count: usize,
|
||||||
|
/// Number of resources to offer to parser modules.
|
||||||
|
pub parse_candidate_count: usize,
|
||||||
|
/// Number of resources to offer to translation modules.
|
||||||
|
pub translation_candidate_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Durable comparison result between a previous complete official release and
|
||||||
|
/// the newly published official release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceChangeSet {
|
||||||
|
/// Change-set schema version.
|
||||||
|
#[serde(default = "default_resource_changes_version")]
|
||||||
|
pub change_set_version: u32,
|
||||||
|
/// Current official release ID.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Previous official release ID, when known.
|
||||||
|
pub previous_release_id: Option<String>,
|
||||||
|
/// Generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Previous complete release root, when available.
|
||||||
|
pub previous_resource_root: Option<PathBuf>,
|
||||||
|
/// Current complete release root.
|
||||||
|
pub current_resource_root: PathBuf,
|
||||||
|
/// Aggregate counters.
|
||||||
|
pub summary: OfficialResourceChangeSummary,
|
||||||
|
/// Stable, destination-sorted list of changed resources.
|
||||||
|
pub changes: Vec<OfficialResourceChange>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialResourceChangeSet {
|
||||||
|
/// Builds a change set from already loaded manifests.
|
||||||
|
pub fn from_manifests(
|
||||||
|
official_release_id: impl Into<String>,
|
||||||
|
previous_release_id: Option<String>,
|
||||||
|
previous_resource_root: Option<PathBuf>,
|
||||||
|
current_resource_root: PathBuf,
|
||||||
|
previous_manifest: Option<&OfficialDownloadManifest>,
|
||||||
|
current_manifest: &OfficialDownloadManifest,
|
||||||
|
) -> Self {
|
||||||
|
let previous_by_destination = previous_manifest
|
||||||
|
.map(entries_by_destination)
|
||||||
|
.unwrap_or_default();
|
||||||
|
let current_by_destination = entries_by_destination(current_manifest);
|
||||||
|
let destinations = previous_by_destination
|
||||||
|
.keys()
|
||||||
|
.chain(current_by_destination.keys())
|
||||||
|
.cloned()
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
|
let mut changes = Vec::new();
|
||||||
|
let mut summary = OfficialResourceChangeSummary {
|
||||||
|
previous_manifest_present: previous_manifest.is_some(),
|
||||||
|
previous_manifest_entry_count: previous_manifest
|
||||||
|
.map(|manifest| manifest.entries.len())
|
||||||
|
.unwrap_or(0),
|
||||||
|
current_manifest_entry_count: current_manifest.entries.len(),
|
||||||
|
..OfficialResourceChangeSummary::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
for destination in destinations {
|
||||||
|
match (
|
||||||
|
previous_by_destination.get(&destination),
|
||||||
|
current_by_destination.get(&destination),
|
||||||
|
) {
|
||||||
|
(None, Some(current)) => {
|
||||||
|
summary.added_count += 1;
|
||||||
|
changes.push(OfficialResourceChange::new(
|
||||||
|
destination,
|
||||||
|
OfficialResourceChangeKind::Added,
|
||||||
|
None,
|
||||||
|
Some((*current).into()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
(Some(previous), Some(current)) if content_changed(previous, current) => {
|
||||||
|
summary.modified_count += 1;
|
||||||
|
changes.push(OfficialResourceChange::new(
|
||||||
|
destination,
|
||||||
|
OfficialResourceChangeKind::Modified,
|
||||||
|
Some((*previous).into()),
|
||||||
|
Some((*current).into()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
(Some(previous), None) => {
|
||||||
|
summary.removed_count += 1;
|
||||||
|
changes.push(OfficialResourceChange::new(
|
||||||
|
destination,
|
||||||
|
OfficialResourceChangeKind::Removed,
|
||||||
|
Some((*previous).into()),
|
||||||
|
None,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
summary.parse_candidate_count = changes
|
||||||
|
.iter()
|
||||||
|
.filter(|change| change.parse_candidate)
|
||||||
|
.count();
|
||||||
|
summary.translation_candidate_count = changes
|
||||||
|
.iter()
|
||||||
|
.filter(|change| change.translation_candidate)
|
||||||
|
.count();
|
||||||
|
|
||||||
|
Self {
|
||||||
|
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
|
||||||
|
official_release_id: official_release_id.into(),
|
||||||
|
previous_release_id,
|
||||||
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
|
previous_resource_root,
|
||||||
|
current_resource_root,
|
||||||
|
summary,
|
||||||
|
changes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the resources that parser modules should inspect for
|
||||||
|
/// incremental work.
|
||||||
|
pub fn parse_candidates(&self) -> Vec<&OfficialResourceChange> {
|
||||||
|
self.changes
|
||||||
|
.iter()
|
||||||
|
.filter(|change| change.parse_candidate)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the resources that translation modules should enqueue.
|
||||||
|
pub fn translation_candidates(&self) -> Vec<&OfficialResourceChange> {
|
||||||
|
self.changes
|
||||||
|
.iter()
|
||||||
|
.filter(|change| change.translation_candidate)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Provider reserved for translation handoff consumers.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationHandoffProvider {
|
||||||
|
/// Crowdin provider. The handoff file does not make a network request.
|
||||||
|
Crowdin,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationHandoffProvider {
|
||||||
|
/// Returns the stable provider label.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Crowdin => "crowdin",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Status of a generated translation handoff.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationHandoffStatus {
|
||||||
|
/// The handoff was written locally and is waiting for a translation worker.
|
||||||
|
QueuedOffline,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationHandoffStatus {
|
||||||
|
/// Returns the stable status label.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::QueuedOffline => "queued_offline",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One resource entry queued for translation-provider processing.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct TranslationHandoffResource {
|
||||||
|
/// Relative path under the official release root.
|
||||||
|
pub destination: String,
|
||||||
|
/// Change kind that caused this resource to be queued.
|
||||||
|
pub kind: OfficialResourceChangeKind,
|
||||||
|
/// Current official URL.
|
||||||
|
pub url: String,
|
||||||
|
/// Verified byte count.
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Verified BLAKE3 digest.
|
||||||
|
pub blake3: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Crowdin-ready local queue file for added or modified official resources.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CrowdinTranslationHandoff {
|
||||||
|
/// Handoff schema version.
|
||||||
|
#[serde(default = "default_crowdin_handoff_version")]
|
||||||
|
pub handoff_version: u32,
|
||||||
|
/// Translation provider reserved for this queue.
|
||||||
|
pub provider: TranslationHandoffProvider,
|
||||||
|
/// Current queue status.
|
||||||
|
pub status: TranslationHandoffStatus,
|
||||||
|
/// Current official release ID.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Previous official release ID, when known.
|
||||||
|
pub previous_release_id: Option<String>,
|
||||||
|
/// Generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Number of queued resources.
|
||||||
|
pub resource_count: usize,
|
||||||
|
/// Added or modified resources to pass into parsing/translation workers.
|
||||||
|
pub resources: Vec<TranslationHandoffResource>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrowdinTranslationHandoff {
|
||||||
|
/// Builds a local Crowdin handoff from a resource change set.
|
||||||
|
pub fn from_change_set(change_set: &OfficialResourceChangeSet) -> Self {
|
||||||
|
let resources = change_set
|
||||||
|
.translation_candidates()
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|change| {
|
||||||
|
let current = change.current.as_ref()?;
|
||||||
|
Some(TranslationHandoffResource {
|
||||||
|
destination: change.destination.clone(),
|
||||||
|
kind: change.kind,
|
||||||
|
url: current.url.clone(),
|
||||||
|
bytes: current.bytes,
|
||||||
|
blake3: current.blake3.clone(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
Self {
|
||||||
|
handoff_version: CROWDIN_TRANSLATION_HANDOFF_VERSION,
|
||||||
|
provider: TranslationHandoffProvider::Crowdin,
|
||||||
|
status: TranslationHandoffStatus::QueuedOffline,
|
||||||
|
official_release_id: change_set.official_release_id.clone(),
|
||||||
|
previous_release_id: change_set.previous_release_id.clone(),
|
||||||
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
|
resource_count: resources.len(),
|
||||||
|
resources,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generates a change set for two complete release roots and writes both the
|
||||||
|
/// change set and the Crowdin handoff under the current release root.
|
||||||
|
pub fn write_official_resource_change_handoff(
|
||||||
|
previous_resource_root: Option<&Path>,
|
||||||
|
current_resource_root: &Path,
|
||||||
|
official_release_id: &str,
|
||||||
|
previous_release_id: Option<String>,
|
||||||
|
) -> Result<OfficialResourceChangeHandoffReport, String> {
|
||||||
|
let current_manifest = read_download_manifest_at(current_resource_root)?.ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"缺少当前官方下载 manifest,无法生成资源变更集:{}",
|
||||||
|
current_resource_root.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let previous_manifest = match previous_resource_root {
|
||||||
|
Some(root) => read_download_manifest_at(root)?,
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
let previous_manifest_root = previous_manifest
|
||||||
|
.as_ref()
|
||||||
|
.and(previous_resource_root)
|
||||||
|
.map(Path::to_path_buf);
|
||||||
|
let previous_release_id = previous_manifest.as_ref().and(previous_release_id);
|
||||||
|
let change_set = OfficialResourceChangeSet::from_manifests(
|
||||||
|
official_release_id,
|
||||||
|
previous_release_id,
|
||||||
|
previous_manifest_root,
|
||||||
|
current_resource_root.to_path_buf(),
|
||||||
|
previous_manifest.as_ref(),
|
||||||
|
¤t_manifest,
|
||||||
|
);
|
||||||
|
write_resource_change_set_at(current_resource_root, &change_set)?;
|
||||||
|
|
||||||
|
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
|
||||||
|
write_crowdin_translation_handoff_at(current_resource_root, &handoff)?;
|
||||||
|
|
||||||
|
Ok(OfficialResourceChangeHandoffReport {
|
||||||
|
change_set_path: current_resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE),
|
||||||
|
crowdin_handoff_path: current_resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE),
|
||||||
|
summary: change_set.summary,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paths and summary produced after writing a resource-change handoff.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceChangeHandoffReport {
|
||||||
|
/// Path to `official-resource-changes.json`.
|
||||||
|
pub change_set_path: PathBuf,
|
||||||
|
/// Path to `crowdin-translation-handoff.json`.
|
||||||
|
pub crowdin_handoff_path: PathBuf,
|
||||||
|
/// Aggregate change counters.
|
||||||
|
pub summary: OfficialResourceChangeSummary,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads a generated resource change set from a release root.
|
||||||
|
pub fn read_resource_change_set_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
) -> Result<Option<OfficialResourceChangeSet>, String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
|
||||||
|
let Some(bytes) = read_file_no_symlink(&path, "官方资源变更集")? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let change_set: OfficialResourceChangeSet = serde_json::from_slice(&bytes)
|
||||||
|
.map_err(|error| format!("解析官方资源变更集失败 {}:{error}", path.display()))?;
|
||||||
|
if change_set.change_set_version != OFFICIAL_RESOURCE_CHANGES_VERSION {
|
||||||
|
return Err(format!(
|
||||||
|
"不支持的官方资源变更集版本 {},文件 {}",
|
||||||
|
change_set.change_set_version,
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Some(change_set))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a generated resource change set under a release root.
|
||||||
|
pub fn write_resource_change_set_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
change_set: &OfficialResourceChangeSet,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE);
|
||||||
|
ensure_path_within_root(resource_root, &path)?;
|
||||||
|
ensure_safe_file_target(resource_root, &path, "官方资源变更集")?;
|
||||||
|
let bytes = serde_json::to_vec_pretty(change_set)
|
||||||
|
.map_err(|error| format!("序列化官方资源变更集失败:{error}"))?;
|
||||||
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方资源变更集")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a generated Crowdin handoff under a release root.
|
||||||
|
pub fn write_crowdin_translation_handoff_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
handoff: &CrowdinTranslationHandoff,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let path = resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE);
|
||||||
|
ensure_path_within_root(resource_root, &path)?;
|
||||||
|
ensure_safe_file_target(resource_root, &path, "Crowdin 翻译 handoff")?;
|
||||||
|
let bytes = serde_json::to_vec_pretty(handoff)
|
||||||
|
.map_err(|error| format!("序列化 Crowdin 翻译 handoff 失败:{error}"))?;
|
||||||
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin 翻译 handoff")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn entries_by_destination(
|
||||||
|
manifest: &OfficialDownloadManifest,
|
||||||
|
) -> BTreeMap<String, &OfficialDownloadManifestEntry> {
|
||||||
|
manifest
|
||||||
|
.entries
|
||||||
|
.values()
|
||||||
|
.map(|entry| (entry.destination.clone(), entry))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn content_changed(
|
||||||
|
previous: &OfficialDownloadManifestEntry,
|
||||||
|
current: &OfficialDownloadManifestEntry,
|
||||||
|
) -> bool {
|
||||||
|
previous.bytes != current.bytes || previous.blake3 != current.blake3
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_seconds_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_resource_changes_version() -> u32 {
|
||||||
|
OFFICIAL_RESOURCE_CHANGES_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_crowdin_handoff_version() -> u32 {
|
||||||
|
CROWDIN_TRANSLATION_HANDOFF_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn manifest(entries: &[(&str, &str, &[u8])]) -> OfficialDownloadManifest {
|
||||||
|
let mut manifest = OfficialDownloadManifest::default();
|
||||||
|
for (url, destination, bytes) in entries {
|
||||||
|
manifest.entries.insert(
|
||||||
|
(*url).to_string(),
|
||||||
|
OfficialDownloadManifestEntry {
|
||||||
|
url: (*url).to_string(),
|
||||||
|
destination: (*destination).to_string(),
|
||||||
|
bytes: bytes.len() as u64,
|
||||||
|
blake3: blake3::hash(bytes).to_hex().to_string(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_manifest(root: &Path, manifest: &OfficialDownloadManifest) {
|
||||||
|
std::fs::create_dir_all(root).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
root.join("official-download-manifest.json"),
|
||||||
|
serde_json::to_vec(manifest).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn change_set_classifies_added_modified_and_removed_resources() {
|
||||||
|
let previous = manifest(&[
|
||||||
|
("https://old/a", "TableBundles/a.bytes", b"old-a"),
|
||||||
|
("https://old/b", "TableBundles/b.bytes", b"same"),
|
||||||
|
("https://old/c", "TableBundles/c.bytes", b"removed"),
|
||||||
|
(
|
||||||
|
"https://old/u",
|
||||||
|
"TableBundles/url-only.bytes",
|
||||||
|
b"same-url-only",
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
let current = manifest(&[
|
||||||
|
("https://new/a", "TableBundles/a.bytes", b"new-a"),
|
||||||
|
("https://new/b", "TableBundles/b.bytes", b"same"),
|
||||||
|
("https://new/d", "TableBundles/d.bytes", b"added"),
|
||||||
|
(
|
||||||
|
"https://changed-host/u",
|
||||||
|
"TableBundles/url-only.bytes",
|
||||||
|
b"same-url-only",
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let change_set = OfficialResourceChangeSet::from_manifests(
|
||||||
|
"release-new",
|
||||||
|
Some("release-old".to_string()),
|
||||||
|
Some(PathBuf::from("/previous")),
|
||||||
|
PathBuf::from("/current"),
|
||||||
|
Some(&previous),
|
||||||
|
¤t,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(change_set.summary.added_count, 1);
|
||||||
|
assert_eq!(change_set.summary.modified_count, 1);
|
||||||
|
assert_eq!(change_set.summary.removed_count, 1);
|
||||||
|
assert_eq!(change_set.summary.parse_candidate_count, 2);
|
||||||
|
assert_eq!(change_set.summary.translation_candidate_count, 2);
|
||||||
|
let destinations = change_set
|
||||||
|
.translation_candidates()
|
||||||
|
.into_iter()
|
||||||
|
.map(|change| change.destination.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
destinations,
|
||||||
|
vec!["TableBundles/a.bytes", "TableBundles/d.bytes"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn first_release_treats_all_current_resources_as_added() {
|
||||||
|
let current = manifest(&[
|
||||||
|
("https://new/a", "a.bundle", b"a"),
|
||||||
|
("https://new/b", "b.bundle", b"b"),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let change_set = OfficialResourceChangeSet::from_manifests(
|
||||||
|
"release-new",
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
PathBuf::from("/current"),
|
||||||
|
None,
|
||||||
|
¤t,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!change_set.summary.previous_manifest_present);
|
||||||
|
assert_eq!(change_set.summary.added_count, 2);
|
||||||
|
assert_eq!(change_set.summary.translation_candidate_count, 2);
|
||||||
|
assert_eq!(change_set.changes.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crowdin_handoff_excludes_removed_resources() {
|
||||||
|
let previous = manifest(&[("https://old/a", "a.bundle", b"a")]);
|
||||||
|
let current = manifest(&[("https://new/b", "b.bundle", b"b")]);
|
||||||
|
let change_set = OfficialResourceChangeSet::from_manifests(
|
||||||
|
"release-new",
|
||||||
|
Some("release-old".to_string()),
|
||||||
|
None,
|
||||||
|
PathBuf::from("/current"),
|
||||||
|
Some(&previous),
|
||||||
|
¤t,
|
||||||
|
);
|
||||||
|
|
||||||
|
let handoff = CrowdinTranslationHandoff::from_change_set(&change_set);
|
||||||
|
|
||||||
|
assert_eq!(handoff.provider, TranslationHandoffProvider::Crowdin);
|
||||||
|
assert_eq!(handoff.status, TranslationHandoffStatus::QueuedOffline);
|
||||||
|
assert_eq!(handoff.resource_count, 1);
|
||||||
|
assert_eq!(handoff.resources[0].destination, "b.bundle");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_handoff_persists_change_set_and_crowdin_queue() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let previous_root = temp.path().join("previous");
|
||||||
|
let current_root = temp.path().join("current");
|
||||||
|
write_manifest(
|
||||||
|
&previous_root,
|
||||||
|
&manifest(&[("https://old/a", "a.bundle", b"old")]),
|
||||||
|
);
|
||||||
|
write_manifest(
|
||||||
|
¤t_root,
|
||||||
|
&manifest(&[
|
||||||
|
("https://new/a", "a.bundle", b"new"),
|
||||||
|
("https://new/b", "b.bundle", b"added"),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
let report = write_official_resource_change_handoff(
|
||||||
|
Some(&previous_root),
|
||||||
|
¤t_root,
|
||||||
|
"release-new",
|
||||||
|
Some("release-old".to_string()),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(report.summary.modified_count, 1);
|
||||||
|
assert_eq!(report.summary.added_count, 1);
|
||||||
|
assert!(report.change_set_path.exists());
|
||||||
|
assert!(report.crowdin_handoff_path.exists());
|
||||||
|
let change_set = read_resource_change_set_at(¤t_root).unwrap().unwrap();
|
||||||
|
assert_eq!(change_set.summary.translation_candidate_count, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -115,8 +115,8 @@ pub struct OfficialResourcePullProgress {
|
|||||||
///
|
///
|
||||||
/// `Started` events report the currently completed count before the URL
|
/// `Started` events report the currently completed count before the URL
|
||||||
/// finishes; `Finished` events report the count after completion. This is
|
/// finishes; `Finished` events report the count after completion. This is
|
||||||
/// intentionally not the URL's plan position, because concurrent downloads
|
/// intentionally not the URL's plan position, so status percentages stay
|
||||||
/// finish out of plan order and status percentages must not move backward.
|
/// monotonic even if execution order or skip/resume mix changes.
|
||||||
pub index: usize,
|
pub index: usize,
|
||||||
/// Total URL count in the pull plan.
|
/// Total URL count in the pull plan.
|
||||||
pub total: usize,
|
pub total: usize,
|
||||||
@@ -2272,14 +2272,16 @@ exit 22
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn write_shell_script(path: &Path, script: &str) {
|
fn write_shell_script(path: &Path, script: &str) {
|
||||||
fs::write(path, script).unwrap();
|
let temp_path = path.with_extension("tmp");
|
||||||
|
fs::write(&temp_path, script).unwrap();
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
{
|
{
|
||||||
use std::os::unix::fs::PermissionsExt;
|
use std::os::unix::fs::PermissionsExt;
|
||||||
let mut permissions = fs::metadata(path).unwrap().permissions();
|
let mut permissions = fs::metadata(&temp_path).unwrap().permissions();
|
||||||
permissions.set_mode(0o755);
|
permissions.set_mode(0o755);
|
||||||
fs::set_permissions(path, permissions).unwrap();
|
fs::set_permissions(&temp_path, permissions).unwrap();
|
||||||
}
|
}
|
||||||
|
fs::rename(temp_path, path).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_fake_curl(path: &Path) {
|
fn write_fake_curl(path: &Path) {
|
||||||
|
|||||||
@@ -9,7 +9,8 @@ use crate::official_download::DownloadError;
|
|||||||
use crate::official_launcher::launcher_package_url;
|
use crate::official_launcher::launcher_package_url;
|
||||||
use crate::official_launcher::OfficialLauncherBootstrapService;
|
use crate::official_launcher::OfficialLauncherBootstrapService;
|
||||||
use crate::official_launcher::{
|
use crate::official_launcher::{
|
||||||
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherRemoteManifest,
|
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl,
|
||||||
|
YostarJpLauncherRemoteManifest,
|
||||||
};
|
};
|
||||||
use crate::zip_validation::validate_zip_structure;
|
use crate::zip_validation::validate_zip_structure;
|
||||||
use bat_adapters::official::game_main_config::YostarJpGameMainConfig;
|
use bat_adapters::official::game_main_config::YostarJpGameMainConfig;
|
||||||
@@ -39,12 +40,44 @@ pub struct OfficialGameMainConfigBootstrap {
|
|||||||
/// may instead point to a directory source plus per-file entries; in that
|
/// may instead point to a directory source plus per-file entries; in that
|
||||||
/// case this is the direct `resources.assets` URL.
|
/// case this is the direct `resources.assets` URL.
|
||||||
pub game_zip_url: String,
|
pub game_zip_url: String,
|
||||||
|
/// Remote launcher manifest used for this bootstrap.
|
||||||
|
pub remote_manifest: YostarJpLauncherRemoteManifest,
|
||||||
|
/// Exact source selected to obtain `GameMainConfig`.
|
||||||
|
pub selected_source: OfficialGameMainConfigSelectedSource,
|
||||||
/// Number of files declared by the remote manifest.
|
/// Number of files declared by the remote manifest.
|
||||||
pub manifest_file_count: usize,
|
pub manifest_file_count: usize,
|
||||||
/// Decrypted `GameMainConfig`.
|
/// Decrypted `GameMainConfig`.
|
||||||
pub game_main_config: YostarJpGameMainConfig,
|
pub game_main_config: YostarJpGameMainConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kind of official launcher package artifact selected for `GameMainConfig`.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum OfficialGameMainConfigSourceKind {
|
||||||
|
/// Older launcher manifests point to a single game ZIP archive.
|
||||||
|
Archive,
|
||||||
|
/// Current launcher manifests expose a directory plus per-file entries.
|
||||||
|
ManifestFile,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exact launcher artifact selected to obtain `resources.assets`.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct OfficialGameMainConfigSelectedSource {
|
||||||
|
/// Source kind.
|
||||||
|
pub kind: OfficialGameMainConfigSourceKind,
|
||||||
|
/// Official URL fetched for this source.
|
||||||
|
pub url: String,
|
||||||
|
/// Relative path under the official launcher package CDN root.
|
||||||
|
pub relative_path: String,
|
||||||
|
/// Original manifest file path when the source is a manifest file entry.
|
||||||
|
pub manifest_path: Option<String>,
|
||||||
|
/// Declared file size from the manifest, when available.
|
||||||
|
pub declared_size: Option<u64>,
|
||||||
|
/// Official launcher manifest `hash` field, when available.
|
||||||
|
pub official_hash: Option<String>,
|
||||||
|
/// Official launcher manifest per-file `vc`, when available.
|
||||||
|
pub vc: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Loads and decrypts the official `GameMainConfig` by following the official
|
/// Loads and decrypts the official `GameMainConfig` by following the official
|
||||||
/// launcher package chain.
|
/// launcher package chain.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -92,6 +125,18 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
/// launcher API, extracts `resources.assets`, and decrypts `GameMainConfig`.
|
/// launcher API, extracts `resources.assets`, and decrypts `GameMainConfig`.
|
||||||
pub fn fetch_bootstrap(&self) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
|
pub fn fetch_bootstrap(&self) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
|
||||||
let (game_config, manifest_url, manifest) = self.launcher.fetch_latest_remote_manifest()?;
|
let (game_config, manifest_url, manifest) = self.launcher.fetch_latest_remote_manifest()?;
|
||||||
|
let cdn_config = self.launcher.fetch_cdn_config()?;
|
||||||
|
self.fetch_bootstrap_from_parts(game_config, cdn_config, manifest_url, manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts `GameMainConfig` from already fetched launcher metadata.
|
||||||
|
pub fn fetch_bootstrap_from_parts(
|
||||||
|
&self,
|
||||||
|
game_config: YostarJpLauncherGameConfig,
|
||||||
|
cdn_config: YostarJpLauncherCdnConfig,
|
||||||
|
manifest_url: YostarJpLauncherManifestUrl,
|
||||||
|
manifest: YostarJpLauncherRemoteManifest,
|
||||||
|
) -> Result<OfficialGameMainConfigBootstrap, DownloadError> {
|
||||||
let manifest_source = manifest
|
let manifest_source = manifest
|
||||||
.source
|
.source
|
||||||
.clone()
|
.clone()
|
||||||
@@ -102,9 +147,8 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
"官方启动器远端 manifest 缺少 source",
|
"官方启动器远端 manifest 缺少 source",
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let cdn_config = self.launcher.fetch_cdn_config()?;
|
|
||||||
let temp_dir = TempDir::new().map_err(|error| format!("创建临时目录失败:{error}"))?;
|
let temp_dir = TempDir::new().map_err(|error| format!("创建临时目录失败:{error}"))?;
|
||||||
let (game_zip_url, resources_assets) = self.fetch_resources_assets(
|
let (game_zip_url, resources_assets, selected_source) = self.fetch_resources_assets(
|
||||||
&game_config,
|
&game_config,
|
||||||
&manifest,
|
&manifest,
|
||||||
&manifest_source,
|
&manifest_source,
|
||||||
@@ -113,6 +157,7 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
)?;
|
)?;
|
||||||
let game_main_config = YostarJpGameMainConfig::from_resources_assets(resources_assets)
|
let game_main_config = YostarJpGameMainConfig::from_resources_assets(resources_assets)
|
||||||
.map_err(|error| DownloadError::new(ErrorCode::GAME_MAIN_CONFIG_FAILED, error))?;
|
.map_err(|error| DownloadError::new(ErrorCode::GAME_MAIN_CONFIG_FAILED, error))?;
|
||||||
|
let manifest_file_count = manifest.files.len();
|
||||||
|
|
||||||
Ok(OfficialGameMainConfigBootstrap {
|
Ok(OfficialGameMainConfigBootstrap {
|
||||||
game_config,
|
game_config,
|
||||||
@@ -120,7 +165,9 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
manifest_url: manifest_url.url,
|
manifest_url: manifest_url.url,
|
||||||
manifest_source: Some(manifest_source),
|
manifest_source: Some(manifest_source),
|
||||||
game_zip_url,
|
game_zip_url,
|
||||||
manifest_file_count: manifest.files.len(),
|
remote_manifest: manifest,
|
||||||
|
selected_source,
|
||||||
|
manifest_file_count,
|
||||||
game_main_config,
|
game_main_config,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -203,10 +250,12 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
manifest_source: &str,
|
manifest_source: &str,
|
||||||
cdn_config: &YostarJpLauncherCdnConfig,
|
cdn_config: &YostarJpLauncherCdnConfig,
|
||||||
temp_root: &Path,
|
temp_root: &Path,
|
||||||
) -> Result<(String, PathBuf), DownloadError> {
|
) -> Result<(String, PathBuf, OfficialGameMainConfigSelectedSource), DownloadError> {
|
||||||
|
let selected_source =
|
||||||
|
resolve_game_main_config_source(game_config, manifest, manifest_source, cdn_config)?;
|
||||||
match select_game_main_config_source(game_config, manifest_source, manifest)? {
|
match select_game_main_config_source(game_config, manifest_source, manifest)? {
|
||||||
GameMainConfigSource::Archive(package_path) => {
|
GameMainConfigSource::Archive(package_path) => {
|
||||||
let game_zip_url = launcher_package_url(&cdn_config.primary_cdn, package_path)?;
|
let game_zip_url = selected_source.url.clone();
|
||||||
let archive_path = temp_root.join("official-game.zip");
|
let archive_path = temp_root.join("official-game.zip");
|
||||||
self.download_file_with_fallback(
|
self.download_file_with_fallback(
|
||||||
&game_zip_url,
|
&game_zip_url,
|
||||||
@@ -227,12 +276,11 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
"官方启动器包内没有找到 resources.assets",
|
"官方启动器包内没有找到 resources.assets",
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
Ok((game_zip_url, resources_assets))
|
Ok((game_zip_url, resources_assets, selected_source))
|
||||||
}
|
}
|
||||||
GameMainConfigSource::ManifestFile { source_dir, file } => {
|
GameMainConfigSource::ManifestFile { source_dir, file } => {
|
||||||
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)?;
|
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)?;
|
||||||
let resources_assets_url =
|
let resources_assets_url = selected_source.url.clone();
|
||||||
launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
|
|
||||||
let resources_assets = temp_root.join("resources.assets");
|
let resources_assets = temp_root.join("resources.assets");
|
||||||
self.download_file_with_fallback(
|
self.download_file_with_fallback(
|
||||||
&resources_assets_url,
|
&resources_assets_url,
|
||||||
@@ -241,7 +289,7 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
&resources_assets,
|
&resources_assets,
|
||||||
)?;
|
)?;
|
||||||
verify_manifest_file_size(&resources_assets, file)?;
|
verify_manifest_file_size(&resources_assets, file)?;
|
||||||
Ok((resources_assets_url, resources_assets))
|
Ok((resources_assets_url, resources_assets, selected_source))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -283,6 +331,49 @@ impl OfficialGameMainConfigBootstrapService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolves the exact official launcher artifact used to obtain `GameMainConfig`.
|
||||||
|
pub fn resolve_game_main_config_source(
|
||||||
|
game_config: &YostarJpLauncherGameConfig,
|
||||||
|
manifest: &YostarJpLauncherRemoteManifest,
|
||||||
|
manifest_source: &str,
|
||||||
|
cdn_config: &YostarJpLauncherCdnConfig,
|
||||||
|
) -> Result<OfficialGameMainConfigSelectedSource, DownloadError> {
|
||||||
|
match select_game_main_config_source(game_config, manifest_source, manifest)? {
|
||||||
|
GameMainConfigSource::Archive(package_path) => {
|
||||||
|
let url = launcher_package_url(&cdn_config.primary_cdn, package_path)?;
|
||||||
|
Ok(OfficialGameMainConfigSelectedSource {
|
||||||
|
kind: OfficialGameMainConfigSourceKind::Archive,
|
||||||
|
url,
|
||||||
|
relative_path: package_path.to_string(),
|
||||||
|
manifest_path: None,
|
||||||
|
declared_size: None,
|
||||||
|
official_hash: None,
|
||||||
|
vc: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
GameMainConfigSource::ManifestFile { source_dir, file } => {
|
||||||
|
let relative_path = launcher_manifest_file_relative_path(source_dir, &file.path)
|
||||||
|
.map_err(|error| DownloadError::new(ErrorCode::LAUNCHER_RESPONSE_INVALID, error))?;
|
||||||
|
let url = launcher_package_url(&cdn_config.primary_cdn, &relative_path)?;
|
||||||
|
let declared_size = file.size.parse::<u64>().map_err(|error| {
|
||||||
|
DownloadError::new(
|
||||||
|
ErrorCode::LAUNCHER_RESPONSE_INVALID,
|
||||||
|
format!("官方启动器 manifest 中 {} 的 size 无效:{error}", file.path),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
Ok(OfficialGameMainConfigSelectedSource {
|
||||||
|
kind: OfficialGameMainConfigSourceKind::ManifestFile,
|
||||||
|
url,
|
||||||
|
relative_path,
|
||||||
|
manifest_path: Some(file.path.clone()),
|
||||||
|
declared_size: Some(declared_size),
|
||||||
|
official_hash: Some(file.hash.clone()),
|
||||||
|
vc: file.vc.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
enum GameMainConfigSource<'a> {
|
enum GameMainConfigSource<'a> {
|
||||||
Archive(&'a str),
|
Archive(&'a str),
|
||||||
|
|||||||
@@ -10,9 +10,11 @@ use crate::path_security::{
|
|||||||
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
||||||
STATE_FILE_MODE,
|
STATE_FILE_MODE,
|
||||||
};
|
};
|
||||||
use bat_assetbundle::{Parser, UnityFsParser};
|
use bat_assetbundle::{
|
||||||
|
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
|
||||||
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
@@ -20,7 +22,11 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
|||||||
/// Parse-cache file name stored under a published official resource root.
|
/// Parse-cache file name stored under a published official resource root.
|
||||||
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
|
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
|
||||||
/// Current parse-cache schema version.
|
/// Current parse-cache schema version.
|
||||||
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1;
|
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
|
||||||
|
/// TextUnit detail index file name stored under a published official resource root.
|
||||||
|
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
|
||||||
|
/// Current TextUnit detail-index schema version.
|
||||||
|
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
|
||||||
|
|
||||||
/// Configuration for one official resource parse-cache refresh.
|
/// Configuration for one official resource parse-cache refresh.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -55,6 +61,10 @@ pub struct OfficialParseReport {
|
|||||||
pub cache_path: PathBuf,
|
pub cache_path: PathBuf,
|
||||||
/// Aggregate parse-cache summary.
|
/// Aggregate parse-cache summary.
|
||||||
pub summary: OfficialParseSummary,
|
pub summary: OfficialParseSummary,
|
||||||
|
/// TextUnit detail-index path written by the refresh.
|
||||||
|
pub textunit_index_path: PathBuf,
|
||||||
|
/// Aggregate TextUnit detail-index summary.
|
||||||
|
pub textunit_index_summary: OfficialTextUnitIndexSummary,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Aggregate counters for a parse-cache refresh.
|
/// Aggregate counters for a parse-cache refresh.
|
||||||
@@ -78,6 +88,15 @@ pub struct OfficialParseSummary {
|
|||||||
pub failed_count: usize,
|
pub failed_count: usize,
|
||||||
/// Total TextAsset objects found in parsed Unity serialized files.
|
/// Total TextAsset objects found in parsed Unity serialized files.
|
||||||
pub text_asset_count: usize,
|
pub text_asset_count: usize,
|
||||||
|
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
|
||||||
|
#[serde(default)]
|
||||||
|
pub skipped_binary_text_asset_count: usize,
|
||||||
|
/// Number of non-fatal TypeTree field extraction diagnostics.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_unit_error_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persistent parse cache for one official resource root.
|
/// Persistent parse cache for one official resource root.
|
||||||
@@ -127,10 +146,162 @@ pub struct OfficialParseCacheEntry {
|
|||||||
pub text_assets: Vec<String>,
|
pub text_assets: Vec<String>,
|
||||||
/// Non-fatal serialized-file parse diagnostic count.
|
/// Non-fatal serialized-file parse diagnostic count.
|
||||||
pub serialized_parse_error_count: usize,
|
pub serialized_parse_error_count: usize,
|
||||||
|
/// Number of translation-ready TextUnit entries extracted from this bundle.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_unit_formats: Vec<String>,
|
||||||
|
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
|
||||||
|
#[serde(default)]
|
||||||
|
pub skipped_binary_text_asset_count: usize,
|
||||||
|
/// Non-fatal TypeTree field extraction diagnostic count.
|
||||||
|
#[serde(default)]
|
||||||
|
pub text_unit_error_count: usize,
|
||||||
/// Human-readable error or skip reason.
|
/// Human-readable error or skip reason.
|
||||||
pub error: Option<String>,
|
pub error: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Persistent TextUnit detail index for one official resource root.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitIndex {
|
||||||
|
/// Index schema version.
|
||||||
|
#[serde(default = "default_textunit_index_version")]
|
||||||
|
pub version: u32,
|
||||||
|
/// Index generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Official resource root used to generate this index.
|
||||||
|
pub resource_root: PathBuf,
|
||||||
|
/// Aggregate counters for this index.
|
||||||
|
pub summary: OfficialTextUnitIndexSummary,
|
||||||
|
/// Extracted TextUnit details in deterministic order.
|
||||||
|
#[serde(default)]
|
||||||
|
pub units: Vec<OfficialTextUnitIndexUnit>,
|
||||||
|
/// Parse and extraction diagnostics in deterministic order.
|
||||||
|
#[serde(default)]
|
||||||
|
pub errors: Vec<OfficialTextUnitIndexError>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate counters for a TextUnit detail index.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitIndexSummary {
|
||||||
|
/// Number of TextUnit detail entries.
|
||||||
|
pub unit_count: usize,
|
||||||
|
/// Number of parse/extraction diagnostics.
|
||||||
|
pub error_count: usize,
|
||||||
|
/// Number of binary or invalid TextAsset payloads skipped by extraction.
|
||||||
|
pub skipped_binary_text_asset_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One TextUnit detail entry.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitIndexUnit {
|
||||||
|
/// Stable unit ID within the official release.
|
||||||
|
pub id: String,
|
||||||
|
/// Parse-cache entry key that produced this unit.
|
||||||
|
pub parse_entry_key: String,
|
||||||
|
/// Official URL from the download manifest.
|
||||||
|
pub source_url: String,
|
||||||
|
/// Relative destination path under the official resource root.
|
||||||
|
pub destination: String,
|
||||||
|
/// Inner archive path when the source is a zip file.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Source classification used by the parser.
|
||||||
|
pub source_kind: OfficialParseSourceKind,
|
||||||
|
/// Unity editor version when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub unity_version: Option<String>,
|
||||||
|
/// Original source text.
|
||||||
|
pub source_text: String,
|
||||||
|
/// Unity serialized file path.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub serialized_file: Option<String>,
|
||||||
|
/// Unity object path ID.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Unity class ID.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// TypeTree field path.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Byte offset relative to the beginning of the Unity object payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_offset: Option<usize>,
|
||||||
|
/// Number of bytes consumed by this field, including alignment padding.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_byte_size: Option<usize>,
|
||||||
|
/// TextUnit payload format such as json/csv/tsv/plain.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub format: Option<String>,
|
||||||
|
/// Extraction source kind such as TextAsset or TypeTreeField.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_source_kind: Option<String>,
|
||||||
|
/// TextAsset name when the unit came from a TextAsset payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub asset_name: Option<String>,
|
||||||
|
/// Stable extraction context copied from the parser.
|
||||||
|
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||||
|
pub context: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One parse or extraction diagnostic in the TextUnit detail index.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitIndexError {
|
||||||
|
/// Stable diagnostic ID within the official release.
|
||||||
|
pub id: String,
|
||||||
|
/// Parse-cache entry key associated with this diagnostic.
|
||||||
|
pub parse_entry_key: String,
|
||||||
|
/// Official URL from the download manifest.
|
||||||
|
pub source_url: String,
|
||||||
|
/// Relative destination path under the official resource root.
|
||||||
|
pub destination: String,
|
||||||
|
/// Inner archive path when the source is a zip file.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Source classification used by the parser.
|
||||||
|
pub source_kind: OfficialParseSourceKind,
|
||||||
|
/// Parse status associated with this diagnostic.
|
||||||
|
pub status: OfficialParseStatus,
|
||||||
|
/// Unity serialized file path when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub serialized_file: Option<String>,
|
||||||
|
/// Unity object path ID when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Unity class ID when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// TypeTree field path when known.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Byte offset relative to the beginning of the Unity object payload.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub offset: Option<usize>,
|
||||||
|
/// Human-readable diagnostic.
|
||||||
|
pub error: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Query filters for TextUnit detail entries.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct OfficialTextUnitQuery {
|
||||||
|
/// Filter by destination path.
|
||||||
|
pub destination: Option<String>,
|
||||||
|
/// Filter by destination glob pattern.
|
||||||
|
pub path_pattern: Option<String>,
|
||||||
|
/// Filter by archive entry.
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Filter by Unity object path ID.
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Filter by Unity class ID.
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// Filter by field path.
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Filter by TextUnit format.
|
||||||
|
pub format: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Source kind for a parse-cache entry.
|
/// Source kind for a parse-cache entry.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -187,21 +358,36 @@ impl OfficialParseCacheService {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let previous_cache = read_parse_cache_at(&config.resource_root)?;
|
let previous_cache = read_parse_cache_at(&config.resource_root)?;
|
||||||
|
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
|
||||||
let mut summary = OfficialParseSummary {
|
let mut summary = OfficialParseSummary {
|
||||||
manifest_entry_count: manifest.entries.len(),
|
manifest_entry_count: manifest.entries.len(),
|
||||||
..OfficialParseSummary::default()
|
..OfficialParseSummary::default()
|
||||||
};
|
};
|
||||||
let mut entries = BTreeMap::new();
|
let mut entries = BTreeMap::new();
|
||||||
|
let mut units = Vec::new();
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
|
||||||
for manifest_entry in manifest.entries.values() {
|
for manifest_entry in manifest.entries.values() {
|
||||||
let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref());
|
let produced = process_manifest_entry(
|
||||||
for entry in produced {
|
config,
|
||||||
summary.record_entry(&entry);
|
manifest_entry,
|
||||||
entries.insert(entry.key.clone(), entry);
|
previous_cache.as_ref(),
|
||||||
|
previous_textunit_index.as_ref(),
|
||||||
|
);
|
||||||
|
for produced in produced {
|
||||||
|
summary.record_entry(&produced.entry);
|
||||||
|
units.extend(produced.units);
|
||||||
|
errors.extend(produced.errors);
|
||||||
|
entries.insert(produced.entry.key.clone(), produced.entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
summary.cache_entry_count = entries.len();
|
summary.cache_entry_count = entries.len();
|
||||||
|
let textunit_index_summary = OfficialTextUnitIndexSummary {
|
||||||
|
unit_count: units.len(),
|
||||||
|
error_count: errors.len(),
|
||||||
|
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
|
||||||
|
};
|
||||||
let cache = OfficialParseCache {
|
let cache = OfficialParseCache {
|
||||||
version: OFFICIAL_PARSE_CACHE_VERSION,
|
version: OFFICIAL_PARSE_CACHE_VERSION,
|
||||||
generated_unix_seconds: unix_seconds_now(),
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
@@ -209,15 +395,67 @@ impl OfficialParseCacheService {
|
|||||||
entries,
|
entries,
|
||||||
};
|
};
|
||||||
write_parse_cache_at(&config.resource_root, &cache)?;
|
write_parse_cache_at(&config.resource_root, &cache)?;
|
||||||
|
let textunit_index = OfficialTextUnitIndex {
|
||||||
|
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
|
||||||
|
generated_unix_seconds: cache.generated_unix_seconds,
|
||||||
|
resource_root: config.resource_root.clone(),
|
||||||
|
summary: textunit_index_summary.clone(),
|
||||||
|
units,
|
||||||
|
errors,
|
||||||
|
};
|
||||||
|
write_textunit_index_at(&config.resource_root, &textunit_index)?;
|
||||||
|
|
||||||
Ok(OfficialParseReport {
|
Ok(OfficialParseReport {
|
||||||
resource_root: config.resource_root.clone(),
|
resource_root: config.resource_root.clone(),
|
||||||
cache_path: config.cache_path(),
|
cache_path: config.cache_path(),
|
||||||
summary,
|
summary,
|
||||||
|
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
|
||||||
|
textunit_index_summary,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ParseProduced {
|
||||||
|
entry: OfficialParseCacheEntry,
|
||||||
|
units: Vec<OfficialTextUnitIndexUnit>,
|
||||||
|
errors: Vec<OfficialTextUnitIndexError>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ParseProduced {
|
||||||
|
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
|
||||||
|
let errors = error_from_status_entry(&entry).into_iter().collect();
|
||||||
|
Self {
|
||||||
|
entry,
|
||||||
|
units: Vec::new(),
|
||||||
|
errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_cached(
|
||||||
|
mut entry: OfficialParseCacheEntry,
|
||||||
|
previous_index: Option<&OfficialTextUnitIndex>,
|
||||||
|
) -> Self {
|
||||||
|
entry.reused_from_previous_cache = true;
|
||||||
|
let units = previous_index
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|index| index.units.iter())
|
||||||
|
.filter(|unit| unit.parse_entry_key == entry.key)
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
let errors = previous_index
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|index| index.errors.iter())
|
||||||
|
.filter(|error| error.parse_entry_key == entry.key)
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
Self {
|
||||||
|
entry,
|
||||||
|
units,
|
||||||
|
errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl OfficialParseSummary {
|
impl OfficialParseSummary {
|
||||||
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
|
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
|
||||||
match entry.source_kind {
|
match entry.source_kind {
|
||||||
@@ -234,6 +472,9 @@ impl OfficialParseSummary {
|
|||||||
OfficialParseStatus::Parsed => {
|
OfficialParseStatus::Parsed => {
|
||||||
self.parsed_bundle_count += 1;
|
self.parsed_bundle_count += 1;
|
||||||
self.text_asset_count += entry.text_asset_count;
|
self.text_asset_count += entry.text_asset_count;
|
||||||
|
self.text_unit_count += entry.text_unit_count;
|
||||||
|
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
|
||||||
|
self.text_unit_error_count += entry.text_unit_error_count;
|
||||||
}
|
}
|
||||||
OfficialParseStatus::SkippedUnsupported => {
|
OfficialParseStatus::SkippedUnsupported => {
|
||||||
self.unsupported_count += 1;
|
self.unsupported_count += 1;
|
||||||
@@ -279,21 +520,81 @@ pub fn write_parse_cache_at(
|
|||||||
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads the TextUnit detail index under a published official resource root.
|
||||||
|
pub fn read_textunit_index_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
) -> Result<Option<OfficialTextUnitIndex>, String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||||||
|
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Ok(Some(index))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes the TextUnit detail index under a published official resource root.
|
||||||
|
pub fn write_textunit_index_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
index: &OfficialTextUnitIndex,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
|
||||||
|
ensure_path_within_root(resource_root, &path)?;
|
||||||
|
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
|
||||||
|
let bytes = serde_json::to_vec_pretty(index)
|
||||||
|
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
|
||||||
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns TextUnit detail entries matching a query.
|
||||||
|
pub fn query_textunit_index_units<'a>(
|
||||||
|
index: &'a OfficialTextUnitIndex,
|
||||||
|
query: &OfficialTextUnitQuery,
|
||||||
|
) -> Vec<&'a OfficialTextUnitIndexUnit> {
|
||||||
|
index
|
||||||
|
.units
|
||||||
|
.iter()
|
||||||
|
.filter(|unit| textunit_unit_matches(unit, query))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns parse/extraction diagnostics matching a query.
|
||||||
|
pub fn query_textunit_index_errors<'a>(
|
||||||
|
index: &'a OfficialTextUnitIndex,
|
||||||
|
query: &OfficialTextUnitQuery,
|
||||||
|
) -> Vec<&'a OfficialTextUnitIndexError> {
|
||||||
|
index
|
||||||
|
.errors
|
||||||
|
.iter()
|
||||||
|
.filter(|error| textunit_error_matches(error, query))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn process_manifest_entry(
|
fn process_manifest_entry(
|
||||||
config: &OfficialParseConfig,
|
config: &OfficialParseConfig,
|
||||||
manifest_entry: &OfficialDownloadManifestEntry,
|
manifest_entry: &OfficialDownloadManifestEntry,
|
||||||
previous_cache: Option<&OfficialParseCache>,
|
previous_cache: Option<&OfficialParseCache>,
|
||||||
) -> Vec<OfficialParseCacheEntry> {
|
previous_index: Option<&OfficialTextUnitIndex>,
|
||||||
|
) -> Vec<ParseProduced> {
|
||||||
let fingerprint = fingerprint_for(manifest_entry);
|
let fingerprint = fingerprint_for(manifest_entry);
|
||||||
if looks_like_zip_source(manifest_entry) {
|
if looks_like_zip_source(manifest_entry) {
|
||||||
return process_zip_entry(config, manifest_entry, previous_cache, fingerprint);
|
return process_zip_entry(
|
||||||
|
config,
|
||||||
|
manifest_entry,
|
||||||
|
previous_cache,
|
||||||
|
previous_index,
|
||||||
|
fingerprint,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if looks_like_direct_bundle_source(manifest_entry) {
|
if looks_like_direct_bundle_source(manifest_entry) {
|
||||||
let key = direct_key(&manifest_entry.url);
|
let key = direct_key(&manifest_entry.url);
|
||||||
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||||
cached.reused_from_previous_cache = true;
|
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||||||
return vec![cached];
|
|
||||||
}
|
}
|
||||||
return vec![parse_direct_bundle(
|
return vec![parse_direct_bundle(
|
||||||
config,
|
config,
|
||||||
@@ -304,61 +605,58 @@ fn process_manifest_entry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let key = unsupported_key(&manifest_entry.url);
|
let key = unsupported_key(&manifest_entry.url);
|
||||||
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
|
||||||
cached.reused_from_previous_cache = true;
|
return vec![ParseProduced::from_cached(cached, previous_index)];
|
||||||
return vec![cached];
|
|
||||||
}
|
}
|
||||||
vec![unsupported_entry(
|
vec![ParseProduced::from_entry(unsupported_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::Unsupported,
|
OfficialParseSourceKind::Unsupported,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
"非 UnityFS 候选资源",
|
"非 UnityFS 候选资源",
|
||||||
)]
|
))]
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_zip_entry(
|
fn process_zip_entry(
|
||||||
config: &OfficialParseConfig,
|
config: &OfficialParseConfig,
|
||||||
manifest_entry: &OfficialDownloadManifestEntry,
|
manifest_entry: &OfficialDownloadManifestEntry,
|
||||||
previous_cache: Option<&OfficialParseCache>,
|
previous_cache: Option<&OfficialParseCache>,
|
||||||
|
previous_index: Option<&OfficialTextUnitIndex>,
|
||||||
fingerprint: OfficialParseSourceFingerprint,
|
fingerprint: OfficialParseSourceFingerprint,
|
||||||
) -> Vec<OfficialParseCacheEntry> {
|
) -> Vec<ParseProduced> {
|
||||||
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
|
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
|
||||||
if !cached_entries.is_empty() {
|
if !cached_entries.is_empty() {
|
||||||
return cached_entries
|
return cached_entries
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|mut entry| {
|
.map(|entry| ParseProduced::from_cached(entry, previous_index))
|
||||||
entry.reused_from_previous_cache = true;
|
|
||||||
entry
|
|
||||||
})
|
|
||||||
.collect();
|
.collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
|
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||||||
Ok(path) => path,
|
Ok(path) => path,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return vec![failed_entry(
|
return vec![ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::ZipEntry,
|
OfficialParseSourceKind::ZipEntry,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
zip_list_key(&manifest_entry.url),
|
zip_list_key(&manifest_entry.url),
|
||||||
error,
|
error,
|
||||||
)]
|
))]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
|
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
|
||||||
Ok(entries) => entries,
|
Ok(entries) => entries,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return vec![failed_entry(
|
return vec![ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::ZipEntry,
|
OfficialParseSourceKind::ZipEntry,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
zip_list_key(&manifest_entry.url),
|
zip_list_key(&manifest_entry.url),
|
||||||
error,
|
error,
|
||||||
)]
|
))]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -369,14 +667,14 @@ fn process_zip_entry(
|
|||||||
{
|
{
|
||||||
Ok(bytes) => bytes,
|
Ok(bytes) => bytes,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
produced.push(failed_entry(
|
produced.push(ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
Some(archive_entry),
|
Some(archive_entry),
|
||||||
OfficialParseSourceKind::ZipEntry,
|
OfficialParseSourceKind::ZipEntry,
|
||||||
fingerprint.clone(),
|
fingerprint.clone(),
|
||||||
key,
|
key,
|
||||||
error,
|
error,
|
||||||
));
|
)));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -390,14 +688,14 @@ fn process_zip_entry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if produced.is_empty() {
|
if produced.is_empty() {
|
||||||
produced.push(unsupported_entry(
|
produced.push(ParseProduced::from_entry(unsupported_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::Unsupported,
|
OfficialParseSourceKind::Unsupported,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
zip_list_key(&manifest_entry.url),
|
zip_list_key(&manifest_entry.url),
|
||||||
"ZIP 内没有可检查文件条目",
|
"ZIP 内没有可检查文件条目",
|
||||||
));
|
)));
|
||||||
}
|
}
|
||||||
produced
|
produced
|
||||||
}
|
}
|
||||||
@@ -407,42 +705,42 @@ fn parse_direct_bundle(
|
|||||||
manifest_entry: &OfficialDownloadManifestEntry,
|
manifest_entry: &OfficialDownloadManifestEntry,
|
||||||
key: String,
|
key: String,
|
||||||
fingerprint: OfficialParseSourceFingerprint,
|
fingerprint: OfficialParseSourceFingerprint,
|
||||||
) -> OfficialParseCacheEntry {
|
) -> ParseProduced {
|
||||||
let path = match resource_path_for(&config.resource_root, manifest_entry) {
|
let path = match resource_path_for(&config.resource_root, manifest_entry) {
|
||||||
Ok(path) => path,
|
Ok(path) => path,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return failed_entry(
|
return ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::DirectBundle,
|
OfficialParseSourceKind::DirectBundle,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
error,
|
error,
|
||||||
)
|
))
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let bytes = match read_resource_file(&path) {
|
let bytes = match read_resource_file(&path) {
|
||||||
Ok(bytes) => bytes,
|
Ok(bytes) => bytes,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return failed_entry(
|
return ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::DirectBundle,
|
OfficialParseSourceKind::DirectBundle,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
error,
|
error,
|
||||||
)
|
))
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if !UnityFsParser::has_unityfs_signature(&bytes) {
|
if !UnityFsParser::has_unityfs_signature(&bytes) {
|
||||||
return unsupported_entry(
|
return ParseProduced::from_entry(unsupported_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
None,
|
None,
|
||||||
OfficialParseSourceKind::DirectBundle,
|
OfficialParseSourceKind::DirectBundle,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
"文件不是 UnityFS bundle",
|
"文件不是 UnityFS bundle",
|
||||||
);
|
));
|
||||||
}
|
}
|
||||||
parsed_bundle_entry(
|
parsed_bundle_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
@@ -460,16 +758,16 @@ fn parse_zip_inner_file(
|
|||||||
fingerprint: OfficialParseSourceFingerprint,
|
fingerprint: OfficialParseSourceFingerprint,
|
||||||
key: String,
|
key: String,
|
||||||
bytes: &[u8],
|
bytes: &[u8],
|
||||||
) -> OfficialParseCacheEntry {
|
) -> ParseProduced {
|
||||||
if !UnityFsParser::has_unityfs_signature(bytes) {
|
if !UnityFsParser::has_unityfs_signature(bytes) {
|
||||||
return unsupported_entry(
|
return ParseProduced::from_entry(unsupported_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
Some(archive_entry),
|
Some(archive_entry),
|
||||||
OfficialParseSourceKind::ZipEntry,
|
OfficialParseSourceKind::ZipEntry,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
"ZIP 条目不是 UnityFS bundle",
|
"ZIP 条目不是 UnityFS bundle",
|
||||||
);
|
));
|
||||||
}
|
}
|
||||||
parsed_bundle_entry(
|
parsed_bundle_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
@@ -488,10 +786,24 @@ fn parsed_bundle_entry(
|
|||||||
fingerprint: OfficialParseSourceFingerprint,
|
fingerprint: OfficialParseSourceFingerprint,
|
||||||
key: String,
|
key: String,
|
||||||
bytes: &[u8],
|
bytes: &[u8],
|
||||||
) -> OfficialParseCacheEntry {
|
) -> ParseProduced {
|
||||||
let parser = UnityFsParser::new();
|
let parser = UnityFsParser::new();
|
||||||
match parser.parse(bytes) {
|
match parser.parse(bytes) {
|
||||||
Ok(parsed) => OfficialParseCacheEntry {
|
Ok(parsed) => {
|
||||||
|
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
|
||||||
|
&parsed,
|
||||||
|
Some(&manifest_entry.destination),
|
||||||
|
archive_entry.as_deref(),
|
||||||
|
);
|
||||||
|
let text_unit_formats = text_units
|
||||||
|
.units
|
||||||
|
.iter()
|
||||||
|
.filter_map(|unit| unit.context.get("format").cloned())
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let entry = OfficialParseCacheEntry {
|
||||||
key,
|
key,
|
||||||
source_url: manifest_entry.url.clone(),
|
source_url: manifest_entry.url.clone(),
|
||||||
destination: manifest_entry.destination.clone(),
|
destination: manifest_entry.destination.clone(),
|
||||||
@@ -510,16 +822,218 @@ fn parsed_bundle_entry(
|
|||||||
.map(|asset| asset.name.clone())
|
.map(|asset| asset.name.clone())
|
||||||
.collect(),
|
.collect(),
|
||||||
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
|
||||||
|
text_unit_count: text_units.units.len(),
|
||||||
|
text_unit_formats,
|
||||||
|
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
|
||||||
|
text_unit_error_count: text_units.errors.len(),
|
||||||
error: None,
|
error: None,
|
||||||
},
|
};
|
||||||
Err(error) => failed_entry(
|
let units = indexed_text_units_for_entry(&entry, &text_units.units);
|
||||||
|
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
|
||||||
|
ParseProduced {
|
||||||
|
entry,
|
||||||
|
units,
|
||||||
|
errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => ParseProduced::from_entry(failed_entry(
|
||||||
manifest_entry,
|
manifest_entry,
|
||||||
archive_entry,
|
archive_entry,
|
||||||
source_kind,
|
source_kind,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
key,
|
key,
|
||||||
error.to_string(),
|
error.to_string(),
|
||||||
),
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn indexed_text_units_for_entry(
|
||||||
|
entry: &OfficialParseCacheEntry,
|
||||||
|
units: &[TextUnit],
|
||||||
|
) -> Vec<OfficialTextUnitIndexUnit> {
|
||||||
|
units
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, unit)| OfficialTextUnitIndexUnit {
|
||||||
|
id: format!("{}#unit:{index}", entry.key),
|
||||||
|
parse_entry_key: entry.key.clone(),
|
||||||
|
source_url: entry.source_url.clone(),
|
||||||
|
destination: entry.destination.clone(),
|
||||||
|
archive_entry: entry.archive_entry.clone(),
|
||||||
|
source_kind: entry.source_kind,
|
||||||
|
unity_version: entry.unity_version.clone(),
|
||||||
|
source_text: unit.source_text.clone(),
|
||||||
|
serialized_file: unit.serialized_file.clone(),
|
||||||
|
path_id: unit.path_id,
|
||||||
|
class_id: unit.class_id,
|
||||||
|
field_path: unit.field_path.clone(),
|
||||||
|
field_offset: unit.field_offset,
|
||||||
|
field_byte_size: unit.field_byte_size,
|
||||||
|
format: unit.context.get("format").cloned(),
|
||||||
|
text_source_kind: unit.context.get("source_kind").cloned(),
|
||||||
|
asset_name: unit.context.get("asset_name").cloned(),
|
||||||
|
context: unit.context.clone(),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn indexed_extraction_errors_for_entry(
|
||||||
|
entry: &OfficialParseCacheEntry,
|
||||||
|
errors: &[TextUnitExtractionError],
|
||||||
|
) -> Vec<OfficialTextUnitIndexError> {
|
||||||
|
errors
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, error)| OfficialTextUnitIndexError {
|
||||||
|
id: format!("{}#extract-error:{index}", entry.key),
|
||||||
|
parse_entry_key: entry.key.clone(),
|
||||||
|
source_url: entry.source_url.clone(),
|
||||||
|
destination: entry.destination.clone(),
|
||||||
|
archive_entry: entry.archive_entry.clone(),
|
||||||
|
source_kind: entry.source_kind,
|
||||||
|
status: entry.status,
|
||||||
|
serialized_file: error.serialized_file.clone(),
|
||||||
|
path_id: error.path_id,
|
||||||
|
class_id: error.class_id,
|
||||||
|
field_path: error.field_path.clone(),
|
||||||
|
offset: error.offset,
|
||||||
|
error: error.error.clone(),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
|
||||||
|
let error = entry.error.as_ref()?;
|
||||||
|
Some(OfficialTextUnitIndexError {
|
||||||
|
id: format!("{}#status", entry.key),
|
||||||
|
parse_entry_key: entry.key.clone(),
|
||||||
|
source_url: entry.source_url.clone(),
|
||||||
|
destination: entry.destination.clone(),
|
||||||
|
archive_entry: entry.archive_entry.clone(),
|
||||||
|
source_kind: entry.source_kind,
|
||||||
|
status: entry.status,
|
||||||
|
serialized_file: None,
|
||||||
|
path_id: None,
|
||||||
|
class_id: None,
|
||||||
|
field_path: None,
|
||||||
|
offset: None,
|
||||||
|
error: error.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
|
||||||
|
if query
|
||||||
|
.destination
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|destination| &unit.destination != destination)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.path_pattern
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.archive_entry
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.path_id
|
||||||
|
.is_some_and(|path_id| unit.path_id != Some(path_id))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.class_id
|
||||||
|
.is_some_and(|class_id| unit.class_id != Some(class_id))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.field_path
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.format
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|format| unit.format.as_ref() != Some(format))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn textunit_error_matches(
|
||||||
|
error: &OfficialTextUnitIndexError,
|
||||||
|
query: &OfficialTextUnitQuery,
|
||||||
|
) -> bool {
|
||||||
|
if query
|
||||||
|
.destination
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|destination| &error.destination != destination)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.path_pattern
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.archive_entry
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.path_id
|
||||||
|
.is_some_and(|path_id| error.path_id != Some(path_id))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.class_id
|
||||||
|
.is_some_and(|class_id| error.class_id != Some(class_id))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if query
|
||||||
|
.field_path
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn glob_matches(pattern: &str, value: &str) -> bool {
|
||||||
|
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
|
||||||
|
match pattern.split_first() {
|
||||||
|
None => value.is_empty(),
|
||||||
|
Some((&b'*', rest)) => {
|
||||||
|
glob_matches_bytes(rest, value)
|
||||||
|
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
|
||||||
|
}
|
||||||
|
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
|
||||||
|
Some((&literal, rest)) => value
|
||||||
|
.split_first()
|
||||||
|
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -585,6 +1099,10 @@ fn status_entry(
|
|||||||
text_asset_count: 0,
|
text_asset_count: 0,
|
||||||
text_assets: Vec::new(),
|
text_assets: Vec::new(),
|
||||||
serialized_parse_error_count: 0,
|
serialized_parse_error_count: 0,
|
||||||
|
text_unit_count: 0,
|
||||||
|
text_unit_formats: Vec::new(),
|
||||||
|
skipped_binary_text_asset_count: 0,
|
||||||
|
text_unit_error_count: 0,
|
||||||
error: Some(reason),
|
error: Some(reason),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -735,6 +1253,10 @@ fn default_parse_cache_version() -> u32 {
|
|||||||
OFFICIAL_PARSE_CACHE_VERSION
|
OFFICIAL_PARSE_CACHE_VERSION
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_textunit_index_version() -> u32 {
|
||||||
|
OFFICIAL_TEXTUNIT_INDEX_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
fn unix_seconds_now() -> u64 {
|
fn unix_seconds_now() -> u64 {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
|
|||||||
@@ -0,0 +1,543 @@
|
|||||||
|
//! Import of a verified official release into CAS and ResourceRepository.
|
||||||
|
|
||||||
|
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
|
||||||
|
use crate::official_parse::{
|
||||||
|
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus,
|
||||||
|
OfficialParseSummary,
|
||||||
|
};
|
||||||
|
use crate::path_security::{
|
||||||
|
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink,
|
||||||
|
};
|
||||||
|
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
|
||||||
|
use bat_core::repositories::{CasRepository, ResourceRepository};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
/// Configuration for importing one already-published official release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct OfficialReleaseImportConfig {
|
||||||
|
/// Published release root containing `official-download-manifest.json`.
|
||||||
|
pub release_root: PathBuf,
|
||||||
|
/// Official release ID associated with this root, when known.
|
||||||
|
pub official_release_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialReleaseImportConfig {
|
||||||
|
/// Creates an import configuration.
|
||||||
|
pub fn new(release_root: impl Into<PathBuf>) -> Self {
|
||||||
|
Self {
|
||||||
|
release_root: release_root.into(),
|
||||||
|
official_release_id: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attaches the official release ID that should be stored in resource metadata.
|
||||||
|
pub fn with_official_release_id(mut self, release_id: impl Into<String>) -> Self {
|
||||||
|
self.official_release_id = Some(release_id.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Summary returned by an official release repository import.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialReleaseImportReport {
|
||||||
|
/// Number of verified manifest entries.
|
||||||
|
pub manifest_entry_count: usize,
|
||||||
|
/// Number of new or changed repository rows.
|
||||||
|
pub imported_count: usize,
|
||||||
|
/// Number of rows already pointing at the same CAS object.
|
||||||
|
pub unchanged_count: usize,
|
||||||
|
/// Number of unchanged rows whose metadata was refreshed from parse cache.
|
||||||
|
#[serde(default)]
|
||||||
|
pub metadata_updated_count: usize,
|
||||||
|
/// Number of resources classified as AssetBundle.
|
||||||
|
pub asset_bundle_count: usize,
|
||||||
|
/// Number of resources classified as text-like payloads.
|
||||||
|
pub text_asset_count: usize,
|
||||||
|
/// Number of resources classified as tables.
|
||||||
|
pub table_count: usize,
|
||||||
|
/// Number of resources classified as media.
|
||||||
|
pub media_count: usize,
|
||||||
|
/// Parse-cache summary associated with this release, when available.
|
||||||
|
pub parse_summary: Option<OfficialParseSummary>,
|
||||||
|
/// Non-fatal cleanup warnings, such as an old CAS reference that could not
|
||||||
|
/// be decremented after a successful row replacement.
|
||||||
|
pub warnings: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Imports verified official resources into CAS and the resource index.
|
||||||
|
pub struct OfficialReleaseImportService<'a> {
|
||||||
|
cas: &'a dyn CasRepository,
|
||||||
|
resources: &'a dyn ResourceRepository,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> OfficialReleaseImportService<'a> {
|
||||||
|
/// Creates an import service.
|
||||||
|
pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self {
|
||||||
|
Self { cas, resources }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Imports every entry in one published release manifest.
|
||||||
|
///
|
||||||
|
/// The source files remain untouched. Every file is checked against the
|
||||||
|
/// verified download manifest before it can enter CAS. Repository IDs are
|
||||||
|
/// stable by destination, making repeated imports idempotent.
|
||||||
|
pub async fn import_release(
|
||||||
|
&self,
|
||||||
|
config: &OfficialReleaseImportConfig,
|
||||||
|
) -> bat_core::Result<OfficialReleaseImportReport> {
|
||||||
|
let manifest = read_download_manifest_at(&config.release_root)
|
||||||
|
.map_err(bat_core::Error::InvalidArgument)?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
bat_core::Error::NotFound(format!(
|
||||||
|
"官方下载 manifest 不存在:{}",
|
||||||
|
config.release_root.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let parse_cache =
|
||||||
|
read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?;
|
||||||
|
let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone());
|
||||||
|
let parse_entries_by_destination = parse_cache
|
||||||
|
.as_ref()
|
||||||
|
.map(parse_entries_by_destination)
|
||||||
|
.unwrap_or_default();
|
||||||
|
let release_id = config
|
||||||
|
.official_release_id
|
||||||
|
.clone()
|
||||||
|
.or_else(|| release_id_from_root(&config.release_root));
|
||||||
|
let mut report = OfficialReleaseImportReport {
|
||||||
|
manifest_entry_count: manifest.entries.len(),
|
||||||
|
imported_count: 0,
|
||||||
|
unchanged_count: 0,
|
||||||
|
metadata_updated_count: 0,
|
||||||
|
asset_bundle_count: 0,
|
||||||
|
text_asset_count: 0,
|
||||||
|
table_count: 0,
|
||||||
|
media_count: 0,
|
||||||
|
parse_summary,
|
||||||
|
warnings: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in manifest.entries.values() {
|
||||||
|
let parse_entries = parse_entries_by_destination
|
||||||
|
.get(&entry.destination)
|
||||||
|
.map(Vec::as_slice)
|
||||||
|
.unwrap_or(&[]);
|
||||||
|
self.import_entry(
|
||||||
|
&config.release_root,
|
||||||
|
release_id.as_deref(),
|
||||||
|
entry,
|
||||||
|
parse_entries,
|
||||||
|
&mut report,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(report)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn import_entry(
|
||||||
|
&self,
|
||||||
|
release_root: &Path,
|
||||||
|
official_release_id: Option<&str>,
|
||||||
|
manifest_entry: &OfficialDownloadManifestEntry,
|
||||||
|
parse_entries: &[&OfficialParseCacheEntry],
|
||||||
|
report: &mut OfficialReleaseImportReport,
|
||||||
|
) -> bat_core::Result<()> {
|
||||||
|
let path = release_root.join(Path::new(&manifest_entry.destination));
|
||||||
|
ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?;
|
||||||
|
ensure_safe_file_target(release_root, &path, "官方资源导入输入")
|
||||||
|
.map_err(bat_core::Error::InvalidArgument)?;
|
||||||
|
let bytes = read_file_no_symlink(&path, "官方资源导入输入")
|
||||||
|
.map_err(bat_core::Error::InvalidArgument)?
|
||||||
|
.ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?;
|
||||||
|
if bytes.len() as u64 != manifest_entry.bytes {
|
||||||
|
return Err(bat_core::Error::InvalidArgument(format!(
|
||||||
|
"官方资源导入 size 校验失败 {}:期望 {},实际 {}",
|
||||||
|
manifest_entry.destination,
|
||||||
|
manifest_entry.bytes,
|
||||||
|
bytes.len()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let actual_hash = blake3::hash(&bytes).to_hex().to_string();
|
||||||
|
if actual_hash != manifest_entry.blake3 {
|
||||||
|
return Err(bat_core::Error::InvalidArgument(format!(
|
||||||
|
"官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}",
|
||||||
|
manifest_entry.destination, manifest_entry.blake3, actual_hash
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let resource_type = resource_type_for_path(&manifest_entry.destination);
|
||||||
|
let metadata =
|
||||||
|
metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries);
|
||||||
|
let resource_id = resource_id_for_destination(&manifest_entry.destination);
|
||||||
|
let previous = match self.resources.find_by_id(&resource_id).await {
|
||||||
|
Ok(resource) => Some(resource),
|
||||||
|
Err(bat_core::Error::NotFound(_)) => None,
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
if previous
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|resource| resource.entry.hash == actual_hash)
|
||||||
|
{
|
||||||
|
if let Some(mut resource) = previous {
|
||||||
|
let should_refresh_metadata = resource.metadata != metadata
|
||||||
|
|| resource.entry.resource_type != resource_type
|
||||||
|
|| resource.entry.size != manifest_entry.bytes
|
||||||
|
|| resource.local_path.as_path() != Path::new(&manifest_entry.destination);
|
||||||
|
if should_refresh_metadata {
|
||||||
|
resource.local_path = PathBuf::from(&manifest_entry.destination);
|
||||||
|
resource.entry.size = manifest_entry.bytes;
|
||||||
|
resource.entry.resource_type = resource_type;
|
||||||
|
resource.metadata = metadata;
|
||||||
|
self.resources.update(resource).await?;
|
||||||
|
report.metadata_updated_count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.unchanged_count += 1;
|
||||||
|
count_resource_type(report, resource_type);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let object_id = self.cas.store(&bytes).await?;
|
||||||
|
let resource = Resource {
|
||||||
|
id: resource_id,
|
||||||
|
local_path: PathBuf::from(&manifest_entry.destination),
|
||||||
|
entry: ResourceEntry {
|
||||||
|
path: manifest_entry.destination.clone(),
|
||||||
|
hash: object_id.clone(),
|
||||||
|
size: manifest_entry.bytes,
|
||||||
|
resource_type,
|
||||||
|
address: None,
|
||||||
|
dependencies: Vec::new(),
|
||||||
|
crc: None,
|
||||||
|
},
|
||||||
|
metadata,
|
||||||
|
};
|
||||||
|
if let Err(error) = self.resources.add(resource).await {
|
||||||
|
let _ = self.cas.remove_reference(&object_id).await;
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(previous) = previous {
|
||||||
|
if previous.entry.hash != object_id {
|
||||||
|
match self.cas.remove_reference(&previous.entry.hash).await {
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(error) => report.warnings.push(format!(
|
||||||
|
"旧 CAS 引用清理失败 {}:{}",
|
||||||
|
previous.entry.hash, error
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.imported_count += 1;
|
||||||
|
count_resource_type(report, resource_type);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_entries_by_destination(
|
||||||
|
cache: &OfficialParseCache,
|
||||||
|
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
|
||||||
|
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
|
||||||
|
for entry in cache.entries.values() {
|
||||||
|
by_destination
|
||||||
|
.entry(entry.destination.clone())
|
||||||
|
.or_default()
|
||||||
|
.push(entry);
|
||||||
|
}
|
||||||
|
by_destination
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metadata_for_manifest_entry(
|
||||||
|
official_release_id: Option<&str>,
|
||||||
|
manifest_entry: &OfficialDownloadManifestEntry,
|
||||||
|
parse_entries: &[&OfficialParseCacheEntry],
|
||||||
|
) -> ResourceMetadata {
|
||||||
|
let mut metadata = ResourceMetadata {
|
||||||
|
official_release_id: official_release_id.map(ToOwned::to_owned),
|
||||||
|
platform: platform_for_destination(&manifest_entry.destination),
|
||||||
|
bundle_path: Some(manifest_entry.destination.clone()),
|
||||||
|
..ResourceMetadata::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut archive_entries = BTreeSet::new();
|
||||||
|
let mut parse_statuses = BTreeSet::new();
|
||||||
|
let mut unity_versions = BTreeSet::new();
|
||||||
|
let mut text_assets = BTreeSet::new();
|
||||||
|
let mut text_unit_formats = BTreeSet::new();
|
||||||
|
|
||||||
|
for entry in parse_entries {
|
||||||
|
if let Some(archive_entry) = entry.archive_entry.as_ref() {
|
||||||
|
archive_entries.insert(archive_entry.clone());
|
||||||
|
}
|
||||||
|
parse_statuses.insert(parse_status_label(entry.status).to_string());
|
||||||
|
if let Some(unity_version) = entry.unity_version.as_ref() {
|
||||||
|
unity_versions.insert(unity_version.clone());
|
||||||
|
}
|
||||||
|
metadata.unityfs_file_count += entry.file_count as u64;
|
||||||
|
metadata.serialized_file_count += entry.serialized_file_count as u64;
|
||||||
|
metadata.text_asset_count += entry.text_asset_count as u64;
|
||||||
|
metadata.text_unit_count += entry.text_unit_count as u64;
|
||||||
|
metadata.text_unit_error_count += entry.text_unit_error_count as u64;
|
||||||
|
text_assets.extend(entry.text_assets.iter().cloned());
|
||||||
|
text_unit_formats.extend(entry.text_unit_formats.iter().cloned());
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata.archive_entries = archive_entries.into_iter().collect();
|
||||||
|
metadata.parse_statuses = parse_statuses.into_iter().collect();
|
||||||
|
metadata.unity_versions = unity_versions.into_iter().collect();
|
||||||
|
metadata.text_assets = text_assets.into_iter().collect();
|
||||||
|
metadata.text_unit_formats = text_unit_formats.into_iter().collect();
|
||||||
|
metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_status_label(status: OfficialParseStatus) -> &'static str {
|
||||||
|
match status {
|
||||||
|
OfficialParseStatus::Parsed => "parsed",
|
||||||
|
OfficialParseStatus::SkippedUnsupported => "skipped_unsupported",
|
||||||
|
OfficialParseStatus::Failed => "failed",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn platform_for_destination(destination: &str) -> Option<String> {
|
||||||
|
let normalized = destination.replace('\\', "/").to_ascii_lowercase();
|
||||||
|
if normalized.contains("windows") || normalized.contains("/win/") {
|
||||||
|
Some("windows".to_string())
|
||||||
|
} else if normalized.contains("android") {
|
||||||
|
Some("android".to_string())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_id_from_root(release_root: &Path) -> Option<String> {
|
||||||
|
release_root
|
||||||
|
.file_name()
|
||||||
|
.and_then(|name| name.to_str())
|
||||||
|
.filter(|name| !name.is_empty() && *name != "current")
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resource_id_for_destination(destination: &str) -> String {
|
||||||
|
format!("official/{}", destination.replace('\\', "/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resource_type_for_path(path: &str) -> ResourceType {
|
||||||
|
let normalized = path.replace('\\', "/").to_ascii_lowercase();
|
||||||
|
if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") {
|
||||||
|
ResourceType::AssetBundle
|
||||||
|
} else if normalized.contains("tablebundles/") {
|
||||||
|
ResourceType::TableBundle
|
||||||
|
} else if normalized.contains("textassets/")
|
||||||
|
|| matches!(
|
||||||
|
normalized.rsplit('.').next(),
|
||||||
|
Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml")
|
||||||
|
)
|
||||||
|
{
|
||||||
|
ResourceType::TextAsset
|
||||||
|
} else if normalized.contains("mediaresources/")
|
||||||
|
|| matches!(
|
||||||
|
normalized.rsplit('.').next(),
|
||||||
|
Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp")
|
||||||
|
)
|
||||||
|
{
|
||||||
|
ResourceType::Media
|
||||||
|
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
|
||||||
|
ResourceType::Manifest
|
||||||
|
} else {
|
||||||
|
ResourceType::Other
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) {
|
||||||
|
match resource_type {
|
||||||
|
ResourceType::AssetBundle => report.asset_bundle_count += 1,
|
||||||
|
ResourceType::TextAsset => report.text_asset_count += 1,
|
||||||
|
ResourceType::TableBundle => report.table_count += 1,
|
||||||
|
ResourceType::Media => report.media_count += 1,
|
||||||
|
ResourceType::Manifest | ResourceType::Other => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::{FileSystemCasRepository, InMemoryResourceRepository};
|
||||||
|
use crate::{
|
||||||
|
OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION,
|
||||||
|
};
|
||||||
|
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) {
|
||||||
|
let path = root.join(destination);
|
||||||
|
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(&path, bytes).unwrap();
|
||||||
|
let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
|
||||||
|
let entry = OfficialDownloadManifestEntry {
|
||||||
|
url: url.clone(),
|
||||||
|
destination: destination.to_string(),
|
||||||
|
bytes: bytes.len() as u64,
|
||||||
|
blake3: blake3::hash(bytes).to_hex().to_string(),
|
||||||
|
};
|
||||||
|
let manifest = serde_json::json!({
|
||||||
|
"version": 1,
|
||||||
|
"entries": BTreeMap::from([(url, entry)]),
|
||||||
|
});
|
||||||
|
std::fs::write(
|
||||||
|
root.join("official-download-manifest.json"),
|
||||||
|
serde_json::to_vec(&manifest).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) {
|
||||||
|
let source_url =
|
||||||
|
format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
|
||||||
|
let entry = OfficialParseCacheEntry {
|
||||||
|
key: format!("direct:{source_url}"),
|
||||||
|
source_url: source_url.clone(),
|
||||||
|
destination: destination.to_string(),
|
||||||
|
archive_entry: None,
|
||||||
|
source_kind: OfficialParseSourceKind::DirectBundle,
|
||||||
|
fingerprint: OfficialParseSourceFingerprint {
|
||||||
|
source_url,
|
||||||
|
destination: destination.to_string(),
|
||||||
|
bytes: bytes.len() as u64,
|
||||||
|
blake3: blake3::hash(bytes).to_hex().to_string(),
|
||||||
|
},
|
||||||
|
status: OfficialParseStatus::Parsed,
|
||||||
|
reused_from_previous_cache: false,
|
||||||
|
unity_version: Some("2021.3.56f2".to_string()),
|
||||||
|
file_count: 1,
|
||||||
|
serialized_file_count: 1,
|
||||||
|
text_asset_count: 1,
|
||||||
|
text_assets: vec!["Scenario".to_string()],
|
||||||
|
serialized_parse_error_count: 0,
|
||||||
|
text_unit_count: 2,
|
||||||
|
text_unit_formats: vec!["json".to_string(), "plain".to_string()],
|
||||||
|
skipped_binary_text_asset_count: 0,
|
||||||
|
text_unit_error_count: 1,
|
||||||
|
error: None,
|
||||||
|
};
|
||||||
|
let cache = OfficialParseCache {
|
||||||
|
version: OFFICIAL_PARSE_CACHE_VERSION,
|
||||||
|
generated_unix_seconds: 123,
|
||||||
|
summary: OfficialParseSummary {
|
||||||
|
manifest_entry_count: 1,
|
||||||
|
cache_entry_count: 1,
|
||||||
|
candidate_file_count: 1,
|
||||||
|
zip_entry_count: 0,
|
||||||
|
skipped_unchanged_count: 0,
|
||||||
|
parsed_bundle_count: 1,
|
||||||
|
unsupported_count: 0,
|
||||||
|
failed_count: 0,
|
||||||
|
text_asset_count: 1,
|
||||||
|
text_unit_count: 2,
|
||||||
|
skipped_binary_text_asset_count: 0,
|
||||||
|
text_unit_error_count: 1,
|
||||||
|
},
|
||||||
|
entries: BTreeMap::from([(entry.key.clone(), entry)]),
|
||||||
|
};
|
||||||
|
std::fs::write(
|
||||||
|
root.join("official-parse-cache.json"),
|
||||||
|
serde_json::to_vec(&cache).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn imports_verified_release_idempotently() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog");
|
||||||
|
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
|
||||||
|
let resources = InMemoryResourceRepository::new();
|
||||||
|
let service = OfficialReleaseImportService::new(&cas, &resources);
|
||||||
|
let config = OfficialReleaseImportConfig::new(temp.path());
|
||||||
|
|
||||||
|
let first = service.import_release(&config).await.unwrap();
|
||||||
|
let second = service.import_release(&config).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(first.imported_count, 1);
|
||||||
|
assert_eq!(second.imported_count, 0);
|
||||||
|
assert_eq!(second.unchanged_count, 1);
|
||||||
|
assert_eq!(
|
||||||
|
resources
|
||||||
|
.count(ResourceQuery::by_type(ResourceType::TableBundle))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
let id = resource_id_for_destination("TableBundles/TableCatalog.bytes");
|
||||||
|
let resource = resources.find_by_id(&id).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
cas.get_reference_count(&resource.entry.hash).await.unwrap(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn imports_parse_cache_metadata_into_resource_index() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let destination = "Windows/Bundles/scenario.bundle";
|
||||||
|
let bytes = b"bundle-bytes";
|
||||||
|
write_manifest(temp.path(), destination, bytes);
|
||||||
|
write_parse_cache(temp.path(), destination, bytes);
|
||||||
|
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
|
||||||
|
let resources = InMemoryResourceRepository::new();
|
||||||
|
let service = OfficialReleaseImportService::new(&cas, &resources);
|
||||||
|
|
||||||
|
let report = service
|
||||||
|
.import_release(
|
||||||
|
&OfficialReleaseImportConfig::new(temp.path())
|
||||||
|
.with_official_release_id("release-current"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(report.imported_count, 1);
|
||||||
|
assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2);
|
||||||
|
let id = resource_id_for_destination(destination);
|
||||||
|
let resource = resources.find_by_id(&id).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
resource.metadata.official_release_id.as_deref(),
|
||||||
|
Some("release-current")
|
||||||
|
);
|
||||||
|
assert_eq!(resource.metadata.platform.as_deref(), Some("windows"));
|
||||||
|
assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination));
|
||||||
|
assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]);
|
||||||
|
assert_eq!(
|
||||||
|
resource.metadata.unity_versions,
|
||||||
|
vec!["2021.3.56f2".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]);
|
||||||
|
assert_eq!(resource.metadata.text_unit_count, 2);
|
||||||
|
assert_eq!(
|
||||||
|
resource.metadata.text_unit_formats,
|
||||||
|
vec!["json".to_string(), "plain".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(resource.metadata.text_unit_error_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rejects_manifest_hash_mismatch_before_cas_write() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original");
|
||||||
|
let path = temp.path().join("TextAssets/dialogue.txt");
|
||||||
|
std::fs::write(&path, b"tampered").unwrap();
|
||||||
|
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
|
||||||
|
let resources = InMemoryResourceRepository::new();
|
||||||
|
let service = OfficialReleaseImportService::new(&cas, &resources);
|
||||||
|
|
||||||
|
let error = service
|
||||||
|
.import_release(&OfficialReleaseImportConfig::new(temp.path()))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("BLAKE3"));
|
||||||
|
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,834 @@
|
|||||||
|
//! Incremental TextUnit task and Crowdin offline queues for official releases.
|
||||||
|
//!
|
||||||
|
//! The queue is derived only from a verified official release, its
|
||||||
|
//! `official-resource-changes.json`, and its `official-parse-cache.json`.
|
||||||
|
//! It does not call Crowdin or any network service.
|
||||||
|
|
||||||
|
use crate::official_changes::{
|
||||||
|
read_resource_change_set_at, OfficialResourceChange, OfficialResourceChangeKind,
|
||||||
|
OfficialResourceChangeSet, TranslationHandoffProvider, TranslationHandoffStatus,
|
||||||
|
};
|
||||||
|
use crate::official_parse::{
|
||||||
|
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceKind,
|
||||||
|
OfficialParseStatus,
|
||||||
|
};
|
||||||
|
use crate::path_security::{
|
||||||
|
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
|
||||||
|
STATE_FILE_MODE,
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
/// Current TextUnit task-queue schema version.
|
||||||
|
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION: u32 = 1;
|
||||||
|
/// File name stored under a published official release root.
|
||||||
|
pub const OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE: &str = "official-textunit-tasks.json";
|
||||||
|
/// Current Crowdin TextUnit queue schema version.
|
||||||
|
pub const CROWDIN_TEXTUNIT_QUEUE_VERSION: u32 = 1;
|
||||||
|
/// File name stored under a published official release root.
|
||||||
|
pub const CROWDIN_TEXTUNIT_QUEUE_FILE: &str = "crowdin-textunit-queue.json";
|
||||||
|
|
||||||
|
/// Processing status for one incremental TextUnit task candidate.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum OfficialTextUnitTaskStatus {
|
||||||
|
/// The task has parse metadata and at least one TextUnit.
|
||||||
|
QueuedOffline,
|
||||||
|
/// No parse-cache entry exists for this changed resource yet.
|
||||||
|
SkippedNoParseEntry,
|
||||||
|
/// The resource parsed successfully, but did not produce TextUnits.
|
||||||
|
SkippedNoTextUnit,
|
||||||
|
/// Parse cache says the candidate failed to parse.
|
||||||
|
SkippedParseFailed,
|
||||||
|
/// Parse cache says the candidate is unsupported.
|
||||||
|
SkippedUnsupported,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialTextUnitTaskStatus {
|
||||||
|
/// Returns the stable status label.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::QueuedOffline => "queued_offline",
|
||||||
|
Self::SkippedNoParseEntry => "skipped_no_parse_entry",
|
||||||
|
Self::SkippedNoTextUnit => "skipped_no_text_unit",
|
||||||
|
Self::SkippedParseFailed => "skipped_parse_failed",
|
||||||
|
Self::SkippedUnsupported => "skipped_unsupported",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One parse-cache-backed incremental task.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitTask {
|
||||||
|
/// Stable task ID.
|
||||||
|
pub task_id: String,
|
||||||
|
/// Current official release ID.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Relative resource path under the official release root.
|
||||||
|
pub destination: String,
|
||||||
|
/// Change kind that caused this task candidate.
|
||||||
|
pub change_kind: OfficialResourceChangeKind,
|
||||||
|
/// Current official URL.
|
||||||
|
pub url: String,
|
||||||
|
/// Verified byte count.
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Verified BLAKE3 digest.
|
||||||
|
pub blake3: String,
|
||||||
|
/// Parse-cache entry key, when available.
|
||||||
|
pub parse_entry_key: Option<String>,
|
||||||
|
/// ZIP/archive entry containing the parsed bundle, when available.
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Parse-cache source kind, when available.
|
||||||
|
pub source_kind: Option<OfficialParseSourceKind>,
|
||||||
|
/// Parse status, when available.
|
||||||
|
pub parse_status: Option<OfficialParseStatus>,
|
||||||
|
/// Number of TextAsset objects in this parse entry.
|
||||||
|
pub text_asset_count: usize,
|
||||||
|
/// TextAsset names in this parse entry.
|
||||||
|
pub text_assets: Vec<String>,
|
||||||
|
/// Number of TextUnits in this parse entry.
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// TextUnit format labels.
|
||||||
|
pub text_unit_formats: Vec<String>,
|
||||||
|
/// Non-fatal TextUnit extraction diagnostic count.
|
||||||
|
pub text_unit_error_count: usize,
|
||||||
|
/// Queue status.
|
||||||
|
pub status: OfficialTextUnitTaskStatus,
|
||||||
|
/// Diagnostic reason for skipped tasks.
|
||||||
|
pub reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate counters for an incremental TextUnit task queue.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitTaskSummary {
|
||||||
|
/// Number of added/modified resources considered.
|
||||||
|
pub resource_candidate_count: usize,
|
||||||
|
/// Number of parse-cache entries inspected for those resources.
|
||||||
|
pub parse_entry_count: usize,
|
||||||
|
/// Number of tasks queued for offline translation.
|
||||||
|
pub queued_task_count: usize,
|
||||||
|
/// Number of changed resources with no parse-cache entry.
|
||||||
|
pub skipped_no_parse_entry_count: usize,
|
||||||
|
/// Number of parse entries with zero TextUnits.
|
||||||
|
pub skipped_no_text_unit_count: usize,
|
||||||
|
/// Number of failed parse entries.
|
||||||
|
pub skipped_parse_failed_count: usize,
|
||||||
|
/// Number of unsupported parse entries.
|
||||||
|
pub skipped_unsupported_count: usize,
|
||||||
|
/// Total queued TextUnit count.
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialTextUnitTaskSummary {
|
||||||
|
fn record(&mut self, task: &OfficialTextUnitTask) {
|
||||||
|
match task.status {
|
||||||
|
OfficialTextUnitTaskStatus::QueuedOffline => {
|
||||||
|
self.queued_task_count += 1;
|
||||||
|
self.text_unit_count += task.text_unit_count;
|
||||||
|
}
|
||||||
|
OfficialTextUnitTaskStatus::SkippedNoParseEntry => {
|
||||||
|
self.skipped_no_parse_entry_count += 1;
|
||||||
|
}
|
||||||
|
OfficialTextUnitTaskStatus::SkippedNoTextUnit => {
|
||||||
|
self.skipped_no_text_unit_count += 1;
|
||||||
|
}
|
||||||
|
OfficialTextUnitTaskStatus::SkippedParseFailed => {
|
||||||
|
self.skipped_parse_failed_count += 1;
|
||||||
|
}
|
||||||
|
OfficialTextUnitTaskStatus::SkippedUnsupported => {
|
||||||
|
self.skipped_unsupported_count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Durable incremental TextUnit task queue for one official release.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitTaskQueue {
|
||||||
|
/// Queue schema version.
|
||||||
|
#[serde(default = "default_textunit_task_queue_version")]
|
||||||
|
pub queue_version: u32,
|
||||||
|
/// Current official release ID.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Previous official release ID, when known.
|
||||||
|
pub previous_release_id: Option<String>,
|
||||||
|
/// Generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Current official release root.
|
||||||
|
pub current_resource_root: PathBuf,
|
||||||
|
/// Queue summary.
|
||||||
|
pub summary: OfficialTextUnitTaskSummary,
|
||||||
|
/// All task candidates, including skipped diagnostics.
|
||||||
|
pub tasks: Vec<OfficialTextUnitTask>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OfficialTextUnitTaskQueue {
|
||||||
|
/// Builds a task queue from an official change set and parse cache.
|
||||||
|
pub fn from_change_set_and_parse_cache(
|
||||||
|
change_set: &OfficialResourceChangeSet,
|
||||||
|
parse_cache: &OfficialParseCache,
|
||||||
|
) -> Self {
|
||||||
|
let parse_entries = parse_entries_by_destination(parse_cache);
|
||||||
|
let mut summary = OfficialTextUnitTaskSummary {
|
||||||
|
resource_candidate_count: change_set.parse_candidates().len(),
|
||||||
|
..OfficialTextUnitTaskSummary::default()
|
||||||
|
};
|
||||||
|
let mut tasks = Vec::new();
|
||||||
|
|
||||||
|
for change in change_set.parse_candidates() {
|
||||||
|
let entries = parse_entries
|
||||||
|
.get(&change.destination)
|
||||||
|
.map(Vec::as_slice)
|
||||||
|
.unwrap_or(&[]);
|
||||||
|
if entries.is_empty() {
|
||||||
|
let task = skipped_no_parse_entry_task(change_set, change);
|
||||||
|
summary.record(&task);
|
||||||
|
tasks.push(task);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
summary.parse_entry_count += entries.len();
|
||||||
|
for entry in entries {
|
||||||
|
let task = task_from_parse_entry(change_set, change, entry);
|
||||||
|
summary.record(&task);
|
||||||
|
tasks.push(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Self {
|
||||||
|
queue_version: OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION,
|
||||||
|
official_release_id: change_set.official_release_id.clone(),
|
||||||
|
previous_release_id: change_set.previous_release_id.clone(),
|
||||||
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
|
current_resource_root: change_set.current_resource_root.clone(),
|
||||||
|
summary,
|
||||||
|
tasks,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_derivation(&self, expected: &Self) -> bool {
|
||||||
|
self.official_release_id == expected.official_release_id
|
||||||
|
&& self.previous_release_id == expected.previous_release_id
|
||||||
|
&& self.current_resource_root == expected.current_resource_root
|
||||||
|
&& self.summary == expected.summary
|
||||||
|
&& self.tasks == expected.tasks
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One Crowdin offline queue item referencing a TextUnit task.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CrowdinTextUnitQueueItem {
|
||||||
|
/// Stable TextUnit task ID.
|
||||||
|
pub task_id: String,
|
||||||
|
/// Relative resource path under the official release root.
|
||||||
|
pub destination: String,
|
||||||
|
/// ZIP/archive entry, when applicable.
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Number of queued TextUnits.
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// TextUnit format labels.
|
||||||
|
pub text_unit_formats: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Crowdin offline queue derived from queued TextUnit tasks.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CrowdinTextUnitQueue {
|
||||||
|
/// Queue schema version.
|
||||||
|
#[serde(default = "default_crowdin_textunit_queue_version")]
|
||||||
|
pub queue_version: u32,
|
||||||
|
/// Provider reserved for this queue.
|
||||||
|
pub provider: TranslationHandoffProvider,
|
||||||
|
/// Queue status.
|
||||||
|
pub status: TranslationHandoffStatus,
|
||||||
|
/// Current official release ID.
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// Generation time as Unix seconds.
|
||||||
|
pub generated_unix_seconds: u64,
|
||||||
|
/// Path to the source TextUnit task queue.
|
||||||
|
pub textunit_task_queue_path: PathBuf,
|
||||||
|
/// Number of queued task items.
|
||||||
|
pub task_count: usize,
|
||||||
|
/// Total queued TextUnit count.
|
||||||
|
pub text_unit_count: usize,
|
||||||
|
/// Queued items for the future Crowdin worker.
|
||||||
|
pub items: Vec<CrowdinTextUnitQueueItem>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrowdinTextUnitQueue {
|
||||||
|
/// Builds a Crowdin offline queue from queued TextUnit tasks.
|
||||||
|
pub fn from_textunit_task_queue(
|
||||||
|
task_queue_path: PathBuf,
|
||||||
|
task_queue: &OfficialTextUnitTaskQueue,
|
||||||
|
) -> Self {
|
||||||
|
let items = task_queue
|
||||||
|
.tasks
|
||||||
|
.iter()
|
||||||
|
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
|
||||||
|
.map(|task| CrowdinTextUnitQueueItem {
|
||||||
|
task_id: task.task_id.clone(),
|
||||||
|
destination: task.destination.clone(),
|
||||||
|
archive_entry: task.archive_entry.clone(),
|
||||||
|
text_unit_count: task.text_unit_count,
|
||||||
|
text_unit_formats: task.text_unit_formats.clone(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let text_unit_count = items.iter().map(|item| item.text_unit_count).sum();
|
||||||
|
|
||||||
|
Self {
|
||||||
|
queue_version: CROWDIN_TEXTUNIT_QUEUE_VERSION,
|
||||||
|
provider: TranslationHandoffProvider::Crowdin,
|
||||||
|
status: TranslationHandoffStatus::QueuedOffline,
|
||||||
|
official_release_id: task_queue.official_release_id.clone(),
|
||||||
|
generated_unix_seconds: unix_seconds_now(),
|
||||||
|
textunit_task_queue_path: task_queue_path,
|
||||||
|
task_count: items.len(),
|
||||||
|
text_unit_count,
|
||||||
|
items,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_derivation(&self, expected: &Self) -> bool {
|
||||||
|
self.provider == expected.provider
|
||||||
|
&& self.status == expected.status
|
||||||
|
&& self.official_release_id == expected.official_release_id
|
||||||
|
&& self.textunit_task_queue_path == expected.textunit_task_queue_path
|
||||||
|
&& self.task_count == expected.task_count
|
||||||
|
&& self.text_unit_count == expected.text_unit_count
|
||||||
|
&& self.items == expected.items
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paths and summary produced after writing incremental TextUnit queues.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialTextUnitQueueReport {
|
||||||
|
/// Path to `official-textunit-tasks.json`.
|
||||||
|
pub textunit_task_queue_path: PathBuf,
|
||||||
|
/// Path to `crowdin-textunit-queue.json`.
|
||||||
|
pub crowdin_textunit_queue_path: PathBuf,
|
||||||
|
/// Aggregate queue counters.
|
||||||
|
pub summary: OfficialTextUnitTaskSummary,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads change/parse derived queues from a release root and writes them back.
|
||||||
|
pub fn write_official_textunit_queues(
|
||||||
|
resource_root: &Path,
|
||||||
|
) -> Result<OfficialTextUnitQueueReport, String> {
|
||||||
|
let change_set = read_resource_change_set_at(resource_root)?.ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"缺少官方资源变更集,无法生成 TextUnit 队列:{}",
|
||||||
|
resource_root.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let parse_cache = read_parse_cache_at(resource_root)?.ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"缺少官方解析缓存,无法生成 TextUnit 队列:{}",
|
||||||
|
resource_root.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let task_queue =
|
||||||
|
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
|
||||||
|
write_textunit_task_queue_at(resource_root, &task_queue)?;
|
||||||
|
|
||||||
|
let task_queue_path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
|
||||||
|
let crowdin_queue =
|
||||||
|
CrowdinTextUnitQueue::from_textunit_task_queue(task_queue_path.clone(), &task_queue);
|
||||||
|
write_crowdin_textunit_queue_at(resource_root, &crowdin_queue)?;
|
||||||
|
|
||||||
|
Ok(OfficialTextUnitQueueReport {
|
||||||
|
textunit_task_queue_path: task_queue_path,
|
||||||
|
crowdin_textunit_queue_path: resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE),
|
||||||
|
summary: task_queue.summary,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads a generated TextUnit task queue from a release root.
|
||||||
|
pub fn read_textunit_task_queue_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
) -> Result<Option<OfficialTextUnitTaskQueue>, String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
|
||||||
|
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 任务队列")? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let queue: OfficialTextUnitTaskQueue = serde_json::from_slice(&bytes)
|
||||||
|
.map_err(|error| format!("解析官方 TextUnit 任务队列失败 {}:{error}", path.display()))?;
|
||||||
|
if queue.queue_version != OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION {
|
||||||
|
return Err(format!(
|
||||||
|
"不支持的官方 TextUnit 任务队列版本 {},文件 {}",
|
||||||
|
queue.queue_version,
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Some(queue))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the persisted TextUnit queue still matches current inputs.
|
||||||
|
pub fn is_textunit_task_queue_current(
|
||||||
|
resource_root: &Path,
|
||||||
|
queue: &OfficialTextUnitTaskQueue,
|
||||||
|
) -> Result<bool, String> {
|
||||||
|
let Some(change_set) = read_resource_change_set_at(resource_root)? else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
let Some(parse_cache) = read_parse_cache_at(resource_root)? else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
let expected =
|
||||||
|
OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(&change_set, &parse_cache);
|
||||||
|
Ok(queue.matches_derivation(&expected))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a TextUnit task queue under a release root.
|
||||||
|
pub fn write_textunit_task_queue_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
queue: &OfficialTextUnitTaskQueue,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE);
|
||||||
|
ensure_path_within_root(resource_root, &path)?;
|
||||||
|
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 任务队列")?;
|
||||||
|
let bytes = serde_json::to_vec_pretty(queue)
|
||||||
|
.map_err(|error| format!("序列化官方 TextUnit 任务队列失败:{error}"))?;
|
||||||
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 任务队列")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads a generated Crowdin offline TextUnit queue from a release root.
|
||||||
|
pub fn read_crowdin_textunit_queue_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
) -> Result<Option<CrowdinTextUnitQueue>, String> {
|
||||||
|
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
|
||||||
|
let Some(bytes) = read_file_no_symlink(&path, "Crowdin TextUnit 离线队列")? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let queue: CrowdinTextUnitQueue = serde_json::from_slice(&bytes).map_err(|error| {
|
||||||
|
format!(
|
||||||
|
"解析 Crowdin TextUnit 离线队列失败 {}:{error}",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
if queue.queue_version != CROWDIN_TEXTUNIT_QUEUE_VERSION {
|
||||||
|
return Err(format!(
|
||||||
|
"不支持的 Crowdin TextUnit 离线队列版本 {},文件 {}",
|
||||||
|
queue.queue_version,
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Some(queue))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the persisted Crowdin queue still matches a TextUnit queue.
|
||||||
|
pub fn is_crowdin_textunit_queue_current(
|
||||||
|
resource_root: &Path,
|
||||||
|
task_queue: &OfficialTextUnitTaskQueue,
|
||||||
|
) -> Result<bool, String> {
|
||||||
|
let Some(queue) = read_crowdin_textunit_queue_at(resource_root)? else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
let expected = CrowdinTextUnitQueue::from_textunit_task_queue(
|
||||||
|
resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
|
||||||
|
task_queue,
|
||||||
|
);
|
||||||
|
Ok(queue.matches_derivation(&expected))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a Crowdin offline TextUnit queue under a release root.
|
||||||
|
pub fn write_crowdin_textunit_queue_at(
|
||||||
|
resource_root: &Path,
|
||||||
|
queue: &CrowdinTextUnitQueue,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let path = resource_root.join(CROWDIN_TEXTUNIT_QUEUE_FILE);
|
||||||
|
ensure_path_within_root(resource_root, &path)?;
|
||||||
|
ensure_safe_file_target(resource_root, &path, "Crowdin TextUnit 离线队列")?;
|
||||||
|
let bytes = serde_json::to_vec_pretty(queue)
|
||||||
|
.map_err(|error| format!("序列化 Crowdin TextUnit 离线队列失败:{error}"))?;
|
||||||
|
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin TextUnit 离线队列")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_entries_by_destination(
|
||||||
|
cache: &OfficialParseCache,
|
||||||
|
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
|
||||||
|
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
|
||||||
|
for entry in cache.entries.values() {
|
||||||
|
by_destination
|
||||||
|
.entry(entry.destination.clone())
|
||||||
|
.or_default()
|
||||||
|
.push(entry);
|
||||||
|
}
|
||||||
|
by_destination
|
||||||
|
}
|
||||||
|
|
||||||
|
fn skipped_no_parse_entry_task(
|
||||||
|
change_set: &OfficialResourceChangeSet,
|
||||||
|
change: &OfficialResourceChange,
|
||||||
|
) -> OfficialTextUnitTask {
|
||||||
|
let current = change
|
||||||
|
.current
|
||||||
|
.as_ref()
|
||||||
|
.expect("parse candidate has current");
|
||||||
|
OfficialTextUnitTask {
|
||||||
|
task_id: task_id_for(&change_set.official_release_id, &change.destination, None),
|
||||||
|
official_release_id: change_set.official_release_id.clone(),
|
||||||
|
destination: change.destination.clone(),
|
||||||
|
change_kind: change.kind,
|
||||||
|
url: current.url.clone(),
|
||||||
|
bytes: current.bytes,
|
||||||
|
blake3: current.blake3.clone(),
|
||||||
|
parse_entry_key: None,
|
||||||
|
archive_entry: None,
|
||||||
|
source_kind: None,
|
||||||
|
parse_status: None,
|
||||||
|
text_asset_count: 0,
|
||||||
|
text_assets: Vec::new(),
|
||||||
|
text_unit_count: 0,
|
||||||
|
text_unit_formats: Vec::new(),
|
||||||
|
text_unit_error_count: 0,
|
||||||
|
status: OfficialTextUnitTaskStatus::SkippedNoParseEntry,
|
||||||
|
reason: Some("parse cache entry not found for changed resource".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn task_from_parse_entry(
|
||||||
|
change_set: &OfficialResourceChangeSet,
|
||||||
|
change: &OfficialResourceChange,
|
||||||
|
entry: &OfficialParseCacheEntry,
|
||||||
|
) -> OfficialTextUnitTask {
|
||||||
|
let current = change
|
||||||
|
.current
|
||||||
|
.as_ref()
|
||||||
|
.expect("parse candidate has current");
|
||||||
|
let (status, reason) = status_for_parse_entry(entry);
|
||||||
|
OfficialTextUnitTask {
|
||||||
|
task_id: task_id_for(
|
||||||
|
&change_set.official_release_id,
|
||||||
|
&change.destination,
|
||||||
|
entry.archive_entry.as_deref(),
|
||||||
|
),
|
||||||
|
official_release_id: change_set.official_release_id.clone(),
|
||||||
|
destination: change.destination.clone(),
|
||||||
|
change_kind: change.kind,
|
||||||
|
url: current.url.clone(),
|
||||||
|
bytes: current.bytes,
|
||||||
|
blake3: current.blake3.clone(),
|
||||||
|
parse_entry_key: Some(entry.key.clone()),
|
||||||
|
archive_entry: entry.archive_entry.clone(),
|
||||||
|
source_kind: Some(entry.source_kind),
|
||||||
|
parse_status: Some(entry.status),
|
||||||
|
text_asset_count: entry.text_asset_count,
|
||||||
|
text_assets: entry.text_assets.clone(),
|
||||||
|
text_unit_count: entry.text_unit_count,
|
||||||
|
text_unit_formats: entry.text_unit_formats.clone(),
|
||||||
|
text_unit_error_count: entry.text_unit_error_count,
|
||||||
|
status,
|
||||||
|
reason,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn status_for_parse_entry(
|
||||||
|
entry: &OfficialParseCacheEntry,
|
||||||
|
) -> (OfficialTextUnitTaskStatus, Option<String>) {
|
||||||
|
match entry.status {
|
||||||
|
OfficialParseStatus::Parsed if entry.text_unit_count > 0 => {
|
||||||
|
(OfficialTextUnitTaskStatus::QueuedOffline, None)
|
||||||
|
}
|
||||||
|
OfficialParseStatus::Parsed => (
|
||||||
|
OfficialTextUnitTaskStatus::SkippedNoTextUnit,
|
||||||
|
Some("parsed resource has no TextUnit".to_string()),
|
||||||
|
),
|
||||||
|
OfficialParseStatus::Failed => (
|
||||||
|
OfficialTextUnitTaskStatus::SkippedParseFailed,
|
||||||
|
entry.error.clone(),
|
||||||
|
),
|
||||||
|
OfficialParseStatus::SkippedUnsupported => (
|
||||||
|
OfficialTextUnitTaskStatus::SkippedUnsupported,
|
||||||
|
entry.error.clone(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn task_id_for(
|
||||||
|
official_release_id: &str,
|
||||||
|
destination: &str,
|
||||||
|
archive_entry: Option<&str>,
|
||||||
|
) -> String {
|
||||||
|
match archive_entry {
|
||||||
|
Some(entry) => format!("textunit/{official_release_id}/{destination}#{entry}"),
|
||||||
|
None => format!("textunit/{official_release_id}/{destination}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_seconds_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_textunit_task_queue_version() -> u32 {
|
||||||
|
OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_crowdin_textunit_queue_version() -> u32 {
|
||||||
|
CROWDIN_TEXTUNIT_QUEUE_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::official_changes::{
|
||||||
|
OfficialResourceChangeSet, OfficialResourceDescriptor, OFFICIAL_RESOURCE_CHANGES_VERSION,
|
||||||
|
};
|
||||||
|
use crate::official_parse::{
|
||||||
|
OfficialParseCache, OfficialParseCacheEntry, OfficialParseSourceFingerprint,
|
||||||
|
OfficialParseSourceKind, OfficialParseSummary, OFFICIAL_PARSE_CACHE_VERSION,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn change_set(root: &Path) -> OfficialResourceChangeSet {
|
||||||
|
OfficialResourceChangeSet {
|
||||||
|
change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION,
|
||||||
|
official_release_id: "release-new".to_string(),
|
||||||
|
previous_release_id: Some("release-old".to_string()),
|
||||||
|
generated_unix_seconds: 123,
|
||||||
|
previous_resource_root: None,
|
||||||
|
current_resource_root: root.to_path_buf(),
|
||||||
|
summary: crate::OfficialResourceChangeSummary {
|
||||||
|
previous_manifest_present: true,
|
||||||
|
previous_manifest_entry_count: 2,
|
||||||
|
current_manifest_entry_count: 3,
|
||||||
|
added_count: 2,
|
||||||
|
modified_count: 1,
|
||||||
|
removed_count: 1,
|
||||||
|
parse_candidate_count: 3,
|
||||||
|
translation_candidate_count: 3,
|
||||||
|
},
|
||||||
|
changes: vec![
|
||||||
|
change("Bundles/a.bundle", OfficialResourceChangeKind::Added, b"a"),
|
||||||
|
change(
|
||||||
|
"Bundles/b.bundle",
|
||||||
|
OfficialResourceChangeKind::Modified,
|
||||||
|
b"b",
|
||||||
|
),
|
||||||
|
change("Bundles/c.bundle", OfficialResourceChangeKind::Added, b"c"),
|
||||||
|
crate::OfficialResourceChange {
|
||||||
|
destination: "Bundles/removed.bundle".to_string(),
|
||||||
|
kind: OfficialResourceChangeKind::Removed,
|
||||||
|
previous: Some(OfficialResourceDescriptor {
|
||||||
|
url: "https://old/removed".to_string(),
|
||||||
|
destination: "Bundles/removed.bundle".to_string(),
|
||||||
|
bytes: 1,
|
||||||
|
blake3: "old".to_string(),
|
||||||
|
}),
|
||||||
|
current: None,
|
||||||
|
parse_candidate: false,
|
||||||
|
translation_candidate: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn change(
|
||||||
|
destination: &str,
|
||||||
|
kind: OfficialResourceChangeKind,
|
||||||
|
bytes: &[u8],
|
||||||
|
) -> crate::OfficialResourceChange {
|
||||||
|
crate::OfficialResourceChange {
|
||||||
|
destination: destination.to_string(),
|
||||||
|
kind,
|
||||||
|
previous: None,
|
||||||
|
current: Some(OfficialResourceDescriptor {
|
||||||
|
url: format!("https://new/{destination}"),
|
||||||
|
destination: destination.to_string(),
|
||||||
|
bytes: bytes.len() as u64,
|
||||||
|
blake3: blake3::hash(bytes).to_hex().to_string(),
|
||||||
|
}),
|
||||||
|
parse_candidate: true,
|
||||||
|
translation_candidate: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_cache() -> OfficialParseCache {
|
||||||
|
OfficialParseCache {
|
||||||
|
version: OFFICIAL_PARSE_CACHE_VERSION,
|
||||||
|
generated_unix_seconds: 124,
|
||||||
|
summary: OfficialParseSummary {
|
||||||
|
manifest_entry_count: 3,
|
||||||
|
cache_entry_count: 2,
|
||||||
|
candidate_file_count: 2,
|
||||||
|
zip_entry_count: 0,
|
||||||
|
skipped_unchanged_count: 0,
|
||||||
|
parsed_bundle_count: 1,
|
||||||
|
unsupported_count: 1,
|
||||||
|
failed_count: 0,
|
||||||
|
text_asset_count: 1,
|
||||||
|
text_unit_count: 2,
|
||||||
|
skipped_binary_text_asset_count: 0,
|
||||||
|
text_unit_error_count: 0,
|
||||||
|
},
|
||||||
|
entries: BTreeMap::from([
|
||||||
|
(
|
||||||
|
"direct:a".to_string(),
|
||||||
|
parse_entry(
|
||||||
|
"direct:a",
|
||||||
|
"Bundles/a.bundle",
|
||||||
|
OfficialParseStatus::Parsed,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"direct:b".to_string(),
|
||||||
|
parse_entry(
|
||||||
|
"direct:b",
|
||||||
|
"Bundles/b.bundle",
|
||||||
|
OfficialParseStatus::SkippedUnsupported,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_entry(
|
||||||
|
key: &str,
|
||||||
|
destination: &str,
|
||||||
|
status: OfficialParseStatus,
|
||||||
|
text_unit_count: usize,
|
||||||
|
) -> OfficialParseCacheEntry {
|
||||||
|
OfficialParseCacheEntry {
|
||||||
|
key: key.to_string(),
|
||||||
|
source_url: format!("https://new/{destination}"),
|
||||||
|
destination: destination.to_string(),
|
||||||
|
archive_entry: None,
|
||||||
|
source_kind: OfficialParseSourceKind::DirectBundle,
|
||||||
|
fingerprint: OfficialParseSourceFingerprint {
|
||||||
|
source_url: format!("https://new/{destination}"),
|
||||||
|
destination: destination.to_string(),
|
||||||
|
bytes: 1,
|
||||||
|
blake3: "hash".to_string(),
|
||||||
|
},
|
||||||
|
status,
|
||||||
|
reused_from_previous_cache: false,
|
||||||
|
unity_version: Some("2021.3.56f2".to_string()),
|
||||||
|
file_count: 1,
|
||||||
|
serialized_file_count: 1,
|
||||||
|
text_asset_count: usize::from(text_unit_count > 0),
|
||||||
|
text_assets: if text_unit_count > 0 {
|
||||||
|
vec!["Scenario".to_string()]
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
},
|
||||||
|
serialized_parse_error_count: 0,
|
||||||
|
text_unit_count,
|
||||||
|
text_unit_formats: if text_unit_count > 0 {
|
||||||
|
vec!["plain".to_string()]
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
},
|
||||||
|
skipped_binary_text_asset_count: 0,
|
||||||
|
text_unit_error_count: 0,
|
||||||
|
error: (status != OfficialParseStatus::Parsed).then(|| "unsupported".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn textunit_queue_uses_only_added_and_modified_parse_candidates() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
|
||||||
|
&change_set(temp.path()),
|
||||||
|
&parse_cache(),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(queue.summary.resource_candidate_count, 3);
|
||||||
|
assert_eq!(queue.summary.parse_entry_count, 2);
|
||||||
|
assert_eq!(queue.summary.queued_task_count, 1);
|
||||||
|
assert_eq!(queue.summary.skipped_unsupported_count, 1);
|
||||||
|
assert_eq!(queue.summary.skipped_no_parse_entry_count, 1);
|
||||||
|
assert_eq!(queue.summary.text_unit_count, 2);
|
||||||
|
assert_eq!(
|
||||||
|
queue
|
||||||
|
.tasks
|
||||||
|
.iter()
|
||||||
|
.filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline)
|
||||||
|
.map(|task| task.destination.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["Bundles/a.bundle"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crowdin_queue_contains_only_queued_textunit_tasks() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
|
||||||
|
&change_set(temp.path()),
|
||||||
|
&parse_cache(),
|
||||||
|
);
|
||||||
|
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
|
||||||
|
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
|
||||||
|
&queue,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(crowdin.provider, TranslationHandoffProvider::Crowdin);
|
||||||
|
assert_eq!(crowdin.status, TranslationHandoffStatus::QueuedOffline);
|
||||||
|
assert_eq!(crowdin.task_count, 1);
|
||||||
|
assert_eq!(crowdin.text_unit_count, 2);
|
||||||
|
assert_eq!(crowdin.items[0].destination, "Bundles/a.bundle");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_textunit_queues_persists_files() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
|
||||||
|
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
|
||||||
|
|
||||||
|
let report = write_official_textunit_queues(temp.path()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(report.summary.queued_task_count, 1);
|
||||||
|
assert!(report.textunit_task_queue_path.is_file());
|
||||||
|
assert!(report.crowdin_textunit_queue_path.is_file());
|
||||||
|
let persisted = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
|
||||||
|
assert_eq!(persisted.summary.text_unit_count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn textunit_queue_current_detects_parse_cache_changes() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
crate::write_resource_change_set_at(temp.path(), &change_set(temp.path())).unwrap();
|
||||||
|
crate::write_parse_cache_at(temp.path(), &parse_cache()).unwrap();
|
||||||
|
write_official_textunit_queues(temp.path()).unwrap();
|
||||||
|
|
||||||
|
let queue = read_textunit_task_queue_at(temp.path()).unwrap().unwrap();
|
||||||
|
assert!(is_textunit_task_queue_current(temp.path(), &queue).unwrap());
|
||||||
|
|
||||||
|
let mut changed_cache = parse_cache();
|
||||||
|
changed_cache.entries.insert(
|
||||||
|
"direct:c".to_string(),
|
||||||
|
parse_entry(
|
||||||
|
"direct:c",
|
||||||
|
"Bundles/c.bundle",
|
||||||
|
OfficialParseStatus::Parsed,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
crate::write_parse_cache_at(temp.path(), &changed_cache).unwrap();
|
||||||
|
|
||||||
|
assert!(!is_textunit_task_queue_current(temp.path(), &queue).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crowdin_queue_current_requires_matching_file() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let queue = OfficialTextUnitTaskQueue::from_change_set_and_parse_cache(
|
||||||
|
&change_set(temp.path()),
|
||||||
|
&parse_cache(),
|
||||||
|
);
|
||||||
|
|
||||||
|
write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||||
|
assert!(!is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
|
||||||
|
|
||||||
|
let crowdin = CrowdinTextUnitQueue::from_textunit_task_queue(
|
||||||
|
temp.path().join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE),
|
||||||
|
&queue,
|
||||||
|
);
|
||||||
|
write_crowdin_textunit_queue_at(temp.path(), &crowdin).unwrap();
|
||||||
|
|
||||||
|
assert!(is_crowdin_textunit_queue_current(temp.path(), &queue).unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,424 @@
|
|||||||
|
//! File-level Patch and UnityFS write operations.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use bat_assetbundle::{
|
||||||
|
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
|
||||||
|
StringFieldPatch, TextAssetPatch, UnitySerializedReplacementValue,
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::path_security::{
|
||||||
|
ensure_safe_file_target, lexical_absolute, read_file_no_symlink, write_file_atomic,
|
||||||
|
STATE_FILE_MODE,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// File patch algorithm selected by `patch.apply`.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum PatchApplyKind {
|
||||||
|
/// Deterministic binary hunk patch.
|
||||||
|
Binary,
|
||||||
|
/// RFC 6902 JSON Patch.
|
||||||
|
Json,
|
||||||
|
/// UTF-8 Text Patch.
|
||||||
|
Text,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PatchApplyKind {
|
||||||
|
/// Stable RPC/CLI label.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Binary => "binary",
|
||||||
|
Self::Json => "json",
|
||||||
|
Self::Text => "text",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parameters for applying one file-level patch.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct PatchApplyParams {
|
||||||
|
/// Patch algorithm.
|
||||||
|
pub kind: PatchApplyKind,
|
||||||
|
/// Source file path.
|
||||||
|
pub source_path: PathBuf,
|
||||||
|
/// Patch document path.
|
||||||
|
pub patch_path: PathBuf,
|
||||||
|
/// Target file path written atomically.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parameters for patching one UnityFS TextAsset object.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct UnityFsTextAssetPatchParams {
|
||||||
|
/// Source UnityFS bundle file.
|
||||||
|
pub bundle_path: PathBuf,
|
||||||
|
/// Serialized file path inside the UnityFS directory table.
|
||||||
|
pub serialized_file_path: String,
|
||||||
|
/// Unity object path ID.
|
||||||
|
pub path_id: i64,
|
||||||
|
/// Replacement raw bytes path.
|
||||||
|
pub replacement_path: PathBuf,
|
||||||
|
/// Target bundle file path written atomically.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
/// Optional expected TextAsset name.
|
||||||
|
#[serde(default)]
|
||||||
|
pub expected_name: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parameters for patching one TypeTree string field inside a UnityFS bundle.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct UnityFsStringFieldPatchParams {
|
||||||
|
/// Source UnityFS bundle file.
|
||||||
|
pub bundle_path: PathBuf,
|
||||||
|
/// Serialized file path inside the UnityFS directory table.
|
||||||
|
pub serialized_file_path: String,
|
||||||
|
/// Unity object path ID.
|
||||||
|
pub path_id: i64,
|
||||||
|
/// Stable TypeTree field path.
|
||||||
|
pub field_path: String,
|
||||||
|
/// Replacement string supplied inline.
|
||||||
|
#[serde(default)]
|
||||||
|
pub replacement_text: Option<String>,
|
||||||
|
/// Replacement UTF-8 file path.
|
||||||
|
#[serde(default)]
|
||||||
|
pub replacement_path: Option<PathBuf>,
|
||||||
|
/// Target bundle file path written atomically.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
/// Optional expected source string.
|
||||||
|
#[serde(default)]
|
||||||
|
pub expected_value: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parameters for patching one semantic TypeTree field inside a UnityFS bundle.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct UnityFsFieldPatchParams {
|
||||||
|
/// Source UnityFS bundle file.
|
||||||
|
pub bundle_path: PathBuf,
|
||||||
|
/// Serialized file path inside the UnityFS directory table.
|
||||||
|
pub serialized_file_path: String,
|
||||||
|
/// Unity object path ID.
|
||||||
|
pub path_id: i64,
|
||||||
|
/// Stable TypeTree field path.
|
||||||
|
pub field_path: String,
|
||||||
|
/// Replacement value encoded according to the current TypeTree field type.
|
||||||
|
pub replacement: UnitySerializedReplacementValue,
|
||||||
|
/// Target bundle file path written atomically.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
/// Optional expected source value.
|
||||||
|
#[serde(default)]
|
||||||
|
pub expected_value: Option<UnitySerializedReplacementValue>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of a file-level patch operation.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct PatchApplyReport {
|
||||||
|
/// RPC/CLI command name.
|
||||||
|
pub command: &'static str,
|
||||||
|
/// Operation status.
|
||||||
|
pub status: &'static str,
|
||||||
|
/// Human-readable summary.
|
||||||
|
pub message: &'static str,
|
||||||
|
/// Patch algorithm.
|
||||||
|
pub kind: PatchApplyKind,
|
||||||
|
/// Absolute source path.
|
||||||
|
pub source_path: PathBuf,
|
||||||
|
/// Absolute patch path.
|
||||||
|
pub patch_path: PathBuf,
|
||||||
|
/// Absolute target path.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
/// Source byte length.
|
||||||
|
pub source_size: u64,
|
||||||
|
/// Patch document byte length.
|
||||||
|
pub patch_size: u64,
|
||||||
|
/// Target byte length.
|
||||||
|
pub target_size: u64,
|
||||||
|
/// Source BLAKE3 hash.
|
||||||
|
pub source_blake3: String,
|
||||||
|
/// Patch document BLAKE3 hash.
|
||||||
|
pub patch_blake3: String,
|
||||||
|
/// Target BLAKE3 hash.
|
||||||
|
pub target_blake3: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of a UnityFS write operation.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct UnityFsPatchReport {
|
||||||
|
/// RPC/CLI command name.
|
||||||
|
pub command: &'static str,
|
||||||
|
/// Operation status.
|
||||||
|
pub status: &'static str,
|
||||||
|
/// Human-readable summary.
|
||||||
|
pub message: &'static str,
|
||||||
|
/// Absolute source UnityFS bundle path.
|
||||||
|
pub bundle_path: PathBuf,
|
||||||
|
/// Serialized file path inside the UnityFS directory table.
|
||||||
|
pub serialized_file_path: String,
|
||||||
|
/// Unity object path ID.
|
||||||
|
pub path_id: i64,
|
||||||
|
/// Optional TypeTree field path for string-field patches.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// Absolute target bundle path.
|
||||||
|
pub target_path: PathBuf,
|
||||||
|
/// Source bundle byte length.
|
||||||
|
pub source_size: u64,
|
||||||
|
/// Replacement byte length.
|
||||||
|
pub replacement_size: u64,
|
||||||
|
/// Target bundle byte length.
|
||||||
|
pub target_size: u64,
|
||||||
|
/// Source bundle BLAKE3 hash.
|
||||||
|
pub source_blake3: String,
|
||||||
|
/// Replacement BLAKE3 hash.
|
||||||
|
pub replacement_blake3: String,
|
||||||
|
/// Target bundle BLAKE3 hash.
|
||||||
|
pub target_blake3: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies a Binary/JSON/Text patch document to one source file.
|
||||||
|
pub fn apply_patch_file(params: &PatchApplyParams) -> anyhow::Result<PatchApplyReport> {
|
||||||
|
let source_path = lexical_absolute(¶ms.source_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let patch_path = lexical_absolute(¶ms.patch_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let target_path = lexical_absolute(¶ms.target_path).map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_target_is_not_input(&target_path, &[&source_path, &patch_path])?;
|
||||||
|
|
||||||
|
let source = read_required_file(&source_path, "patch source")?;
|
||||||
|
let patch = read_required_file(&patch_path, "patch document")?;
|
||||||
|
let target = match params.kind {
|
||||||
|
PatchApplyKind::Binary => bat_patch::binary::apply_patch(&source, &patch)?,
|
||||||
|
PatchApplyKind::Json => {
|
||||||
|
let source = std::str::from_utf8(&source)
|
||||||
|
.map_err(|error| anyhow::anyhow!("JSON patch source 不是 UTF-8:{error}"))?;
|
||||||
|
let patch = std::str::from_utf8(&patch)
|
||||||
|
.map_err(|error| anyhow::anyhow!("JSON patch document 不是 UTF-8:{error}"))?;
|
||||||
|
bat_patch::json::apply_json_patch(source, patch)?.into_bytes()
|
||||||
|
}
|
||||||
|
PatchApplyKind::Text => bat_patch::text::apply_patch_bytes(&source, &patch)?,
|
||||||
|
};
|
||||||
|
write_output_file(&target_path, &target, "patch target")?;
|
||||||
|
|
||||||
|
Ok(PatchApplyReport {
|
||||||
|
command: "patch.apply",
|
||||||
|
status: "patched",
|
||||||
|
message: "patch 已应用并原子写入目标文件",
|
||||||
|
kind: params.kind,
|
||||||
|
source_path,
|
||||||
|
patch_path,
|
||||||
|
target_path,
|
||||||
|
source_size: source.len() as u64,
|
||||||
|
patch_size: patch.len() as u64,
|
||||||
|
target_size: target.len() as u64,
|
||||||
|
source_blake3: blake3_hex(&source),
|
||||||
|
patch_blake3: blake3_hex(&patch),
|
||||||
|
target_blake3: blake3_hex(&target),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Patches one UnityFS TextAsset and writes the rebuilt bundle atomically.
|
||||||
|
pub fn apply_unityfs_text_asset_patch_file(
|
||||||
|
params: &UnityFsTextAssetPatchParams,
|
||||||
|
) -> anyhow::Result<UnityFsPatchReport> {
|
||||||
|
let bundle_path = lexical_absolute(¶ms.bundle_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let replacement_path =
|
||||||
|
lexical_absolute(¶ms.replacement_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let target_path = lexical_absolute(¶ms.target_path).map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_target_is_not_input(&target_path, &[&bundle_path, &replacement_path])?;
|
||||||
|
|
||||||
|
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
|
||||||
|
let replacement = read_required_file(&replacement_path, "TextAsset replacement")?;
|
||||||
|
let mut patch = TextAssetPatch::new(
|
||||||
|
params.serialized_file_path.clone(),
|
||||||
|
params.path_id,
|
||||||
|
replacement.clone(),
|
||||||
|
);
|
||||||
|
patch.expected_name = params.expected_name.clone();
|
||||||
|
let target = patch_unityfs_text_asset(&bundle, &patch)?;
|
||||||
|
write_output_file(&target_path, &target, "UnityFS target")?;
|
||||||
|
|
||||||
|
Ok(UnityFsPatchReport {
|
||||||
|
command: "unityfs.patch_text_asset",
|
||||||
|
status: "patched",
|
||||||
|
message: "UnityFS TextAsset patch 已应用并原子写入目标 bundle",
|
||||||
|
bundle_path,
|
||||||
|
serialized_file_path: params.serialized_file_path.clone(),
|
||||||
|
path_id: params.path_id,
|
||||||
|
field_path: None,
|
||||||
|
target_path,
|
||||||
|
source_size: bundle.len() as u64,
|
||||||
|
replacement_size: replacement.len() as u64,
|
||||||
|
target_size: target.len() as u64,
|
||||||
|
source_blake3: blake3_hex(&bundle),
|
||||||
|
replacement_blake3: blake3_hex(&replacement),
|
||||||
|
target_blake3: blake3_hex(&target),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Patches one TypeTree string field and writes the rebuilt UnityFS bundle atomically.
|
||||||
|
pub fn apply_unityfs_string_field_patch_file(
|
||||||
|
params: &UnityFsStringFieldPatchParams,
|
||||||
|
) -> anyhow::Result<UnityFsPatchReport> {
|
||||||
|
let bundle_path = lexical_absolute(¶ms.bundle_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let target_path = lexical_absolute(¶ms.target_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let replacement = replacement_text(params)?;
|
||||||
|
let extra_inputs = params
|
||||||
|
.replacement_path
|
||||||
|
.as_ref()
|
||||||
|
.map(|path| lexical_absolute(path).map_err(anyhow::Error::msg))
|
||||||
|
.transpose()?;
|
||||||
|
let mut inputs = vec![bundle_path.as_path()];
|
||||||
|
if let Some(path) = extra_inputs.as_ref() {
|
||||||
|
inputs.push(path.as_path());
|
||||||
|
}
|
||||||
|
ensure_target_is_not_input(&target_path, &inputs)?;
|
||||||
|
|
||||||
|
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
|
||||||
|
let patch = StringFieldPatch {
|
||||||
|
serialized_file_path: params.serialized_file_path.clone(),
|
||||||
|
path_id: params.path_id,
|
||||||
|
field_path: params.field_path.clone(),
|
||||||
|
expected_value: params.expected_value.clone(),
|
||||||
|
replacement: replacement.clone(),
|
||||||
|
};
|
||||||
|
let target = patch_unityfs_string_field(&bundle, &patch)?;
|
||||||
|
write_output_file(&target_path, &target, "UnityFS target")?;
|
||||||
|
|
||||||
|
Ok(UnityFsPatchReport {
|
||||||
|
command: "unityfs.patch_string_field",
|
||||||
|
status: "patched",
|
||||||
|
message: "UnityFS TypeTree string field patch 已应用并原子写入目标 bundle",
|
||||||
|
bundle_path,
|
||||||
|
serialized_file_path: params.serialized_file_path.clone(),
|
||||||
|
path_id: params.path_id,
|
||||||
|
field_path: Some(params.field_path.clone()),
|
||||||
|
target_path,
|
||||||
|
source_size: bundle.len() as u64,
|
||||||
|
replacement_size: replacement.len() as u64,
|
||||||
|
target_size: target.len() as u64,
|
||||||
|
source_blake3: blake3_hex(&bundle),
|
||||||
|
replacement_blake3: blake3_hex(replacement.as_bytes()),
|
||||||
|
target_blake3: blake3_hex(&target),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Patches one semantic TypeTree field and writes the rebuilt UnityFS bundle atomically.
|
||||||
|
pub fn apply_unityfs_field_patch_file(
|
||||||
|
params: &UnityFsFieldPatchParams,
|
||||||
|
) -> anyhow::Result<UnityFsPatchReport> {
|
||||||
|
let bundle_path = lexical_absolute(¶ms.bundle_path).map_err(anyhow::Error::msg)?;
|
||||||
|
let target_path = lexical_absolute(¶ms.target_path).map_err(anyhow::Error::msg)?;
|
||||||
|
ensure_target_is_not_input(&target_path, &[&bundle_path])?;
|
||||||
|
|
||||||
|
let bundle = read_required_file(&bundle_path, "UnityFS bundle")?;
|
||||||
|
let replacement_size = serde_json::to_vec(¶ms.replacement)
|
||||||
|
.map_err(anyhow::Error::from)?
|
||||||
|
.len() as u64;
|
||||||
|
let patch = FieldPatch {
|
||||||
|
serialized_file_path: params.serialized_file_path.clone(),
|
||||||
|
path_id: params.path_id,
|
||||||
|
field_path: params.field_path.clone(),
|
||||||
|
expected_value: params.expected_value.clone(),
|
||||||
|
replacement: params.replacement.clone(),
|
||||||
|
};
|
||||||
|
let target = patch_unityfs_field(&bundle, &patch)?;
|
||||||
|
write_output_file(&target_path, &target, "UnityFS target")?;
|
||||||
|
|
||||||
|
Ok(UnityFsPatchReport {
|
||||||
|
command: "unityfs.patch_field",
|
||||||
|
status: "patched",
|
||||||
|
message: "UnityFS TypeTree field patch 已应用并原子写入目标 bundle",
|
||||||
|
bundle_path,
|
||||||
|
serialized_file_path: params.serialized_file_path.clone(),
|
||||||
|
path_id: params.path_id,
|
||||||
|
field_path: Some(params.field_path.clone()),
|
||||||
|
target_path,
|
||||||
|
source_size: bundle.len() as u64,
|
||||||
|
replacement_size,
|
||||||
|
target_size: target.len() as u64,
|
||||||
|
source_blake3: blake3_hex(&bundle),
|
||||||
|
replacement_blake3: blake3_hex(&serde_json::to_vec(¶ms.replacement)?),
|
||||||
|
target_blake3: blake3_hex(&target),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replacement_text(params: &UnityFsStringFieldPatchParams) -> anyhow::Result<String> {
|
||||||
|
match (¶ms.replacement_text, ¶ms.replacement_path) {
|
||||||
|
(Some(_), Some(_)) => Err(anyhow::anyhow!(
|
||||||
|
"replacement_text 和 replacement_path 只能指定一个"
|
||||||
|
)),
|
||||||
|
(Some(text), None) => Ok(text.clone()),
|
||||||
|
(None, Some(path)) => {
|
||||||
|
let path = lexical_absolute(path).map_err(anyhow::Error::msg)?;
|
||||||
|
let bytes = read_required_file(&path, "string replacement")?;
|
||||||
|
String::from_utf8(bytes)
|
||||||
|
.map_err(|error| anyhow::anyhow!("string replacement 不是 UTF-8:{error}"))
|
||||||
|
}
|
||||||
|
(None, None) => Err(anyhow::anyhow!(
|
||||||
|
"必须指定 replacement_text 或 replacement_path"
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_required_file(path: &Path, label: &str) -> anyhow::Result<Vec<u8>> {
|
||||||
|
read_file_no_symlink(path, label)
|
||||||
|
.map_err(anyhow::Error::msg)?
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("{label} 不存在:{}", path.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_output_file(path: &Path, bytes: &[u8], label: &str) -> anyhow::Result<()> {
|
||||||
|
let parent = path
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("{label} 缺少父目录:{}", path.display()))?;
|
||||||
|
ensure_safe_file_target(parent, path, label).map_err(anyhow::Error::msg)?;
|
||||||
|
write_file_atomic(path, bytes, STATE_FILE_MODE, label).map_err(anyhow::Error::msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_target_is_not_input(target: &Path, inputs: &[&Path]) -> anyhow::Result<()> {
|
||||||
|
for input in inputs {
|
||||||
|
if target == *input {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"target_path 不能与输入文件相同:{}",
|
||||||
|
target.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn blake3_hex(bytes: &[u8]) -> String {
|
||||||
|
blake3::hash(bytes).to_hex().to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patch_apply_rejects_in_place_target() {
|
||||||
|
let path = PathBuf::from("/tmp/source.bin");
|
||||||
|
let params = PatchApplyParams {
|
||||||
|
kind: PatchApplyKind::Binary,
|
||||||
|
source_path: path.clone(),
|
||||||
|
patch_path: PathBuf::from("/tmp/patch.json"),
|
||||||
|
target_path: path,
|
||||||
|
};
|
||||||
|
let error = apply_patch_file(¶ms).unwrap_err().to_string();
|
||||||
|
assert!(error.contains("target_path 不能与输入文件相同"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn string_field_params_require_one_replacement_source() {
|
||||||
|
let params = UnityFsStringFieldPatchParams {
|
||||||
|
bundle_path: PathBuf::from("/tmp/source.bundle"),
|
||||||
|
serialized_file_path: "CAB".to_string(),
|
||||||
|
path_id: 1,
|
||||||
|
field_path: "message".to_string(),
|
||||||
|
replacement_text: Some("a".to_string()),
|
||||||
|
replacement_path: Some(PathBuf::from("/tmp/replacement.txt")),
|
||||||
|
target_path: PathBuf::from("/tmp/target.bundle"),
|
||||||
|
expected_value: None,
|
||||||
|
};
|
||||||
|
let error = replacement_text(¶ms).unwrap_err().to_string();
|
||||||
|
assert!(error.contains("只能指定一个"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
//! 内存资源仓储实现。
|
//! 内存资源仓储实现。
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use bat_core::domain::{Resource, ResourceEntry, ResourceType};
|
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
|
||||||
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
|
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
|
||||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult};
|
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult};
|
||||||
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
|
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
|
||||||
@@ -130,7 +130,8 @@ impl SqliteResourceRepository {
|
|||||||
local_path TEXT NOT NULL,
|
local_path TEXT NOT NULL,
|
||||||
address TEXT,
|
address TEXT,
|
||||||
dependencies_json TEXT NOT NULL DEFAULT '[]',
|
dependencies_json TEXT NOT NULL DEFAULT '[]',
|
||||||
crc INTEGER
|
crc INTEGER,
|
||||||
|
metadata_json TEXT NOT NULL DEFAULT '{}'
|
||||||
)
|
)
|
||||||
"#,
|
"#,
|
||||||
),
|
),
|
||||||
@@ -140,6 +141,13 @@ impl SqliteResourceRepository {
|
|||||||
// 向后兼容:早于 crc 列的旧库缺少该列,按需补加(新建库已含该列,
|
// 向后兼容:早于 crc 列的旧库缺少该列,按需补加(新建库已含该列,
|
||||||
// pragma 检查后不会重复 ALTER)。
|
// pragma 检查后不会重复 ALTER)。
|
||||||
Self::ensure_column(&self.pool, "resources", "crc", "INTEGER").await?;
|
Self::ensure_column(&self.pool, "resources", "crc", "INTEGER").await?;
|
||||||
|
Self::ensure_column(
|
||||||
|
&self.pool,
|
||||||
|
"resources",
|
||||||
|
"metadata_json",
|
||||||
|
"TEXT NOT NULL DEFAULT '{}'",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Self::execute_query(
|
Self::execute_query(
|
||||||
&self.pool,
|
&self.pool,
|
||||||
@@ -250,9 +258,32 @@ impl SqliteResourceRepository {
|
|||||||
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
|
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn metadata_to_json(metadata: &ResourceMetadata) -> bat_core::Result<String> {
|
||||||
|
serde_json::to_string(metadata)
|
||||||
|
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metadata_from_json(value: &str) -> bat_core::Result<ResourceMetadata> {
|
||||||
|
if value.trim().is_empty() {
|
||||||
|
return Ok(ResourceMetadata::default());
|
||||||
|
}
|
||||||
|
serde_json::from_str(value)
|
||||||
|
.map_err(|error| bat_core::Error::Serialization(error.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
fn resource_from_row(row: ResourceRow) -> bat_core::Result<Resource> {
|
fn resource_from_row(row: ResourceRow) -> bat_core::Result<Resource> {
|
||||||
let (id, path, hash, size, resource_type, local_path, address, dependencies_json, crc) =
|
let (
|
||||||
row;
|
id,
|
||||||
|
path,
|
||||||
|
hash,
|
||||||
|
size,
|
||||||
|
resource_type,
|
||||||
|
local_path,
|
||||||
|
address,
|
||||||
|
dependencies_json,
|
||||||
|
crc,
|
||||||
|
metadata_json,
|
||||||
|
) = row;
|
||||||
Ok(Resource {
|
Ok(Resource {
|
||||||
id,
|
id,
|
||||||
local_path: PathBuf::from(local_path),
|
local_path: PathBuf::from(local_path),
|
||||||
@@ -265,6 +296,7 @@ impl SqliteResourceRepository {
|
|||||||
dependencies: Self::dependencies_from_json(&dependencies_json)?,
|
dependencies: Self::dependencies_from_json(&dependencies_json)?,
|
||||||
crc: crc.and_then(|value| u32::try_from(value).ok()),
|
crc: crc.and_then(|value| u32::try_from(value).ok()),
|
||||||
},
|
},
|
||||||
|
metadata: Self::metadata_from_json(&metadata_json)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -303,7 +335,7 @@ impl SqliteResourceRepository {
|
|||||||
limit: Option<usize>,
|
limit: Option<usize>,
|
||||||
) -> bat_core::Result<Vec<Resource>> {
|
) -> bat_core::Result<Vec<Resource>> {
|
||||||
let mut builder = QueryBuilder::<Sqlite>::new(
|
let mut builder = QueryBuilder::<Sqlite>::new(
|
||||||
"SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc FROM resources",
|
"SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json FROM resources",
|
||||||
);
|
);
|
||||||
Self::apply_filters(&mut builder, query)?;
|
Self::apply_filters(&mut builder, query)?;
|
||||||
builder.push(" ORDER BY id");
|
builder.push(" ORDER BY id");
|
||||||
@@ -337,14 +369,15 @@ impl SqliteResourceRepository {
|
|||||||
impl ResourceRepository for SqliteResourceRepository {
|
impl ResourceRepository for SqliteResourceRepository {
|
||||||
async fn add(&self, resource: Resource) -> bat_core::Result<String> {
|
async fn add(&self, resource: Resource) -> bat_core::Result<String> {
|
||||||
let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?;
|
let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?;
|
||||||
|
let metadata = Self::metadata_to_json(&resource.metadata)?;
|
||||||
Self::execute_query(
|
Self::execute_query(
|
||||||
&self.pool,
|
&self.pool,
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
r#"
|
r#"
|
||||||
INSERT INTO resources (
|
INSERT INTO resources (
|
||||||
id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
|
id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
|
||||||
)
|
)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||||
ON CONFLICT(id) DO UPDATE SET
|
ON CONFLICT(id) DO UPDATE SET
|
||||||
path = excluded.path,
|
path = excluded.path,
|
||||||
hash = excluded.hash,
|
hash = excluded.hash,
|
||||||
@@ -353,7 +386,8 @@ impl ResourceRepository for SqliteResourceRepository {
|
|||||||
local_path = excluded.local_path,
|
local_path = excluded.local_path,
|
||||||
address = excluded.address,
|
address = excluded.address,
|
||||||
dependencies_json = excluded.dependencies_json,
|
dependencies_json = excluded.dependencies_json,
|
||||||
crc = excluded.crc
|
crc = excluded.crc,
|
||||||
|
metadata_json = excluded.metadata_json
|
||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
.bind(resource.id.clone())
|
.bind(resource.id.clone())
|
||||||
@@ -364,7 +398,8 @@ impl ResourceRepository for SqliteResourceRepository {
|
|||||||
.bind(resource.local_path.to_string_lossy().to_string())
|
.bind(resource.local_path.to_string_lossy().to_string())
|
||||||
.bind(resource.entry.address.clone())
|
.bind(resource.entry.address.clone())
|
||||||
.bind(dependencies)
|
.bind(dependencies)
|
||||||
.bind(resource.entry.crc.map(i64::from)),
|
.bind(resource.entry.crc.map(i64::from))
|
||||||
|
.bind(metadata),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -374,7 +409,7 @@ impl ResourceRepository for SqliteResourceRepository {
|
|||||||
async fn find_by_id(&self, id: &str) -> bat_core::Result<Resource> {
|
async fn find_by_id(&self, id: &str) -> bat_core::Result<Resource> {
|
||||||
let row: Option<ResourceRow> = sqlx::query_as(
|
let row: Option<ResourceRow> = sqlx::query_as(
|
||||||
r#"
|
r#"
|
||||||
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
|
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
|
||||||
FROM resources
|
FROM resources
|
||||||
WHERE id = ?1
|
WHERE id = ?1
|
||||||
"#,
|
"#,
|
||||||
@@ -392,7 +427,7 @@ impl ResourceRepository for SqliteResourceRepository {
|
|||||||
async fn find_by_hash(&self, hash: &str) -> bat_core::Result<Resource> {
|
async fn find_by_hash(&self, hash: &str) -> bat_core::Result<Resource> {
|
||||||
let row: Option<ResourceRow> = sqlx::query_as(
|
let row: Option<ResourceRow> = sqlx::query_as(
|
||||||
r#"
|
r#"
|
||||||
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc
|
SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json, crc, metadata_json
|
||||||
FROM resources
|
FROM resources
|
||||||
WHERE hash = ?1
|
WHERE hash = ?1
|
||||||
ORDER BY id
|
ORDER BY id
|
||||||
@@ -455,6 +490,7 @@ type ResourceRow = (
|
|||||||
Option<String>,
|
Option<String>,
|
||||||
String,
|
String,
|
||||||
Option<i64>,
|
Option<i64>,
|
||||||
|
String,
|
||||||
);
|
);
|
||||||
|
|
||||||
fn glob_to_like(pattern: &str) -> String {
|
fn glob_to_like(pattern: &str) -> String {
|
||||||
@@ -544,6 +580,7 @@ mod tests {
|
|||||||
dependencies: Vec::new(),
|
dependencies: Vec::new(),
|
||||||
crc: None,
|
crc: None,
|
||||||
},
|
},
|
||||||
|
metadata: ResourceMetadata::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -638,6 +675,10 @@ mod tests {
|
|||||||
.entry
|
.entry
|
||||||
.dependencies
|
.dependencies
|
||||||
.push("assets/shared.bundle".to_string());
|
.push("assets/shared.bundle".to_string());
|
||||||
|
resource.metadata.official_release_id = Some("release-1".to_string());
|
||||||
|
resource.metadata.platform = Some("windows".to_string());
|
||||||
|
resource.metadata.text_assets = vec!["Scenario".to_string()];
|
||||||
|
resource.metadata.text_unit_count = 3;
|
||||||
|
|
||||||
repository.add(resource.clone()).await.unwrap();
|
repository.add(resource.clone()).await.unwrap();
|
||||||
|
|
||||||
@@ -647,6 +688,13 @@ mod tests {
|
|||||||
by_id.entry.dependencies,
|
by_id.entry.dependencies,
|
||||||
vec!["assets/shared.bundle".to_string()]
|
vec!["assets/shared.bundle".to_string()]
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
by_id.metadata.official_release_id.as_deref(),
|
||||||
|
Some("release-1")
|
||||||
|
);
|
||||||
|
assert_eq!(by_id.metadata.platform.as_deref(), Some("windows"));
|
||||||
|
assert_eq!(by_id.metadata.text_assets, vec!["Scenario".to_string()]);
|
||||||
|
assert_eq!(by_id.metadata.text_unit_count, 3);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
repository.find_by_hash("hash-sqlite-a").await.unwrap().id,
|
repository.find_by_hash("hash-sqlite-a").await.unwrap().id,
|
||||||
resource.id
|
resource.id
|
||||||
|
|||||||
@@ -367,14 +367,16 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
|
|||||||
let curl_failure_state = harness.temp.path().join("curl-failure.state");
|
let curl_failure_state = harness.temp.path().join("curl-failure.state");
|
||||||
write_executable(
|
write_executable(
|
||||||
&harness.curl_script,
|
&harness.curl_script,
|
||||||
&official_curl_script(
|
&official_curl_script(OfficialCurlScriptFixture {
|
||||||
&harness.curl_log,
|
curl_log: &harness.curl_log,
|
||||||
&zip_fixture,
|
zip_fixture: &zip_fixture,
|
||||||
&resources_assets_path,
|
resources_assets_fixture: &resources_assets_path,
|
||||||
fs::metadata(&resources_assets_path).unwrap().len() as usize,
|
resources_assets_size: fs::metadata(&resources_assets_path).unwrap().len() as usize,
|
||||||
TEST_LAUNCHER_MANIFEST_SOURCE,
|
manifest_source: TEST_LAUNCHER_MANIFEST_SOURCE,
|
||||||
Some(&curl_failure_state),
|
failure_state: Some(&curl_failure_state),
|
||||||
),
|
clientpatch_unavailable: false,
|
||||||
|
seed_unavailable_only: false,
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
let config = harness.sync_config("failed-staging-output");
|
let config = harness.sync_config("failed-staging-output");
|
||||||
@@ -399,6 +401,77 @@ fn official_update_reuses_failed_staging_after_interrupted_download() {
|
|||||||
assert!(version_state.current_completed_version.is_some());
|
assert!(version_state.current_completed_version.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn official_update_waits_when_client_patch_markers_are_not_ready() {
|
||||||
|
let harness = TestHarness::new_with_clientpatch_unavailable(false);
|
||||||
|
let config = harness.sync_config("clientpatch-marker-wait-output");
|
||||||
|
let mut events = Vec::<OfficialUpdateProgress>::new();
|
||||||
|
|
||||||
|
let report = OfficialUpdateService::new()
|
||||||
|
.run_with_progress(&config, |event| events.push(event))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
report.update_status,
|
||||||
|
OfficialUpdateStatus::WaitingForOfficialResources
|
||||||
|
);
|
||||||
|
assert!(report.waiting_for_official_resources);
|
||||||
|
assert!(!report.should_download);
|
||||||
|
assert!(report
|
||||||
|
.unavailable_endpoints
|
||||||
|
.iter()
|
||||||
|
.any(|endpoint| endpoint.url.ends_with("/TableBundles/TableCatalog.hash")));
|
||||||
|
assert!(report
|
||||||
|
.unavailable_endpoints
|
||||||
|
.iter()
|
||||||
|
.all(|endpoint| endpoint.error_kind == "http_forbidden"));
|
||||||
|
assert!(report.staging_path.is_none());
|
||||||
|
assert!(report.published_version_path.is_none());
|
||||||
|
assert!(!config.output_root.join("current").exists());
|
||||||
|
assert!(!config.output_root.join(".staging").exists());
|
||||||
|
assert!(read_version_state(&report.version_state_path)
|
||||||
|
.unwrap()
|
||||||
|
.is_none());
|
||||||
|
assert!(events
|
||||||
|
.iter()
|
||||||
|
.any(|event| event.stage == "upstream" && event.message.contains("尚未开放")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn official_update_waits_when_required_seed_catalog_is_not_ready() {
|
||||||
|
let harness = TestHarness::new_with_clientpatch_unavailable(true);
|
||||||
|
let config = harness.sync_config("clientpatch-seed-wait-output");
|
||||||
|
let mut events = Vec::<OfficialUpdateProgress>::new();
|
||||||
|
|
||||||
|
let report = OfficialUpdateService::new()
|
||||||
|
.run_with_progress(&config, |event| events.push(event))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
report.update_status,
|
||||||
|
OfficialUpdateStatus::WaitingForOfficialResources
|
||||||
|
);
|
||||||
|
assert!(report.waiting_for_official_resources);
|
||||||
|
assert_eq!(report.unavailable_endpoints.len(), 1);
|
||||||
|
assert!(report.unavailable_endpoints[0]
|
||||||
|
.url
|
||||||
|
.ends_with("/TableBundles/TableCatalog.bytes"));
|
||||||
|
assert_eq!(report.unavailable_endpoints[0].http_status, Some(403));
|
||||||
|
assert!(report.staging_path.is_none());
|
||||||
|
assert!(report.published_version_path.is_none());
|
||||||
|
assert!(!config.output_root.join("current").exists());
|
||||||
|
assert!(!config.output_root.join(".staging").exists());
|
||||||
|
assert!(read_version_state(&report.version_state_path)
|
||||||
|
.unwrap()
|
||||||
|
.is_none());
|
||||||
|
assert!(events
|
||||||
|
.iter()
|
||||||
|
.any(|event| event.stage == "catalog" && event.message.contains("TableCatalog.bytes")));
|
||||||
|
assert!(events
|
||||||
|
.iter()
|
||||||
|
.any(|event| event.stage == "upstream" && event.message.contains("必需资源")));
|
||||||
|
}
|
||||||
|
|
||||||
struct TestHarness {
|
struct TestHarness {
|
||||||
temp: TempDir,
|
temp: TempDir,
|
||||||
curl_script: std::path::PathBuf,
|
curl_script: std::path::PathBuf,
|
||||||
@@ -412,6 +485,18 @@ impl TestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn new_with_manifest_source(manifest_source: &str) -> Self {
|
fn new_with_manifest_source(manifest_source: &str) -> Self {
|
||||||
|
Self::new_with_options(manifest_source, false, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new_with_clientpatch_unavailable(seed_only: bool) -> Self {
|
||||||
|
Self::new_with_options(TEST_LAUNCHER_MANIFEST_SOURCE, true, seed_only)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new_with_options(
|
||||||
|
manifest_source: &str,
|
||||||
|
clientpatch_unavailable: bool,
|
||||||
|
seed_unavailable_only: bool,
|
||||||
|
) -> Self {
|
||||||
let temp = TempDir::new().unwrap();
|
let temp = TempDir::new().unwrap();
|
||||||
let resources_assets = synthetic_resources_assets();
|
let resources_assets = synthetic_resources_assets();
|
||||||
|
|
||||||
@@ -425,14 +510,16 @@ impl TestHarness {
|
|||||||
let curl_script = temp.path().join("fake-curl");
|
let curl_script = temp.path().join("fake-curl");
|
||||||
write_executable(
|
write_executable(
|
||||||
&curl_script,
|
&curl_script,
|
||||||
&official_curl_script(
|
&official_curl_script(OfficialCurlScriptFixture {
|
||||||
&curl_log,
|
curl_log: &curl_log,
|
||||||
&zip_fixture,
|
zip_fixture: &zip_fixture,
|
||||||
&resources_assets_path,
|
resources_assets_fixture: &resources_assets_path,
|
||||||
resources_assets.len(),
|
resources_assets_size: resources_assets.len(),
|
||||||
manifest_source,
|
manifest_source,
|
||||||
None,
|
failure_state: None,
|
||||||
),
|
clientpatch_unavailable,
|
||||||
|
seed_unavailable_only,
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
let unzip_script = temp.path().join("fake-unzip");
|
let unzip_script = temp.path().join("fake-unzip");
|
||||||
@@ -542,15 +629,20 @@ fn one_file_zip(name: &[u8], data: &[u8]) -> Vec<u8> {
|
|||||||
bytes
|
bytes
|
||||||
}
|
}
|
||||||
|
|
||||||
fn official_curl_script(
|
struct OfficialCurlScriptFixture<'a> {
|
||||||
curl_log: &Path,
|
curl_log: &'a Path,
|
||||||
zip_fixture: &Path,
|
zip_fixture: &'a Path,
|
||||||
resources_assets_fixture: &Path,
|
resources_assets_fixture: &'a Path,
|
||||||
resources_assets_size: usize,
|
resources_assets_size: usize,
|
||||||
manifest_source: &str,
|
manifest_source: &'a str,
|
||||||
failure_state: Option<&Path>,
|
failure_state: Option<&'a Path>,
|
||||||
) -> String {
|
clientpatch_unavailable: bool,
|
||||||
let failure_state = failure_state
|
seed_unavailable_only: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn official_curl_script(fixture: OfficialCurlScriptFixture<'_>) -> String {
|
||||||
|
let failure_state = fixture
|
||||||
|
.failure_state
|
||||||
.map(shell_quote)
|
.map(shell_quote)
|
||||||
.unwrap_or_else(|| "''".to_string());
|
.unwrap_or_else(|| "''".to_string());
|
||||||
format!(
|
format!(
|
||||||
@@ -614,6 +706,31 @@ maybe_fail_once() {{
|
|||||||
fi
|
fi
|
||||||
}}
|
}}
|
||||||
|
|
||||||
|
clientpatch_is_unavailable() {{
|
||||||
|
if [[ "{clientpatch_unavailable}" != "true" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ "$url" != "{addressables_root}/"* ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ "{seed_unavailable_only}" == "true" ]]; then
|
||||||
|
case "$url" in
|
||||||
|
*/TableBundles/TableCatalog.bytes|*/BundlePackingInfo.bytes|*/Catalog/MediaCatalog.bytes)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}}
|
||||||
|
|
||||||
|
if clientpatch_is_unavailable; then
|
||||||
|
echo "curl: (22) The requested URL returned error: 403" >&2
|
||||||
|
exit 22
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$url" == "https://api-launcher-jp.yo-star.com/api/launcher/game/config" ]]; then
|
if [[ "$url" == "https://api-launcher-jp.yo-star.com/api/launcher/game/config" ]]; then
|
||||||
cat <<'JSON'
|
cat <<'JSON'
|
||||||
{{"code":200,"message":"ok","data":{{"game_latest_version":"{launcher_latest_version}","game_latest_file_path":"{launcher_latest_file_path}"}}}}
|
{{"code":200,"message":"ok","data":{{"game_latest_version":"{launcher_latest_version}","game_latest_file_path":"{launcher_latest_file_path}"}}}}
|
||||||
@@ -679,16 +796,16 @@ else
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
"#,
|
"#,
|
||||||
shell_quote(curl_log),
|
shell_quote(fixture.curl_log),
|
||||||
shell_quote(zip_fixture),
|
shell_quote(fixture.zip_fixture),
|
||||||
shell_quote(resources_assets_fixture),
|
shell_quote(fixture.resources_assets_fixture),
|
||||||
launcher_latest_version = TEST_LAUNCHER_LATEST_VERSION,
|
launcher_latest_version = TEST_LAUNCHER_LATEST_VERSION,
|
||||||
launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH,
|
launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH,
|
||||||
launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL,
|
launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL,
|
||||||
launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL,
|
launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL,
|
||||||
launcher_manifest_source = manifest_source,
|
launcher_manifest_source = fixture.manifest_source,
|
||||||
launcher_resources_assets_url = TEST_LAUNCHER_RESOURCES_ASSETS_URL,
|
launcher_resources_assets_url = TEST_LAUNCHER_RESOURCES_ASSETS_URL,
|
||||||
resources_assets_size = resources_assets_size,
|
resources_assets_size = fixture.resources_assets_size,
|
||||||
server_info_url = TEST_SERVER_INFO_URL,
|
server_info_url = TEST_SERVER_INFO_URL,
|
||||||
connection_group = TEST_CONNECTION_GROUP,
|
connection_group = TEST_CONNECTION_GROUP,
|
||||||
addressables_root = TEST_ADDRESSABLES_ROOT,
|
addressables_root = TEST_ADDRESSABLES_ROOT,
|
||||||
@@ -698,6 +815,8 @@ fi
|
|||||||
android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"),
|
android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"),
|
||||||
android_media_catalog_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"),
|
android_media_catalog_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"),
|
||||||
failure_state = failure_state,
|
failure_state = failure_state,
|
||||||
|
clientpatch_unavailable = fixture.clientpatch_unavailable,
|
||||||
|
seed_unavailable_only = fixture.seed_unavailable_only,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user