From 789402c8878d0e46bef3941e538463705566d252 Mon Sep 17 00:00:00 2001 From: Yuyi-Oak <1722157266@qq.com> Date: Sun, 5 Jul 2026 23:49:56 +0800 Subject: [PATCH] feat: add official resource sync pipeline Add the production-facing official update service and bat-official-sync watch CLI for unattended resource synchronization. Support launcher-resource discovery without installing the launcher, remote marker snapshots, local manifest audit and repair, official seed hash validation, bootstrap caching, richer Addressables coverage, SQLite resource persistence, and FFI JSON helpers. --- CURRENT_STATUS.md | 3 +- Cargo.lock | 6 + PROJECT_PLAN.md | 18 +- README.md | 2 +- adapters/src/manifest/addressables.rs | 685 +++++++- adapters/src/official/game_main_config.rs | 8 +- adapters/src/official/mod.rs | 4 +- adapters/src/official/yostar_jp.rs | 16 +- adapters/src/unity/serialized_file.rs | 21 +- .../fixtures/real_addressables/catalog.json | 22 + adapters/tests/golden/real_addressables.json | 45 + adapters/tests/real_addressables_golden.rs | 34 + adapters/tests/real_addressables_local.rs | 3 + core/src/domain/resource.rs | 6 + crates/bat-ffi/Cargo.toml | 4 + crates/bat-ffi/src/lib.rs | 392 ++++- docs/architecture/README.md | 5 +- .../0001-engine-and-application-boundaries.md | 6 +- ...3-cas-core-interface-and-error-boundary.md | 2 +- .../architecture/official-resource-backend.md | 73 +- docs/guides/official-resource-test-pull.md | 148 +- docs/reports/current-stage-prepush.md | 4 +- examples/manifest_demo.rs | 9 +- infrastructure/Cargo.toml | 6 + infrastructure/examples/official_pull_plan.rs | 145 +- .../examples/official_update_check.rs | 1133 ++++++++++++++ infrastructure/src/bin/bat_official_sync.rs | 447 ++++++ infrastructure/src/import.rs | 4 + infrastructure/src/lib.rs | 17 +- infrastructure/src/official_download.rs | 1371 ++++++++++++++++- .../src/official_game_main_config.rs | 46 +- infrastructure/src/official_update.rs | 1193 ++++++++++++++ infrastructure/src/resources.rs | 413 ++++- .../official_game_main_config_bootstrap.rs | 91 +- internal/ffi/ffi.go | 57 + 35 files changed, 6262 insertions(+), 177 deletions(-) create mode 100644 adapters/tests/fixtures/real_addressables/catalog.json create mode 100644 adapters/tests/golden/real_addressables.json create mode 100644 adapters/tests/real_addressables_golden.rs create mode 100644 infrastructure/examples/official_update_check.rs create mode 100644 infrastructure/src/bin/bat_official_sync.rs create mode 100644 infrastructure/src/official_update.rs create mode 100644 internal/ffi/ffi.go diff --git a/CURRENT_STATUS.md b/CURRENT_STATUS.md index 02a73a2..455d7fc 100644 --- a/CURRENT_STATUS.md +++ b/CURRENT_STATUS.md @@ -101,7 +101,8 @@ - Unity bundle parse/serialize 仍是后续阶段能力。 - Addressables Catalog 复杂字段解析未完成。 - 客户端发现、备份、应用补丁流程尚未连接真实实现。 -- 官方 launcher bootstrap、GameMainConfig bootstrap、官方 pull plan 的用户级文档已补齐。 +- 官方资源 pull plan / update check 已明确 Linux 生产路径：显式 `--auto-discover` 或已审计 metadata snapshot，不安装、不执行官方启动器。 +- 官方 launcher bootstrap、GameMainConfig bootstrap 保留为底层显式开发/审计辅助路径。 - 官方 bootstrap / pull 测试夹具已收敛到 `experiment` 分支语义下，并通过了官方链路回归。 ### `bat-cas-engine` diff --git a/Cargo.lock b/Cargo.lock index 8bfefaa..ee192a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,13 +181,17 @@ name = "bat-ffi" version = "0.1.0" dependencies = [ "anyhow", + "bat-adapters", "bat-assetbundle", "bat-cas-engine", + "bat-core", + "bat-infrastructure", "bat-patch", "cbindgen", "libc", "serde", "serde_json", + "tokio", ] [[package]] @@ -199,10 +203,12 @@ dependencies = [ "bat-adapters", "bat-cas-engine", "bat-core", + "blake3", "hex", "md-5", "serde", "serde_json", + "sqlx", "tempfile", "thiserror", "tokio", diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index 1dcef39..7764f21 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -15,7 +15,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 2. **Rust 核心引擎**：负责 CAS、AssetBundle 解析、Patch、二进制安全处理和性能敏感逻辑。 3. **Go 服务层**：负责 CLI 编排、资源同步、下载器、API Server、任务调度和外部集成。 4. **Web 管理后台**：负责翻译审核、术语管理、全文搜索、历史版本、Diff 和 Dashboard。 -5. **SDK/API**：提供稳定的 Go SDK、FFI 边界和 REST/OpenAPI 接口，方便其他工具复用。 +5. **SDK/API**：提供稳定的 Go SDK、进程边界或必要时的 FFI 边界和 REST/OpenAPI 接口，方便其他工具复用。 6. **插件系统**：允许新增解析器、翻译 Provider、存储后端、Patch 算法，而不修改核心代码。 --- @@ -62,7 +62,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 ### 3.2 技术决策 -1. **Rust**：保留为核心引擎语言，用于 CAS、AssetBundle、Patch、FFI。 +1. **Rust**：保留为核心引擎语言，用于 CAS、AssetBundle、Patch、完整资源拉取和更新检查核心逻辑，FFI 仅作为可选边界。 2. **Go**：用于 CLI、同步器、API Server、任务编排、Provider 集成。 3. **PostgreSQL**：作为服务端主数据库，承载翻译记忆库、术语库、任务、审核和用户权限。 4. **SQLite**：仅作为本地 CLI 可选元数据后端，必须通过仓储抽象隔离，不能绑定业务逻辑。 @@ -133,7 +133,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 **目标**：完成可长期使用的 Content Addressable Storage。 -**当前状态**：已完成 CAS V1。FFI/Go 更高层调用边界将在 CLI/SDK 阶段继续扩展。 +**当前状态**：已完成 CAS V1。Go CLI 以最小稳定入口优先，Rust 继续承载完整资源拉取与更新检查核心逻辑；FFI 边界保持可选。 交付物： @@ -142,7 +142,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 3. 实现引用计数、对象元数据、完整性校验、GC、统计信息。 4. 实现本地元数据后端：优先 SQLite，但必须隔离在 repository adapter 中。 5. 编写迁移、恢复、损坏检测和 `doctor cas`。 -6. 提供 FFI/Go 调用边界。 +6. 提供稳定的 Go 调用边界，优先进程或 SDK，必要时再补 FFI。 验收标准： @@ -165,6 +165,8 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 3. 实现 Go 下载器：并发、断点续传、限速、重试、校验、缓存。 4. 实现 `sync`、`manifest inspect`、`cache status`。 5. 将下载结果写入 CAS，并写入 Resource Repository。 +6. Linux 生产同步支持显式 `--auto-discover` 或已审计 metadata snapshot，不依赖已安装官方启动器。 +7. 实现自动更新检查：保存上次官方 snapshot，定期 discovery，对比变化后按需拉取。 验收标准： @@ -172,6 +174,8 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 2. 断点续传和失败重试有可重复测试。 3. Manifest 解析失败时给出可定位的字段和偏移信息。 4. 同一资源跨版本复用同一 CAS 对象。 +5. 生产同步入口必须显式选择 `--auto-discover` 或显式提供官方 `server-info` URL、`connection-group` 和 `app-version`，不得隐式启动 launcher/bootstrap 链路。 +6. 自动更新入口必须做到无变化不下载，有变化下载成功后才写入新 snapshot。 --- @@ -394,9 +398,9 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 处理策略： -1. Rust 提供稳定引擎能力，不承担 CLI 编排。 -2. Go 负责用户命令、服务编排、网络和 Provider。 -3. FFI 只暴露粗粒度、安全、可测试 API。 +1. Rust 提供稳定引擎能力，不承担 CLI 编排，但负责完整资源拉取和更新检查的核心逻辑。 +2. Go 负责用户命令、最小稳定 CLI、服务编排、网络和 Provider。 +3. 跨边界优先进程或 SDK，FFI 只作为可选的粗粒度、安全、可测试 API。 ### 过早做 Web diff --git a/README.md b/README.md index 57ded1b..b7e056e 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ - `bat-cas-engine` CAS V1：原子写入、Hash 校验、引用计数、GC、并发写入测试、损坏检测。 - `bat-infrastructure` CAS 仓储适配层。 - `bat-ffi` 基础 crate 边界。 -- 官方日服资源 bootstrap、官方 launcher bootstrap、官方 pull plan 用户流程。 +- 官方日服资源 auto-discover、pull plan、update check 用户流程；launcher bootstrap 仅作为底层显式开发/审计辅助路径。 - 文档路线图和当前缺口清单。 - 当前阶段说明与推送前核查文档。 diff --git a/adapters/src/manifest/addressables.rs b/adapters/src/manifest/addressables.rs index 4d8a54e..0c6e192 100644 --- a/adapters/src/manifest/addressables.rs +++ b/adapters/src/manifest/addressables.rs @@ -4,10 +4,14 @@ use super::driver::{GenericManifest, ManifestDriver, ManifestFormat, ManifestMetadata}; use async_trait::async_trait; +use base64::{engine::general_purpose::STANDARD, Engine as _}; use bat_core::domain::{ResourceEntry, ResourceType}; use serde_json::Value; use std::collections::HashMap; +const ADDRESSABLES_ENTRY_RECORD_INTS: usize = 7; +const PLATFORM_LOAD_PATH_PLACEHOLDER: &str = "{PlatformUtils.AddressableLoadPath}"; + /// Addressables Catalog Driver pub struct AddressablesCatalogDriver; @@ -19,7 +23,14 @@ impl AddressablesCatalogDriver { fn parse_json(raw_data: &[u8]) -> Result { let text = std::str::from_utf8(raw_data).map_err(|e| format!("Invalid UTF-8: {}", e))?; - serde_json::from_str(text).map_err(|e| format!("Invalid JSON: {}", e)) + serde_json::from_str(text).map_err(|e| { + format!( + "Invalid JSON at line {}, column {}: {}", + e.line(), + e.column(), + e + ) + }) } fn locator_id(json: &Value) -> Option { @@ -42,8 +53,143 @@ impl AddressablesCatalogDriver { prefixes } + fn blob_bytes(json: &Value, field: &str) -> Option> { + json.get(field) + .and_then(|value| value.as_str()) + .and_then(|value| STANDARD.decode(value).ok()) + } + + fn read_u32_le(bytes: &[u8], offset: usize) -> Option { + let end = offset.checked_add(4)?; + let slice = bytes.get(offset..end)?; + Some(u32::from_le_bytes(slice.try_into().ok()?)) + } + + fn read_i32_le(bytes: &[u8], offset: usize) -> Option { + let end = offset.checked_add(4)?; + let slice = bytes.get(offset..end)?; + Some(i32::from_le_bytes(slice.try_into().ok()?)) + } + + fn blob_count(bytes: &[u8]) -> Option { + Self::read_u32_le(bytes, 0).map(|value| value as usize) + } + + fn blob_count_field(json: &Value, field: &str) -> Option { + Self::blob_bytes(json, field).and_then(|bytes| Self::blob_count(&bytes)) + } + + fn internal_id_count(json: &Value) -> Option { + json.get("m_InternalIds") + .and_then(|value| value.as_array()) + .map(|values| values.len()) + } + + fn decode_serialized_string(bytes: &[u8], offset: usize) -> Option<(String, usize)> { + let (object, next_offset) = Self::read_serialized_object(bytes, offset)?; + object.into_key_string().map(|value| (value, next_offset)) + } + + fn read_serialized_object(bytes: &[u8], offset: usize) -> Option<(AddressablesObject, usize)> { + let object_type = *bytes.get(offset)?; + let mut cursor = offset.checked_add(1)?; + + match object_type { + 0 => { + let len = Self::read_u32_le(bytes, cursor)? as usize; + cursor = cursor.checked_add(4)?; + let end = cursor.checked_add(len)?; + let value = std::str::from_utf8(bytes.get(cursor..end)?) + .ok()? + .to_owned(); + Some((AddressablesObject::String(value), end)) + } + 1 => { + let len = Self::read_u32_le(bytes, cursor)? as usize; + cursor = cursor.checked_add(4)?; + let end = cursor.checked_add(len)?; + let units = bytes + .get(cursor..end)? + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + let value = String::from_utf16(&units).ok()?; + Some((AddressablesObject::String(value), end)) + } + 2 => { + let end = cursor.checked_add(2)?; + let value = u16::from_le_bytes(bytes.get(cursor..end)?.try_into().ok()?); + Some((AddressablesObject::Unsigned(value as u32), end)) + } + 3 => { + let end = cursor.checked_add(4)?; + let value = u32::from_le_bytes(bytes.get(cursor..end)?.try_into().ok()?); + Some((AddressablesObject::Unsigned(value), end)) + } + 4 => { + let end = cursor.checked_add(4)?; + let value = i32::from_le_bytes(bytes.get(cursor..end)?.try_into().ok()?); + Some((AddressablesObject::Signed(value), end)) + } + 5 => { + let len = *bytes.get(cursor)? as usize; + cursor = cursor.checked_add(1)?; + let end = cursor.checked_add(len)?; + let value = std::str::from_utf8(bytes.get(cursor..end)?) + .ok()? + .to_owned(); + Some((AddressablesObject::Hash128(value), end)) + } + 7 => Self::read_serialized_json_object(bytes, cursor), + _ => None, + } + } + + fn read_serialized_json_object( + bytes: &[u8], + mut cursor: usize, + ) -> Option<(AddressablesObject, usize)> { + let assembly_len = *bytes.get(cursor)? as usize; + cursor = cursor.checked_add(1)?; + let assembly_end = cursor.checked_add(assembly_len)?; + let assembly_name = std::str::from_utf8(bytes.get(cursor..assembly_end)?) + .ok()? + .to_owned(); + cursor = assembly_end; + + let class_len = *bytes.get(cursor)? as usize; + cursor = cursor.checked_add(1)?; + let class_end = cursor.checked_add(class_len)?; + let class_name = std::str::from_utf8(bytes.get(cursor..class_end)?) + .ok()? + .to_owned(); + cursor = class_end; + + let json_len = Self::read_u32_le(bytes, cursor)? as usize; + cursor = cursor.checked_add(4)?; + let json_end = cursor.checked_add(json_len)?; + let units = bytes + .get(cursor..json_end)? + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + let json_text = String::from_utf16(&units).ok()?; + let json = serde_json::from_str(&json_text).ok(); + + Some(( + AddressablesObject::JsonObject { + assembly_name, + class_name, + json, + }, + json_end, + )) + } + fn resource_type_for_path(path: &str) -> ResourceType { - if path.ends_with(".bundle") { + if path.contains("TableBundles/") || path.ends_with(".bytes") { + ResourceType::TableBundle + } else if path.ends_with(".bundle") { ResourceType::AssetBundle } else if path.contains("catalog") { ResourceType::Manifest @@ -52,6 +198,21 @@ impl AddressablesCatalogDriver { } } + fn resource_type_for_compact_entry( + path: &str, + resource_type_name: Option<&str>, + ) -> ResourceType { + if resource_type_name.is_some_and(|name| name.contains("IAssetBundleResource")) { + return ResourceType::AssetBundle; + } + + if resource_type_name.is_some_and(|name| name.contains("ContentCatalogData")) { + return ResourceType::Manifest; + } + + Self::resource_type_for_path(path) + } + fn resource_from_entry(value: &Value, index: usize) -> Option { let path = value .get("internal_id") @@ -64,9 +225,11 @@ impl AddressablesCatalogDriver { return None; } + let address = Self::string_field(value, &["address", "Address", "m_Address", "key", "Key"]); let hash = value .get("hash") .or_else(|| value.get("Hash")) + .or_else(|| value.get("m_Hash")) .and_then(|value| value.as_str()) .map(ToOwned::to_owned) .unwrap_or_else(|| format!("addressable_{}", index)); @@ -74,17 +237,85 @@ impl AddressablesCatalogDriver { let size = value .get("size") .or_else(|| value.get("Size")) + .or_else(|| value.get("m_Size")) .and_then(|value| value.as_u64()) .unwrap_or_default(); + let dependencies = Self::dependencies_from_entry(value); + Some(ResourceEntry { path: path.to_string(), hash, size, resource_type: Self::resource_type_for_path(path), + address, + dependencies, }) } + fn string_field(value: &Value, fields: &[&str]) -> Option { + fields.iter().find_map(|field| { + value + .get(field) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned) + }) + } + + fn string_array_field(value: &Value, fields: &[&str]) -> Vec { + for field in fields { + if let Some(array) = value.get(field).and_then(|value| value.as_array()) { + let strings = array + .iter() + .filter_map(|item| match item { + Value::String(value) => Some(value.clone()), + Value::Object(object) => object + .get("internal_id") + .or_else(|| object.get("InternalId")) + .or_else(|| object.get("path")) + .or_else(|| object.get("Path")) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned), + _ => None, + }) + .collect::>(); + if !strings.is_empty() { + return strings; + } + } + } + + Vec::new() + } + + fn dependencies_from_entry(value: &Value) -> Vec { + let dependencies = Self::string_array_field( + value, + &[ + "dependencies", + "Dependencies", + "m_Dependencies", + "dependency", + "Dependency", + "m_Dependency", + ], + ); + if !dependencies.is_empty() { + return dependencies; + } + + Self::string_array_field( + value, + &[ + "m_Dependencies", + "asset_dependencies", + "AssetDependencies", + "resource_dependencies", + "ResourceDependencies", + ], + ) + } + fn entry_resources(json: &Value) -> Vec { json.get("m_Entries") .or_else(|| json.get("entries")) @@ -113,17 +344,314 @@ impl AddressablesCatalogDriver { hash: format!("addressable_{}", index), size: 0, resource_type: Self::resource_type_for_path(path), + address: None, + dependencies: Vec::new(), }) }) .collect() } + fn key_data_resources(json: &Value) -> Vec { + let Some(bytes) = Self::blob_bytes(json, "m_KeyDataString") else { + return Vec::new(); + }; + + let expected_count = Self::blob_count(&bytes).unwrap_or_default(); + if expected_count == 0 { + return Vec::new(); + } + + let limit = Self::internal_id_count(json).unwrap_or(expected_count); + let mut resources = Vec::with_capacity(expected_count.min(limit)); + let mut offset = 4; + + while offset < bytes.len() && resources.len() < limit { + let Some((path, next_offset)) = Self::decode_serialized_string(&bytes, offset) else { + break; + }; + offset = next_offset; + if path.is_empty() { + continue; + } + + let index = resources.len(); + let resource_type = Self::resource_type_for_path(&path); + resources.push(ResourceEntry { + path, + hash: format!("addressable_{}", index), + size: 0, + resource_type, + address: None, + dependencies: Vec::new(), + }); + } + + resources + } + + fn compact_entry_resources(json: &Value) -> Vec { + let Some(internal_ids) = Self::string_array(json, "m_InternalIds") else { + return Vec::new(); + }; + let Some(provider_ids) = Self::string_array(json, "m_ProviderIds") else { + return Vec::new(); + }; + let Some(key_bytes) = Self::blob_bytes(json, "m_KeyDataString") else { + return Vec::new(); + }; + let Some(entry_records) = Self::compact_entry_records(json) else { + return Vec::new(); + }; + let Some(buckets) = Self::compact_buckets(json) else { + return Vec::new(); + }; + + let keys = Self::compact_keys(&key_bytes, &buckets); + if keys.is_empty() { + return Vec::new(); + } + + let internal_id_prefixes = Self::string_array(json, "m_InternalIdPrefixes") + .unwrap_or_default() + .into_iter() + .collect::>(); + let extra_data = Self::blob_bytes(json, "m_ExtraDataString").unwrap_or_default(); + + entry_records + .iter() + .enumerate() + .filter_map(|(index, record)| { + let internal_id = internal_ids.get(record.internal_id as usize)?; + provider_ids.get(record.provider_index as usize)?; + let primary_key = keys + .get(record.primary_key_index as usize) + .and_then(|key| key.as_ref()) + .and_then(AddressablesObject::key_string) + .unwrap_or_else(|| format!("addressable_{}", index)); + let path = Self::normalize_internal_id(&internal_id_prefixes, internal_id); + let extra = Self::extra_data_at(&extra_data, record.data_index); + let resource_type_name = Self::resource_type_name(json, record.resource_type_index); + let dependencies = + Self::compact_dependencies(record, &entry_records, &buckets, &keys); + let hash = extra + .hash + .filter(|value| !value.is_empty()) + .or_else(|| extra.bundle_name.filter(|value| !value.is_empty())) + .unwrap_or_else(|| format!("addressable_{}", index)); + + Some(ResourceEntry { + path: path.clone(), + hash, + size: extra.bundle_size.unwrap_or_default(), + resource_type: Self::resource_type_for_compact_entry( + &path, + resource_type_name.as_deref(), + ), + address: if primary_key.is_empty() { + None + } else { + Some(primary_key) + }, + dependencies, + }) + }) + .collect() + } + + fn string_array(json: &Value, field: &str) -> Option> { + json.get(field)? + .as_array()? + .iter() + .map(|value| value.as_str().map(ToOwned::to_owned)) + .collect() + } + + fn compact_entry_records(json: &Value) -> Option> { + let bytes = Self::blob_bytes(json, "m_EntryDataString")?; + let count = Self::blob_count(&bytes)?; + let expected_len = 4usize.checked_add( + count + .checked_mul(ADDRESSABLES_ENTRY_RECORD_INTS)? + .checked_mul(4)?, + )?; + if bytes.len() < expected_len { + return None; + } + + let mut records = Vec::with_capacity(count); + let mut offset = 4; + for _ in 0..count { + records.push(CompactEntryRecord { + internal_id: Self::read_i32_le(&bytes, offset)?, + provider_index: Self::read_i32_le(&bytes, offset + 4)?, + dependency_key_index: Self::read_i32_le(&bytes, offset + 8)?, + data_index: Self::read_i32_le(&bytes, offset + 16)?, + primary_key_index: Self::read_i32_le(&bytes, offset + 20)?, + resource_type_index: Self::read_i32_le(&bytes, offset + 24)?, + }); + offset += ADDRESSABLES_ENTRY_RECORD_INTS * 4; + } + + Some(records) + } + + fn compact_buckets(json: &Value) -> Option> { + let bytes = Self::blob_bytes(json, "m_BucketDataString")?; + let count = Self::blob_count(&bytes)?; + let mut buckets = Vec::with_capacity(count); + let mut offset = 4; + + for _ in 0..count { + let key_offset = Self::read_i32_le(&bytes, offset)?; + let entry_count = Self::read_i32_le(&bytes, offset + 4)?; + if key_offset < 0 || entry_count < 0 { + return None; + } + offset += 8; + + let mut entries = Vec::with_capacity(entry_count as usize); + for _ in 0..entry_count { + let entry_index = Self::read_i32_le(&bytes, offset)?; + if entry_index < 0 { + return None; + } + entries.push(entry_index as usize); + offset += 4; + } + + buckets.push(CompactBucket { + key_offset: key_offset as usize, + entries, + }); + } + + Some(buckets) + } + + fn compact_keys( + key_bytes: &[u8], + buckets: &[CompactBucket], + ) -> Vec> { + buckets + .iter() + .map(|bucket| { + Self::read_serialized_object(key_bytes, bucket.key_offset).map(|(object, _)| object) + }) + .collect() + } + + fn compact_dependencies( + record: &CompactEntryRecord, + records: &[CompactEntryRecord], + buckets: &[CompactBucket], + keys: &[Option], + ) -> Vec { + if record.dependency_key_index < 0 { + return Vec::new(); + } + + buckets + .get(record.dependency_key_index as usize) + .into_iter() + .flat_map(|bucket| bucket.entries.iter().copied()) + .filter_map(|entry_index| records.get(entry_index)) + .filter_map(|dependency_record| keys.get(dependency_record.primary_key_index as usize)) + .filter_map(|key| key.as_ref()) + .filter_map(AddressablesObject::key_string) + .collect() + } + + fn extra_data_at(extra_data: &[u8], data_index: i32) -> AddressablesExtraData { + if data_index < 0 { + return AddressablesExtraData::default(); + } + + let Some((object, _)) = Self::read_serialized_object(extra_data, data_index as usize) + else { + return AddressablesExtraData::default(); + }; + + let AddressablesObject::JsonObject { json, .. } = object else { + return AddressablesExtraData::default(); + }; + + let Some(json) = json else { + return AddressablesExtraData::default(); + }; + + AddressablesExtraData { + hash: json + .get("m_Hash") + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned), + bundle_name: json + .get("m_BundleName") + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned), + bundle_size: json.get("m_BundleSize").and_then(|value| value.as_u64()), + } + } + + fn resource_type_name(json: &Value, index: i32) -> Option { + if index < 0 { + return None; + } + + json.get("m_resourceTypes")? + .as_array()? + .get(index as usize)? + .get("m_ClassName")? + .as_str() + .map(ToOwned::to_owned) + } + + fn normalize_internal_id(prefixes: &[String], internal_id: &str) -> String { + let expanded = Self::expand_internal_id(prefixes, internal_id); + let normalized = expanded.replace('\\', "/"); + normalized + .strip_prefix(PLATFORM_LOAD_PATH_PLACEHOLDER) + .map(|path| path.trim_start_matches('/').to_string()) + .unwrap_or(normalized) + } + + fn expand_internal_id(prefixes: &[String], internal_id: &str) -> String { + if prefixes.is_empty() { + return internal_id.to_string(); + } + + let Some((index, path)) = internal_id.rsplit_once('#') else { + return internal_id.to_string(); + }; + let Ok(index) = index.parse::() else { + return internal_id.to_string(); + }; + let Some(prefix) = prefixes.get(index) else { + return internal_id.to_string(); + }; + + format!("{prefix}{path}") + } + fn resources(json: &Value) -> Vec { let entry_resources = Self::entry_resources(json); if !entry_resources.is_empty() { return entry_resources; } + let compact_resources = Self::compact_entry_resources(json); + if !compact_resources.is_empty() { + return compact_resources; + } + + let key_resources = Self::key_data_resources(json); + if !key_resources.is_empty() + && Self::internal_id_count(json) + .map(|count| key_resources.len() >= count) + .unwrap_or(true) + { + return key_resources; + } + Self::internal_id_resources(json) } @@ -131,13 +659,35 @@ impl AddressablesCatalogDriver { let mut extra = HashMap::new(); Self::insert_array_len(&mut extra, json, "m_InternalIds", "internal_id_count"); Self::insert_array_len(&mut extra, json, "m_Entries", "entry_count"); + Self::insert_array_len(&mut extra, json, "m_resourceTypes", "resource_type_count"); + Self::insert_blob_count(&mut extra, json, "m_KeyDataString", "key_object_count"); + Self::insert_blob_count( + &mut extra, + json, + "m_BucketDataString", + "bucket_record_count", + ); + Self::insert_blob_count(&mut extra, json, "m_EntryDataString", "entry_record_count"); Self::insert_string_len(&mut extra, json, "m_KeyDataString", "key_data_string_len"); + Self::insert_string_len( + &mut extra, + json, + "m_BucketDataString", + "bucket_data_string_len", + ); Self::insert_string_len( &mut extra, json, "m_EntryDataString", "entry_data_string_len", ); + Self::insert_string_len( + &mut extra, + json, + "m_ExtraDataString", + "extra_data_string_len", + ); + Self::insert_dependency_count(&mut extra, json); extra } @@ -165,6 +715,82 @@ impl AddressablesCatalogDriver { extra.insert(key.to_string(), len.to_string()); } } + + fn insert_blob_count( + extra: &mut HashMap, + json: &Value, + field: &str, + key: &str, + ) { + if let Some(count) = Self::blob_count_field(json, field) { + extra.insert(key.to_string(), count.to_string()); + } + } + + fn insert_dependency_count(extra: &mut HashMap, json: &Value) { + let count = Self::entry_resources(json) + .into_iter() + .map(|entry| entry.dependencies.len()) + .sum::(); + if count > 0 { + extra.insert("dependency_count".to_string(), count.to_string()); + } + } +} + +#[derive(Debug, Clone)] +enum AddressablesObject { + String(String), + Unsigned(u32), + Signed(i32), + Hash128(String), + JsonObject { + assembly_name: String, + class_name: String, + json: Option, + }, +} + +impl AddressablesObject { + fn key_string(&self) -> Option { + match self { + Self::String(value) | Self::Hash128(value) => Some(value.clone()), + Self::Unsigned(value) => Some(value.to_string()), + Self::Signed(value) => Some(value.to_string()), + Self::JsonObject { + assembly_name, + class_name, + .. + } => Some(format!("{assembly_name}:{class_name}")), + } + } + + fn into_key_string(self) -> Option { + self.key_string() + } +} + +#[derive(Debug, Clone)] +struct CompactEntryRecord { + internal_id: i32, + provider_index: i32, + dependency_key_index: i32, + data_index: i32, + primary_key_index: i32, + resource_type_index: i32, +} + +#[derive(Debug, Clone)] +struct CompactBucket { + key_offset: usize, + entries: Vec, +} + +#[derive(Debug, Clone, Default)] +struct AddressablesExtraData { + hash: Option, + bundle_name: Option, + bundle_size: Option, } impl Default for AddressablesCatalogDriver { @@ -292,11 +918,14 @@ mod tests { "m_InternalIds": ["synthetic/fallback.bundle"], "m_KeyDataString": "key-data", "m_EntryDataString": "entry-data", + "m_resourceTypes": ["AssetBundle", "Manifest"], "m_Entries": [ { "internal_id": "synthetic/minimal.bundle", "hash": "synthetic-entry-hash", - "size": 119 + "size": 119, + "address": "Character_001", + "dependencies": ["synthetic/shared.bundle"] }, { "internal_id": "synthetic/catalog.json", @@ -312,6 +941,14 @@ mod tests { assert_eq!(manifest.resources[0].path, "synthetic/minimal.bundle"); assert_eq!(manifest.resources[0].hash, "synthetic-entry-hash"); assert_eq!(manifest.resources[0].size, 119); + assert_eq!( + manifest.resources[0].address.as_deref(), + Some("Character_001") + ); + assert_eq!( + manifest.resources[0].dependencies, + vec!["synthetic/shared.bundle".to_string()] + ); assert_eq!( manifest.resources[0].resource_type, ResourceType::AssetBundle @@ -337,5 +974,47 @@ mod tests { manifest.metadata.extra.get("entry_data_string_len"), Some(&"10".to_string()) ); + assert_eq!( + manifest.metadata.extra.get("resource_type_count"), + Some(&"2".to_string()) + ); + assert_eq!( + manifest.metadata.extra.get("dependency_count"), + Some(&"1".to_string()) + ); + } + + #[tokio::test] + async fn test_parse_reports_json_location() { + let driver = AddressablesCatalogDriver::new(); + + let error = driver + .parse( + b"{\n \"m_LocatorId\": \"AddressablesMainContentCatalog\",\n \"m_Entries\": [\n", + ) + .await + .expect_err("expected parse failure"); + + assert!(error.contains("Invalid JSON at line")); + } + + #[tokio::test] + async fn test_parse_table_bundle_resource_types() { + let driver = AddressablesCatalogDriver::new(); + + let catalog_json = r#"{ + "m_LocatorId": "AddressablesMainContentCatalog", + "m_InternalIds": [ + "TableBundles/10031865119468584059_717066257.bytes" + ] + }"#; + + let manifest = driver.parse(catalog_json.as_bytes()).await.unwrap(); + + assert_eq!(manifest.resources.len(), 1); + assert_eq!( + manifest.resources[0].resource_type, + ResourceType::TableBundle + ); } } diff --git a/adapters/src/official/game_main_config.rs b/adapters/src/official/game_main_config.rs index 18e97d6..8d2e501 100644 --- a/adapters/src/official/game_main_config.rs +++ b/adapters/src/official/game_main_config.rs @@ -42,11 +42,9 @@ impl YostarJpGameMainConfig { } fn from_serialized_file(serialized: &UnitySerializedFile) -> Result { - let asset = serialized - .text_asset("GameMainConfig") - .ok_or_else(|| { - "GameMainConfig text asset not found in Unity serialized file".to_string() - })?; + let asset = serialized.text_asset("GameMainConfig").ok_or_else(|| { + "GameMainConfig text asset not found in Unity serialized file".to_string() + })?; Self::from_encrypted_bytes(&asset.bytes) } diff --git a/adapters/src/official/mod.rs b/adapters/src/official/mod.rs index 003ceee..3c4aa42 100644 --- a/adapters/src/official/mod.rs +++ b/adapters/src/official/mod.rs @@ -4,15 +4,15 @@ //! client endpoints. Mirror-specific layers such as `bluearchive.cafe` or //! `text=jp/voice=jp/media=jp` are intentionally excluded. -pub mod inventory; pub mod game_main_config; +pub mod inventory; pub mod launcher; pub mod yostar_jp; +pub use game_main_config::YostarJpGameMainConfig; pub use inventory::{ YostarJpDownloadInventory, YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, }; -pub use game_main_config::YostarJpGameMainConfig; pub use launcher::{ YostarJpInstalledGameBootstrap, YostarJpLauncherConfig, YostarJpLauncherManifest, }; diff --git a/adapters/src/official/yostar_jp.rs b/adapters/src/official/yostar_jp.rs index 03c08f8..1f5011e 100644 --- a/adapters/src/official/yostar_jp.rs +++ b/adapters/src/official/yostar_jp.rs @@ -8,7 +8,7 @@ //! naked `.bundle` names present in `catalog_Remote.json` //! - Android uses its own patch-pack directory -use serde::Deserialize; +use serde::{Deserialize, Serialize}; /// Official JP server-info host. pub const SERVER_INFO_HOST: &str = "yostar-serverinfo.bluearchiveyostar.com"; @@ -230,7 +230,7 @@ impl YostarJpConnectionGroup { } /// Patch platform selected by the game client. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum PatchPlatform { /// Standalone Windows client. Windows, @@ -275,7 +275,7 @@ pub fn verified_official_platforms() -> Vec { } /// Kind of official JP resource endpoint discovered from a resource root. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum YostarJpResourceEndpointKind { /// `TableBundles/TableCatalog.bytes`. TableCatalog, @@ -296,7 +296,7 @@ pub enum YostarJpResourceEndpointKind { } /// One official JP resource URL seed. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct YostarJpResourceEndpoint { /// Endpoint kind. pub kind: YostarJpResourceEndpointKind, @@ -307,7 +307,7 @@ pub struct YostarJpResourceEndpoint { } /// Official JP resource discovery seed plan. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct YostarJpResourceDiscoveryPlan { /// Selected server-info connection group name. pub connection_group_name: String, @@ -322,7 +322,7 @@ pub struct YostarJpResourceDiscoveryPlan { } /// Snapshot of the official JP resource state observed at a point in time. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct YostarJpSyncSnapshot { /// Selected server-info connection group name. pub connection_group_name: String, @@ -368,7 +368,7 @@ impl YostarJpSyncSnapshot { } /// Delta between an official snapshot and a previously stored snapshot. -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub struct YostarJpSyncDelta { /// Whether this is the first observation. pub is_initial: bool, @@ -410,7 +410,7 @@ impl YostarJpSyncDelta { } /// Individual endpoint difference. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum YostarJpResourceEndpointChange { /// Endpoint newly observed. Added(YostarJpResourceEndpoint), diff --git a/adapters/src/unity/serialized_file.rs b/adapters/src/unity/serialized_file.rs index 6f743cf..4546163 100644 --- a/adapters/src/unity/serialized_file.rs +++ b/adapters/src/unity/serialized_file.rs @@ -218,7 +218,10 @@ fn read_serialized_type( if version >= 12 || version == 10 { let node_count = reader.read_i32("type_tree_node_count")?; if node_count < 0 { - return Err(format!("Invalid Unity type tree node count: {}", node_count)); + return Err(format!( + "Invalid Unity type tree node count: {}", + node_count + )); } let string_buffer_size = reader.read_i32("type_tree_string_buffer_size")?; if string_buffer_size < 0 { @@ -229,10 +232,7 @@ fn read_serialized_type( } let node_size = 2 + 1 + 1 + 4 + 4 + 4 + 4 + 4 + if version >= 19 { 8 } else { 0 }; - reader.read_bytes( - node_count as usize * node_size, - "type_tree_nodes", - )?; + reader.read_bytes(node_count as usize * node_size, "type_tree_nodes")?; reader.read_bytes(string_buffer_size as usize, "type_tree_strings")?; } @@ -244,10 +244,7 @@ fn read_serialized_type( dependency_count )); } - reader.read_bytes( - dependency_count as usize * 4, - "type_tree_dependencies", - )?; + reader.read_bytes(dependency_count as usize * 4, "type_tree_dependencies")?; } } @@ -523,9 +520,9 @@ mod tests { assert_eq!( &asset.bytes[..32], &[ - 0x60, 0x23, 0x0a, 0x06, 0x95, 0x72, 0x83, 0x57, 0x54, 0x23, 0x49, 0x06, - 0xfc, 0x72, 0xb4, 0x57, 0x57, 0x23, 0x6b, 0x06, 0x98, 0x72, 0xb5, 0x57, - 0x71, 0x23, 0x1b, 0x06, 0xec, 0x72, 0xf6, 0x57, + 0x60, 0x23, 0x0a, 0x06, 0x95, 0x72, 0x83, 0x57, 0x54, 0x23, 0x49, 0x06, 0xfc, 0x72, + 0xb4, 0x57, 0x57, 0x23, 0x6b, 0x06, 0x98, 0x72, 0xb5, 0x57, 0x71, 0x23, 0x1b, 0x06, + 0xec, 0x72, 0xf6, 0x57, ], ); } diff --git a/adapters/tests/fixtures/real_addressables/catalog.json b/adapters/tests/fixtures/real_addressables/catalog.json new file mode 100644 index 0000000..30d8ab6 --- /dev/null +++ b/adapters/tests/fixtures/real_addressables/catalog.json @@ -0,0 +1,22 @@ +{ + "m_LocatorId": "AddressablesMainContentCatalog", + "m_InternalIdPrefixes": [], + "m_ProviderIds": [ + "UnityEngine.ResourceManagement.ResourceProviders.AssetBundleProvider" + ], + "m_InternalIds": [ + "{PlatformUtils.AddressableLoadPath}\\academy-_mxload-prefabs-2025-07-02_assets_all_638981069.bundle", + "{PlatformUtils.AddressableLoadPath}\\academy-_mxload-prefabs-2025-08-26_assets_all_1581352935.bundle", + "{PlatformUtils.AddressableLoadPath}\\shared_assets_all_123.bundle" + ], + "m_resourceTypes": [ + { + "m_AssemblyName": "Unity.ResourceManager, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", + "m_ClassName": "UnityEngine.ResourceManagement.ResourceProviders.IAssetBundleResource" + } + ], + "m_KeyDataString": "BAAAAAA+AAAAYWNhZGVteS1fbXhsb2FkLXByZWZhYnMtMjAyNS0wNy0wMl9hc3NldHNfYWxsXzYzODk4MTA2OS5idW5kbGUAPwAAAGFjYWRlbXktX214bG9hZC1wcmVmYWJzLTIwMjUtMDgtMjZfYXNzZXRzX2FsbF8xNTgxMzUyOTM1LmJ1bmRsZQAcAAAAc2hhcmVkX2Fzc2V0c19hbGxfMTIzLmJ1bmRsZQAQAAAAZGVwZW5kZW5jeS1zZXQtMA==", + "m_BucketDataString": "BAAAAAQAAAABAAAAAAAAAEcAAAABAAAAAQAAAIsAAAABAAAAAgAAAKwAAAABAAAAAgAAAA==", + "m_EntryDataString": "AwAAAAAAAAAAAAAAAwAAAHsAAAAAAAAAAAAAAAAAAAABAAAAAAAAAP////8AAAAARQEAAAEAAAAAAAAAAgAAAAAAAAD/////AAAAAJACAAACAAAAAAAAAA==", + "m_ExtraDataString": "B0xVbml0eS5SZXNvdXJjZU1hbmFnZXIsIFZlcnNpb249MC4wLjAuMCwgQ3VsdHVyZT1uZXV0cmFsLCBQdWJsaWNLZXlUb2tlbj1udWxsSlVuaXR5RW5naW5lLlJlc291cmNlTWFuYWdlbWVudC5SZXNvdXJjZVByb3ZpZGVycy5Bc3NldEJ1bmRsZVJlcXVlc3RPcHRpb25zqAAAAHsAIgBtAF8ASABhAHMAaAAiADoAIgBoAGEAcwBoAC0AbQBhAGkAbgAiACwAIgBtAF8AQwByAGMAIgA6ADAALAAiAG0AXwBCAHUAbgBkAGwAZQBOAGEAbQBlACIAOgAiAGIAdQBuAGQAbABlAC0AbQBhAGkAbgAiACwAIgBtAF8AQgB1AG4AZABsAGUAUwBpAHoAZQAiADoAMQA5ADMAMgA4ADcAMgB9AAdMVW5pdHkuUmVzb3VyY2VNYW5hZ2VyLCBWZXJzaW9uPTAuMC4wLjAsIEN1bHR1cmU9bmV1dHJhbCwgUHVibGljS2V5VG9rZW49bnVsbEpVbml0eUVuZ2luZS5SZXNvdXJjZU1hbmFnZW1lbnQuUmVzb3VyY2VQcm92aWRlcnMuQXNzZXRCdW5kbGVSZXF1ZXN0T3B0aW9uc64AAAB7ACIAbQBfAEgAYQBzAGgAIgA6ACIAaABhAHMAaAAtAHMAZQBjAG8AbgBkACIALAAiAG0AXwBDAHIAYwAiADoAMAAsACIAbQBfAEIAdQBuAGQAbABlAE4AYQBtAGUAIgA6ACIAYgB1AG4AZABsAGUALQBzAGUAYwBvAG4AZAAiACwAIgBtAF8AQgB1AG4AZABsAGUAUwBpAHoAZQAiADoAMQA2ADIAMQAzADQAfQAHTFVuaXR5LlJlc291cmNlTWFuYWdlciwgVmVyc2lvbj0wLjAuMC4wLCBDdWx0dXJlPW5ldXRyYWwsIFB1YmxpY0tleVRva2VuPW51bGxKVW5pdHlFbmdpbmUuUmVzb3VyY2VNYW5hZ2VtZW50LlJlc291cmNlUHJvdmlkZXJzLkFzc2V0QnVuZGxlUmVxdWVzdE9wdGlvbnOuAAAAewAiAG0AXwBIAGEAcwBoACIAOgAiAGgAYQBzAGgALQBzAGgAYQByAGUAZAAiACwAIgBtAF8AQwByAGMAIgA6ADAALAAiAG0AXwBCAHUAbgBkAGwAZQBOAGEAbQBlACIAOgAiAGIAdQBuAGQAbABlAC0AcwBoAGEAcgBlAGQAIgAsACIAbQBfAEIAdQBuAGQAbABlAFMAaQB6AGUAIgA6ADEANQAzADQAOAAwAH0A" +} diff --git a/adapters/tests/golden/real_addressables.json b/adapters/tests/golden/real_addressables.json new file mode 100644 index 0000000..0d236e2 --- /dev/null +++ b/adapters/tests/golden/real_addressables.json @@ -0,0 +1,45 @@ +{ + "format": "AddressablesCatalog", + "locator_id": "AddressablesMainContentCatalog", + "cdn_prefixes": [], + "resource_count": 3, + "resources": [ + { + "path": "academy-_mxload-prefabs-2025-07-02_assets_all_638981069.bundle", + "hash": "hash-main", + "size": 1932872, + "resource_type": "AssetBundle", + "address": "academy-_mxload-prefabs-2025-07-02_assets_all_638981069.bundle", + "dependencies": [ + "shared_assets_all_123.bundle" + ] + }, + { + "path": "academy-_mxload-prefabs-2025-08-26_assets_all_1581352935.bundle", + "hash": "hash-second", + "size": 162134, + "resource_type": "AssetBundle", + "address": "academy-_mxload-prefabs-2025-08-26_assets_all_1581352935.bundle", + "dependencies": [] + }, + { + "path": "shared_assets_all_123.bundle", + "hash": "hash-shared", + "size": 153480, + "resource_type": "AssetBundle", + "address": "shared_assets_all_123.bundle", + "dependencies": [] + } + ], + "metadata": { + "internal_id_count": "3", + "resource_type_count": "1", + "key_object_count": "4", + "bucket_record_count": "4", + "entry_record_count": "3", + "key_data_string_len": "260", + "bucket_data_string_len": "72", + "entry_data_string_len": "120", + "extra_data_string_len": "1316" + } +} diff --git a/adapters/tests/real_addressables_golden.rs b/adapters/tests/real_addressables_golden.rs new file mode 100644 index 0000000..1824e87 --- /dev/null +++ b/adapters/tests/real_addressables_golden.rs @@ -0,0 +1,34 @@ +use bat_adapters::manifest::ManifestDriverRegistry; +use serde_json::json; + +#[tokio::test] +async fn parses_real_shape_addressables_catalog_against_golden() { + let catalog = include_str!("fixtures/real_addressables/catalog.json"); + let manifest = ManifestDriverRegistry::with_defaults() + .parse(catalog.as_bytes()) + .await + .expect("parse real-shape Addressables catalog"); + + let actual = json!({ + "format": format!("{:?}", manifest.format), + "locator_id": manifest.metadata.locator_id, + "cdn_prefixes": manifest.metadata.cdn_prefixes, + "resource_count": manifest.resources.len(), + "resources": manifest.resources.iter().map(|resource| { + json!({ + "path": resource.path, + "hash": resource.hash, + "size": resource.size, + "resource_type": format!("{:?}", resource.resource_type), + "address": resource.address, + "dependencies": resource.dependencies, + }) + }).collect::>(), + "metadata": manifest.metadata.extra, + }); + + let expected: serde_json::Value = + serde_json::from_str(include_str!("golden/real_addressables.json")).unwrap(); + + assert_eq!(actual, expected); +} diff --git a/adapters/tests/real_addressables_local.rs b/adapters/tests/real_addressables_local.rs index f20a22f..661e3a5 100644 --- a/adapters/tests/real_addressables_local.rs +++ b/adapters/tests/real_addressables_local.rs @@ -18,4 +18,7 @@ async fn parses_local_real_addressables_catalog() { ); assert!(manifest.resources.len() > 1_000); assert!(manifest.metadata.extra.contains_key("internal_id_count")); + assert!(manifest.metadata.extra.contains_key("key_object_count")); + assert!(manifest.metadata.extra.contains_key("bucket_record_count")); + assert!(manifest.metadata.extra.contains_key("entry_record_count")); } diff --git a/core/src/domain/resource.rs b/core/src/domain/resource.rs index 7a2035e..def7584 100644 --- a/core/src/domain/resource.rs +++ b/core/src/domain/resource.rs @@ -26,6 +26,10 @@ pub struct ResourceEntry { pub size: u64, /// 资源类型 pub resource_type: ResourceType, + /// 资源在 Manifest 中的逻辑地址 + pub address: Option, + /// 该资源依赖的其他资源标识 + pub dependencies: Vec, } /// 资源 @@ -50,6 +54,8 @@ mod tests { hash: "abc123".to_string(), size: 1024, resource_type: ResourceType::AssetBundle, + address: None, + dependencies: Vec::new(), }; assert_eq!(entry.path, "test.bundle"); diff --git a/crates/bat-ffi/Cargo.toml b/crates/bat-ffi/Cargo.toml index b5a46ec..ecfaf3f 100644 --- a/crates/bat-ffi/Cargo.toml +++ b/crates/bat-ffi/Cargo.toml @@ -9,6 +9,9 @@ license.workspace = true crate-type = ["cdylib", "staticlib"] [dependencies] +bat-core = { path = "../../core" } +bat-adapters = { path = "../../adapters" } +bat-infrastructure = { path = "../../infrastructure" } bat-cas-engine = { path = "../bat-cas-engine" } bat-assetbundle = { path = "../bat-assetbundle" } bat-patch = { path = "../bat-patch" } @@ -16,6 +19,7 @@ bat-patch = { path = "../bat-patch" } anyhow.workspace = true serde.workspace = true serde_json.workspace = true +tokio.workspace = true # FFI 绑定 libc = "0.2" diff --git a/crates/bat-ffi/src/lib.rs b/crates/bat-ffi/src/lib.rs index 9089fb1..0c1b72a 100644 --- a/crates/bat-ffi/src/lib.rs +++ b/crates/bat-ffi/src/lib.rs @@ -6,23 +6,122 @@ #![warn(clippy::all)] -use std::ffi::CString; +use bat_adapters::manifest::{GenericManifest, ManifestDriverRegistry}; +use bat_adapters::official::yostar_jp::{ + PatchPlatform, YostarJpResourceEndpoint, YostarJpResourceEndpointChange, + YostarJpResourceEndpointKind, YostarJpSyncSnapshot, +}; +use bat_infrastructure::official_sync::{build_official_sync_plan, OfficialSyncDecision}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::ffi::{CStr, CString}; use std::os::raw::c_char; /// FFI 版本号 pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +/// Sync snapshot JSON 输入。 +#[derive(Debug, Deserialize)] +struct SyncSnapshotInput { + connection_group_name: String, + app_version: String, + bundle_version: Option, + addressables_root: String, + endpoints: Vec, +} + +/// Sync endpoint JSON 输入。 +#[derive(Debug, Deserialize)] +struct SyncEndpointInput { + kind: String, + platform: Option, + url: String, +} + +#[derive(Debug, Serialize)] +struct FfiResult { + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + data: Option, +} + +#[derive(Debug, Serialize)] +struct ManifestResourceView { + path: String, + hash: String, + size: u64, + resource_type: String, + address: Option, + dependencies: Vec, +} + +#[derive(Debug, Serialize)] +struct ManifestInspectView { + format: String, + locator_id: Option, + cdn_prefixes: Vec, + resource_count: usize, + resources: Vec, + metadata: HashMap, +} + +#[derive(Debug, Serialize)] +struct SyncEndpointView { + kind: String, + platform: Option, + url: String, +} + +#[derive(Debug, Serialize)] +struct SyncEndpointChangeView { + change_type: String, + previous: Option, + current: Option, +} + +#[derive(Debug, Serialize)] +struct SyncSnapshotView { + connection_group_name: String, + app_version: String, + bundle_version: Option, + addressables_root: String, + endpoints: Vec, +} + +#[derive(Debug, Serialize)] +struct SyncDeltaView { + is_initial: bool, + connection_group_changed: bool, + app_version_changed: bool, + bundle_version_changed: bool, + addressables_root_changed: bool, + root_token_changed: bool, + endpoint_changes: Vec, + needs_download: bool, +} + +#[derive(Debug, Serialize)] +struct SyncPlanView { + decision: String, + should_download: bool, + should_publish: bool, + snapshot: SyncSnapshotView, + delta: SyncDeltaView, +} + /// 获取版本号（C ABI） #[no_mangle] pub extern "C" fn bat_version() -> *const c_char { - CString::new(VERSION).unwrap().into_raw() + c_string(VERSION).into_raw() } /// 释放 C 字符串内存 /// /// # Safety /// 调用者必须确保： -/// - `s` 是通过 `bat_version` 返回的有效指针 +/// - `s` 是通过 `bat_version` / `bat_manifest_inspect_json` / `bat_sync_plan_json` 返回的有效指针 /// - `s` 只被释放一次 #[no_mangle] pub unsafe extern "C" fn bat_free_string(s: *mut c_char) { @@ -31,20 +130,295 @@ pub unsafe extern "C" fn bat_free_string(s: *mut c_char) { } } +/// 解析 Addressables Manifest，并返回 JSON summary。 +#[no_mangle] +pub extern "C" fn bat_manifest_inspect_json(raw_json: *const c_char) -> *mut c_char { + let result = (|| -> Result { + let raw_json = c_str_to_string(raw_json)?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())?; + + let manifest: GenericManifest = runtime.block_on(async { + ManifestDriverRegistry::with_defaults() + .parse(raw_json.as_bytes()) + .await + })?; + + Ok(ManifestInspectView { + format: format!("{:?}", manifest.format), + locator_id: manifest.metadata.locator_id, + cdn_prefixes: manifest.metadata.cdn_prefixes, + resource_count: manifest.resources.len(), + resources: manifest + .resources + .into_iter() + .map(|resource| ManifestResourceView { + path: resource.path, + hash: resource.hash, + size: resource.size, + resource_type: format!("{:?}", resource.resource_type), + address: resource.address, + dependencies: resource.dependencies, + }) + .collect(), + metadata: manifest.metadata.extra, + }) + })(); + + json_result(result) +} + +/// 构建官方同步计划 JSON summary。 +#[no_mangle] +pub extern "C" fn bat_sync_plan_json( + current_json: *const c_char, + previous_json: *const c_char, +) -> *mut c_char { + let result = (|| -> Result { + let current = parse_sync_snapshot(current_json)?; + let previous = if previous_json.is_null() { + None + } else { + Some(parse_sync_snapshot(previous_json)?) + }; + + let plan = build_official_sync_plan(current.clone(), previous.as_ref()); + let snapshot = snapshot_view(&plan.snapshot); + let delta = delta_view(&plan.delta); + + Ok(SyncPlanView { + decision: sync_decision_to_string(plan.decision), + should_download: plan.should_download(), + should_publish: plan.should_publish(), + snapshot, + delta, + }) + })(); + + json_result(result) +} + +fn parse_sync_snapshot(raw_json: *const c_char) -> Result { + let raw_json = c_str_to_string(raw_json)?; + let input: SyncSnapshotInput = serde_json::from_str(&raw_json) + .map_err(|error| format!("Invalid sync snapshot JSON: {error}"))?; + + Ok(YostarJpSyncSnapshot { + connection_group_name: input.connection_group_name, + app_version: input.app_version, + bundle_version: input.bundle_version, + addressables_root: input.addressables_root, + endpoints: input + .endpoints + .into_iter() + .map(endpoint_from_input) + .collect::, _>>()?, + }) +} + +fn endpoint_from_input(input: SyncEndpointInput) -> Result { + Ok(YostarJpResourceEndpoint { + kind: parse_endpoint_kind(&input.kind)?, + platform: input + .platform + .map(|value| parse_platform(&value)) + .transpose()?, + url: input.url, + }) +} + +fn parse_endpoint_kind(value: &str) -> Result { + match value { + "TableCatalog" => Ok(YostarJpResourceEndpointKind::TableCatalog), + "TableCatalogHash" => Ok(YostarJpResourceEndpointKind::TableCatalogHash), + "AddressablesCatalog" => Ok(YostarJpResourceEndpointKind::AddressablesCatalog), + "AddressablesCatalogHash" => Ok(YostarJpResourceEndpointKind::AddressablesCatalogHash), + "BundlePackingInfo" => Ok(YostarJpResourceEndpointKind::BundlePackingInfo), + "BundlePackingInfoHash" => Ok(YostarJpResourceEndpointKind::BundlePackingInfoHash), + "MediaCatalog" => Ok(YostarJpResourceEndpointKind::MediaCatalog), + "MediaCatalogHash" => Ok(YostarJpResourceEndpointKind::MediaCatalogHash), + other => Err(format!("Unknown sync endpoint kind: {other}")), + } +} + +fn parse_platform(value: &str) -> Result { + match value { + "Windows" => Ok(PatchPlatform::Windows), + "Android" => Ok(PatchPlatform::Android), + other => Err(format!("Unknown patch platform: {other}")), + } +} + +fn snapshot_view(snapshot: &YostarJpSyncSnapshot) -> SyncSnapshotView { + SyncSnapshotView { + connection_group_name: snapshot.connection_group_name.clone(), + app_version: snapshot.app_version.clone(), + bundle_version: snapshot.bundle_version.clone(), + addressables_root: snapshot.addressables_root.clone(), + endpoints: snapshot.endpoints.iter().map(endpoint_view).collect(), + } +} + +fn endpoint_view(endpoint: &YostarJpResourceEndpoint) -> SyncEndpointView { + SyncEndpointView { + kind: format!("{:?}", endpoint.kind), + platform: endpoint + .platform + .map(|platform| platform.as_str().to_string()), + url: endpoint.url.clone(), + } +} + +fn delta_view(delta: &bat_adapters::official::yostar_jp::YostarJpSyncDelta) -> SyncDeltaView { + SyncDeltaView { + is_initial: delta.is_initial, + connection_group_changed: delta.connection_group_changed, + app_version_changed: delta.app_version_changed, + bundle_version_changed: delta.bundle_version_changed, + addressables_root_changed: delta.addressables_root_changed, + root_token_changed: delta.root_token_changed, + endpoint_changes: delta + .endpoint_changes + .iter() + .map(endpoint_change_view) + .collect(), + needs_download: delta.needs_download, + } +} + +fn endpoint_change_view(change: &YostarJpResourceEndpointChange) -> SyncEndpointChangeView { + match change { + YostarJpResourceEndpointChange::Added(endpoint) => SyncEndpointChangeView { + change_type: "Added".to_string(), + previous: None, + current: Some(endpoint_view(endpoint)), + }, + YostarJpResourceEndpointChange::Removed(endpoint) => SyncEndpointChangeView { + change_type: "Removed".to_string(), + previous: Some(endpoint_view(endpoint)), + current: None, + }, + YostarJpResourceEndpointChange::Updated { previous, current } => SyncEndpointChangeView { + change_type: "Updated".to_string(), + previous: Some(endpoint_view(previous)), + current: Some(endpoint_view(current)), + }, + } +} + +fn sync_decision_to_string(decision: OfficialSyncDecision) -> String { + match decision { + OfficialSyncDecision::UpToDate => "UpToDate".to_string(), + OfficialSyncDecision::DownloadAndVerify => "DownloadAndVerify".to_string(), + OfficialSyncDecision::DownloadVerifyAndPublish => "DownloadVerifyAndPublish".to_string(), + } +} + +fn json_result(result: Result) -> *mut c_char { + let payload = match result { + Ok(data) => FfiResult { + ok: true, + error: None, + data: Some(data), + }, + Err(error) => FfiResult:: { + ok: false, + error: Some(error), + data: None, + }, + }; + + let json = serde_json::to_string(&payload).unwrap_or_else(|error| { + format!( + r#"{{"ok":false,"error":"failed to serialize FFI response: {}","data":null}}"#, + error + ) + }); + c_string(json).into_raw() +} + +fn c_string(value: impl Into) -> CString { + CString::new(value.into()).unwrap_or_else(|_| CString::new("").unwrap()) +} + +fn c_str_to_string(value: *const c_char) -> Result { + if value.is_null() { + return Err("null pointer".to_string()); + } + + unsafe { + CStr::from_ptr(value) + .to_str() + .map(|value| value.to_string()) + .map_err(|error| error.to_string()) + } +} + #[cfg(test)] mod tests { use super::*; use std::ffi::CStr; + fn take_string(ptr: *const c_char) -> String { + assert!(!ptr.is_null()); + let value = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap().to_string(); + unsafe { bat_free_string(ptr as *mut c_char) }; + value + } + #[test] fn test_ffi_version() { let version_ptr = bat_version(); - assert!(!version_ptr.is_null()); + let version = take_string(version_ptr); + assert!(!version.is_empty()); + } - unsafe { - let version_cstr = CStr::from_ptr(version_ptr); - let version = version_cstr.to_str().unwrap(); - assert!(!version.is_empty()); - } + #[test] + fn test_manifest_inspect_json() { + let catalog_json = r#"{ + "m_LocatorId": "AddressablesMainContentCatalog", + "m_InternalIds": [ + "synthetic/minimal.bundle" + ], + "m_Entries": [ + { + "internal_id": "synthetic/minimal.bundle", + "hash": "abc", + "size": 10, + "address": "Character_001", + "dependencies": ["shared.bundle"] + } + ] + }"#; + + let result_ptr = bat_manifest_inspect_json(CString::new(catalog_json).unwrap().as_ptr()); + let result = take_string(result_ptr); + assert!(result.contains(r#""ok":true"#)); + assert!(result.contains(r#""resource_count":1"#)); + } + + #[test] + fn test_sync_plan_json() { + let current = r#"{ + "connection_group_name":"Prod-Audit", + "app_version":"1.70.0", + "bundle_version":"s8tloc7lo3", + "addressables_root":"https://prod-clientpatch.bluearchiveyostar.com/r93_token", + "endpoints":[ + { + "kind":"TableCatalog", + "platform":null, + "url":"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes" + } + ] + }"#; + + let result_ptr = + bat_sync_plan_json(CString::new(current).unwrap().as_ptr(), std::ptr::null()); + let result = take_string(result_ptr); + assert!(result.contains(r#""ok":true"#)); + assert!(result.contains(r#""decision":"DownloadVerifyAndPublish""#)); } } diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 53a2473..fb6eadf 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -8,8 +8,9 @@ BlueArchive Toolkit 采用 **Monorepo + 多语言混合** 架构，旨在构建 当前已经可用的官方资源入口包括： -- `infrastructure/examples/official_launcher_bootstrap.rs` -- `infrastructure/examples/official_pull_plan.rs` +- `infrastructure/examples/official_pull_plan.rs`：Linux 生产资源拉取路径，显式读取官方 `server-info` 输入。 +- `infrastructure/examples/official_update_check.rs`：Linux 自动更新检查入口，保存 snapshot 并按需拉取。 +- `infrastructure/examples/official_launcher_bootstrap.rs`：显式开发/审计辅助路径，用于核查官方 launcher metadata，不是生产运行依赖。 - `docs/guides/official-resource-test-pull.md` 已接受的架构决策： diff --git a/docs/architecture/adr/0001-engine-and-application-boundaries.md b/docs/architecture/adr/0001-engine-and-application-boundaries.md index dde50c9..434d2ca 100644 --- a/docs/architecture/adr/0001-engine-and-application-boundaries.md +++ b/docs/architecture/adr/0001-engine-and-application-boundaries.md @@ -35,7 +35,7 @@ BlueArchiveToolkit 的最终目标覆盖资源同步、CAS、AssetBundle 解析 3. **Go 应用层** - 负责 CLI、资源同步、下载器、API Server、任务调度、配置、日志、Provider 编排。 - - 通过 FFI、进程边界或稳定 SDK 调用 Rust 引擎能力。 + - 通过稳定 SDK、进程边界，必要时再通过 FFI 调用 Rust 引擎能力。 - 不重复实现 AssetBundle 解析、Patch 算法或 CAS 对象存储核心逻辑。 4. **Web 层** @@ -48,8 +48,8 @@ BlueArchiveToolkit 的最终目标覆盖资源同步、CAS、AssetBundle 解析 1. Rust 引擎 API 必须保持业务无关，不出现 CLI 命令、HTTP 状态码、Web 页面状态。 2. Go 应用层不得复制 Rust 引擎中的 Hash、Patch、AssetBundle 核心算法。 -3. FFI 边界必须避免暴露大量细粒度内部结构，优先暴露批量和事务语义。 -4. 所有跨语言错误必须能映射到统一错误码和可读诊断信息。 +3. 跨边界优先级应是进程边界或稳定 SDK，其次才是 FFI；若使用 FFI，必须只暴露粗粒度批量和事务语义。 +4. 所有跨边界错误必须能映射到统一错误码和可读诊断信息。 --- diff --git a/docs/architecture/adr/0003-cas-core-interface-and-error-boundary.md b/docs/architecture/adr/0003-cas-core-interface-and-error-boundary.md index cc4fcb1..84f9771 100644 --- a/docs/architecture/adr/0003-cas-core-interface-and-error-boundary.md +++ b/docs/architecture/adr/0003-cas-core-interface-and-error-boundary.md @@ -45,6 +45,6 @@ CAS V1 对外领域接口继续使用 `bat_core::repositories::cas_repository::C ## 后果 -1. 后续 Go CLI/API 只能通过领域仓储接口或稳定 FFI 调用 CAS，不直接依赖对象目录结构。 +1. 后续 Go CLI/API 只能通过领域仓储接口、稳定 SDK 或进程边界调用 CAS，不直接依赖对象目录结构；FFI 仅作为可选实现方式。 2. CAS V1 后续可以替换元数据后端，但不能改变领域接口语义。 3. 如果以后需要 dry-run GC、批量引用更新或流式存储，应作为新接口扩展，不破坏当前 trait。 diff --git a/docs/architecture/official-resource-backend.md b/docs/architecture/official-resource-backend.md index 01550b4..e78c9e2 100644 --- a/docs/architecture/official-resource-backend.md +++ b/docs/architecture/official-resource-backend.md @@ -11,6 +11,8 @@ - `bluearchive.cafe` - 任何镜像层、转写层、二次代理层 - 人工拼接出来的样例 URL +- Linux 生产环境安装或执行官方启动器 +- 把已安装客户端目录或官方启动器安装目录当作生产输入 当前默认平台集合是： @@ -29,9 +31,23 @@ | 下载层 | 校验官方 URL，调用下载器，落盘并记录字节数 | 得到本地资源副本 | | 导入层 | 将 bundle 写入 CAS 和 ResourceRepository | 得到可查询的资源索引 | | 同步层 | 比较当前快照和历史快照 | 决定下载、校验、发布 | +| 更新层 | 保存上次官方 snapshot，定期执行 discovery + diff + pull | 形成自动更新闭环 | ## 3. 工作原理 +### 3.0 Linux 生产约束 + +生产环境目标是 Linux 后端任务，不是 Windows 桌面客户端环境。 + +因此生产同步入口必须满足： + +1. 不要求安装官方启动器。 +2. 不要求启动官方启动器进程。 +3. 不要求把生产环境当作客户端安装目录。 +4. 可以显式执行 official metadata discovery 自动发现 `server-info` URL、`connection-group` 和 `app-version`。 +5. 也可以通过配置、调度状态或已审计 metadata snapshot 显式提供这些值。 +6. `--auto-discover` 只允许通过官方 HTTP metadata 和临时目录解析 `GameMainConfig`；launcher metadata 未变时必须复用缓存，metadata 变化时才重新下载必要官方 game zip。 + ### 3.1 发现官方资源根 入口是 `YostarJpServerInfo`。 @@ -105,9 +121,16 @@ 1. 检查 URL 是否属于官方 host。 2. 把 URL 映射到本地输出路径。 -3. 通过系统 `curl` 下载。 -4. 记录文件大小和最终路径。 -5. 非官方 URL 直接拒绝。 +3. 读取输出目录下的 `official-download-manifest.json`。 +4. 已存在目标文件只有在本地清单中的相对路径、size、BLAKE3 都匹配时才跳过。 +5. 缺少清单、清单不匹配或文件损坏时重新下载。 +6. `TableCatalog.bytes`、`BundlePackingInfo.bytes`、`MediaCatalog.bytes` 总是刷新并用官方 `.hash` 强校验；该 `.hash` 是 `xxHash32(seed=0)` 的十进制文本。 +7. `catalog_*.hash` 当前只作为 Addressables catalog 变更标记，不作为 zip/JSON 内容校验算法；Unity Addressables/SBP builder 对 JSON/bin catalog 使用 `HashingMethods.Calculate` 生成 `Hash128` 文本，运行时用它判断 remote catalog cache 是否过期，它不能套用 seed catalog 的 `xxHash32` 规则。 +8. 存在 `.part` 临时文件时通过 `curl --continue-at -` 尝试断点续传。 +9. 新下载写入 `.part`，成功后原子 rename 到最终路径。 +10. 成功下载后更新本地下载清单。 +11. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 +12. 非官方 URL 直接拒绝。 路径映射时会做分段清理，避免把不安全路径写进输出目录。 @@ -152,13 +175,46 @@ - `infrastructure/src/official_sync.rs` +### 3.7 自动更新闭环 + +`OfficialUpdateService` 是当前 Rust 侧的自动更新核心，正式命令入口是 `bat-official-sync`。 + +流程是： + +1. 显式执行 `--auto-discover` 或读取已审计 `server-info` 输入。 +2. `--auto-discover` 先抓官方 launcher metadata；metadata 未变时复用 `official-bootstrap-cache.json` 中的 `GameMainConfig` 摘要，metadata 变化时临时下载官方 game zip 并重新解析。 +3. 生成当前 v2 snapshot，记录 `app_version`、`connection_group`、`bundle_version`、`addressables_root`、endpoint URL、seed `.hash` 内容、`catalog_*.hash` marker、launcher metadata 摘要和 `GameMainConfig` 摘要。 +4. 读取上一次成功同步写出的 snapshot。 +5. 使用 `OfficialSyncPlan` 和扩展 snapshot diff 判断是否需要下载；URL 未变但 `.hash` / marker 内容变化也会触发更新。 +6. 远端无变化时执行本地 download manifest audit，检查路径、size 和 BLAKE3。 +7. 远端变化或本地 audit 发现 repair_needed 时生成 pull plan，下载并校验官方 URL。 +8. 下载成功后写回新的 snapshot。 + +该入口不安装、不执行官方启动器，也不读取生产外的本地客户端目录。Rust 正式 binary 支持 `--watch` 常驻模式，默认每 1 小时执行一次检查；远端和本地一致时静默等待下次检查，不一致时自动下载或 repair。单次运行仍保留为核心幂等路径，systemd service、容器或 Go 进程可以只负责守护该常驻进程；cron/systemd timer 调单次模式只是可选集成方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。 + +对应实现主要在： + +- `infrastructure/src/official_update.rs` +- `infrastructure/src/bin/bat_official_sync.rs` +- `infrastructure/examples/official_update_check.rs`（历史/开发入口） + ## 4. 官方 bootstrap 与用户流程 -当前已经提供的官方用户流程是： +当前已经提供的官方用户流程分两类。 + +Linux 生产路径： + +1. 显式执行 `--auto-discover`，或提供已审计的官方 metadata snapshot。 +2. `official_pull_plan` 依据官方 `server-info`、catalog 和 verified platforms 生成全量拉取计划。 +3. `bat-official-sync` 对比上次 snapshot 和远端 `.hash` marker；远端无变化时 audit 本地清单，有变化或本地损坏时下载、校验并落盘官方资源。 + +开发/审计辅助路径： 1. `official_launcher_bootstrap` 获取官方 launcher 信息。 2. `official_game_main_config` 从官方 ZIP 解出并解密 `GameMainConfig`。 -3. `official_pull_plan` 依据官方 `server-info`、catalog 和 verified platforms 生成全量拉取计划。 +3. 手动确认最新 PC metadata、`server-info` URL 和默认 connection group。 + +自动发现路径和开发/审计辅助路径都只使用官方 HTTP 和临时目录，不安装或启动官方 launcher。 这些入口都只接受官方域名，不走 `bluearchive.cafe` 镜像链。 @@ -171,6 +227,12 @@ - verified inventory 会产出完整内容 URL 集 - pull plan 会同时包含 discovery URLs 和 content URLs - 全量样本下是 `2` 个 discovery URL + `5` 个内容 URL = `7` 个 URL +- `OfficialUpdateService` 能持久化 v2 snapshot，并在远端 marker 内容变化时触发下载决策 +- `bat-official-sync` 输出稳定 JSON report，支持 `--watch --interval 1h` 常驻运行，非 dry-run 使用 `.official-sync.lock` 防止并发写状态目录 +- `OfficialUpdateService` 能读写 `official-bootstrap-cache.json`，并支持默认开启的 `audit_local` / `repair` CLI 行为 +- 下载层能在本地文件 size/BLAKE3/path 或 manifest 不匹配时重新下载 +- 官方 seed `.hash` mismatch 会导致下载失败，而不是降级为本地 BLAKE3 猜测 +- 非官方 URL 会在 fetch/download 入口被拒绝 相关验证主要来自： @@ -178,6 +240,7 @@ - `cargo test -p bat-infrastructure` - `cargo test -p bat-adapters --examples` - `cargo test -p bat-infrastructure --examples` +- `cargo test -p bat-infrastructure --bin bat-official-sync -- --nocapture` - `cargo clippy -p bat-adapters -- -D warnings` - `cargo clippy -p bat-infrastructure -- -D warnings` diff --git a/docs/guides/official-resource-test-pull.md b/docs/guides/official-resource-test-pull.md index 723bcec..8091eef 100644 --- a/docs/guides/official-resource-test-pull.md +++ b/docs/guides/official-resource-test-pull.md @@ -13,26 +13,23 @@ - `macOS` - `bluearchive.cafe` 或其他镜像域名 - 任何本地客户端目录 +- 任何已安装的官方启动器或 Windows 客户端 ## 1. 当前流程 -运行时链路只走官方日服： +Linux 生产运行时链路只走官方日服 HTTP 资源，不安装、不启动、不依赖官方启动器二进制： -1. 请求官方 launcher API。 -2. 从官方 launcher manifest 链路得到最新游戏包 ZIP。 -3. 解包 `resources.assets`。 -4. 解密 `GameMainConfig`。 -5. 从 `GameMainConfig` 自动读取 `server-info` URL 和默认 `connection-group`。 -6. 请求官方 `server-info`。 -7. 生成 Windows + Android 的官方资源 discovery 端点。 -8. 拉取 seed catalog，生成完整官方 pull plan。 -9. dry-run 只输出 URL；非 dry-run 下载全部官方 URL。 +1. 显式执行 `--auto-discover`，通过官方 HTTP metadata 自动发现 `app-version`、`server-info` URL 和默认 `connection-group`。 +2. 请求官方 `server-info`。 +3. 生成 Windows + Android 的官方资源 discovery 端点。 +4. 拉取 seed catalog，生成完整官方 pull plan。 +5. dry-run 只输出 URL；非 dry-run 下载全部官方 URL。 -`--server-info-file` 和 `--server-info-path` 都不是硬依赖。 +`--auto-discover` 会下载官方 metadata 所需的临时包并解析 `GameMainConfig`，但不会安装官方启动器，也不会执行官方启动器进程。`--launcher-bootstrap` 只是旧命名兼容别名，新流程不要再推荐使用。 -## 2. 先做 launcher bootstrap +## 2. 可选 metadata 审计 -先确认 launcher API 和官方 ZIP 链路正常： +如需单独审计官方 metadata API 和官方 ZIP 链路，可以手动运行： ```bash cargo run -p bat-infrastructure --example official_launcher_bootstrap -- \ @@ -54,17 +51,17 @@ cargo run -p bat-infrastructure --example official_launcher_bootstrap -- \ ## 3. 先做 dry-run -推荐默认流程是不提供本地 `server-info` 文件，也不提供本地客户端目录： +推荐开发和生产自动化的默认流程是显式打开 `--auto-discover`： ```bash cargo run -p bat-infrastructure --example official_pull_plan -- \ - --launcher-bootstrap \ + --auto-discover \ --platforms Windows,Android \ --output /tmp/ba-official-pull \ --dry-run ``` -如果你已经有官方 `server-info` 文件，也可以显式传入： +如果你要复现某次已审计的 metadata snapshot，也可以显式传入官方 `server-info` 文件名： ```bash cargo run -p bat-infrastructure --example official_pull_plan -- \ @@ -76,6 +73,18 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ --dry-run ``` +如果你已经把官方 `server-info` JSON 存成只读本地文件，也可以使用： + +```bash +cargo run -p bat-infrastructure --example official_pull_plan -- \ + --server-info-path /path/to/server-info.json \ + --connection-group Prod-Audit \ + --app-version 1.70.0 \ + --platforms Windows,Android \ + --output /tmp/ba-official-pull \ + --dry-run +``` + 检查输出时，重点看： - `platforms=[Windows, Android]` @@ -89,7 +98,7 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ ```bash cargo run -p bat-infrastructure --example official_pull_plan -- \ - --launcher-bootstrap \ + --auto-discover \ --platforms Windows,Android \ --output /tmp/ba-official-pull ``` @@ -98,28 +107,121 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - 只接受官方 URL - 下载完整官方资源集合 +- 下载成功后在输出目录维护 `official-download-manifest.json` +- 已存在目标文件只有在本地下载清单中的 size 和 BLAKE3 校验通过时才会跳过 +- `TableCatalog.bytes`、`BundlePackingInfo.bytes`、`MediaCatalog.bytes` 会用官方 `.hash` 做强校验；该 `.hash` 是 `xxHash32(seed=0)` 的十进制文本 +- `catalog_*.hash` 是 Unity Addressables/SBP 写出的 catalog 变更标记，来源是 `HashingMethods.Calculate(json/bin catalog)` 生成的 `Hash128` 文本；它不是 seed catalog 的 `xxHash32` 规则，目前不作为 zip/JSON 内容强校验 +- 存在 `.part` 临时文件时会尝试断点续传 +- 新下载先写 `.part`，成功后再替换为最终文件 - 把结果写入 `--output` -## 5. 例外输入 +## 5. 自动更新检查 -如果必须使用本地 `server-info`，只接受： +正式自动更新入口是 `bat-official-sync`。它会保存上一次成功同步的 snapshot，下次运行时先自动发现当前官方 metadata，再和 snapshot 对比： +- 每轮都会先拉取轻量官方 metadata 和 `.hash` marker。 +- URL 没变但 `.hash` / marker 内容变更时，也会判定为需要更新。 +- 远端无变化时，默认执行本地 `official-download-manifest.json` audit。 +- 本地文件缺失、路径不一致、size 不一致或 BLAKE3 不一致时，默认进入 repair 并重下必要文件。 +- 远端和本地都一致：单次模式输出 `update_status=up_to_date`，watch 模式默认静默并等待下次检查。 +- 有远端变化或本地 repair：生成 pull plan，下载完整官方资源，成功后更新 snapshot。 +- `--dry-run`：只报告本次是否会下载，不写 snapshot；如果 cache miss，也不会写入新的 bootstrap cache。 +- `--dry-run --plan`：除更新判断外，还会解析 seed catalog 并打印完整下载 URL。 +- 真实更新会输出 `downloaded_count`、`resumed_count`、`skipped_count`、`transferred_bytes`、`official_seed_hash_verified_count`。 +- 校验报告分层输出 `official_seed_hash_verified_count`、`local_manifest_verified_count`、`addressables_marker_checked_count`、`unverified_marker_count`。 +- 下载阶段复用同一套本地清单和 `.part` 续传逻辑；没有清单或校验不匹配的文件会重新下载。 + +状态文件默认位于 `--output` 下： + +- `official-sync-snapshot.json`：上一次成功同步的 v2 snapshot，包含 app version、connection group、bundle version、addressables root、endpoint URL、官方 seed `.hash` 内容、Addressables `catalog_*.hash` marker、launcher metadata 摘要和 `GameMainConfig` 摘要。 +- `official-bootstrap-cache.json`：`--auto-discover` 的 `GameMainConfig` 解析缓存。launcher metadata 未变时复用缓存；metadata 变化时才通过官方 HTTP 下载必要 game zip 到临时目录解析。 +- `official-download-manifest.json`：本地下载强校验清单，记录 URL、相对路径、size 和 BLAKE3。 + +先 dry-run： + +```bash +cargo run -p bat-infrastructure --bin bat-official-sync -- \ + --auto-discover \ + --platforms Windows,Android \ + --output /tmp/ba-official-update \ + --snapshot /tmp/ba-official-update/official-sync-snapshot.json \ + --dry-run +``` + +如果需要在 dry-run 阶段审阅完整下载 URL，加 `--plan`： + +```bash +cargo run -p bat-infrastructure --bin bat-official-sync -- \ + --auto-discover \ + --platforms Windows,Android \ + --output /tmp/ba-official-update \ + --snapshot /tmp/ba-official-update/official-sync-snapshot.json \ + --dry-run \ + --plan +``` + +确认后执行真实更新： + +```bash +cargo run -p bat-infrastructure --bin bat-official-sync -- \ + --auto-discover \ + --platforms Windows,Android \ + --output /tmp/ba-official-update \ + --snapshot /tmp/ba-official-update/official-sync-snapshot.json +``` + +Rust 常驻更新模式默认每 1 小时执行一次检查，符合则静默，不符合则自动拉取/repair 并输出 JSON report： + +```bash +cargo run -p bat-infrastructure --bin bat-official-sync -- \ + --auto-discover \ + --platforms Windows,Android \ + --output /tmp/ba-official-update \ + --snapshot /tmp/ba-official-update/official-sync-snapshot.json \ + --watch +``` + +如需调整间隔： + +```bash +cargo run -p bat-infrastructure --bin bat-official-sync -- \ + --auto-discover \ + --platforms Windows,Android \ + --output /tmp/ba-official-update \ + --watch \ + --interval 30m +``` + +命令成功时 stdout 输出稳定 JSON report；错误时 stderr 输出 JSON error。普通错误 exit `1`，状态目录锁冲突 exit `75`。 + +生产可以直接运行 `--watch`，也可以用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一状态目录。 + +需要只做探测时可以加 `--dry-run`。需要关闭本地 audit 或 repair 时可以显式使用 `--no-audit-local` 或 `--no-repair`，但生产同步默认应保持开启。 + +## 6. 例外输入 + +可接受的 `server-info` 输入是： + +- `--auto-discover` +- `--server-info-url <官方 URL>` - `--server-info-file <官方文件名>` - `--server-info-path <本地官方 JSON>` -但这两项不再是主流程依赖。 +如果没有显式 `server-info` 输入，必须手动加 `--auto-discover` 才会走官方 metadata / `GameMainConfig` 辅助发现链路。不要在 Linux 生产任务里依赖已安装启动器或本地客户端目录。 -## 6. 代码入口 +## 7. 代码入口 当前可用的用户入口： - `infrastructure/examples/official_launcher_bootstrap.rs` - `infrastructure/examples/official_pull_plan.rs` +- `infrastructure/src/bin/bat_official_sync.rs` +- `infrastructure/examples/official_update_check.rs`（历史/开发入口；生产优先使用 `bat-official-sync`） - `adapters/examples/yostar_jp_client_bootstrap.rs` - `adapters/examples/yostar_jp_discovery.rs` - `adapters/examples/yostar_jp_inventory.rs` -## 7. 验证 +## 8. 验证 推荐验证命令： @@ -128,5 +230,5 @@ cargo test -p bat-adapters cargo test -p bat-infrastructure cargo test -p bat-infrastructure --example official_launcher_bootstrap -- --nocapture cargo test -p bat-infrastructure --example official_pull_plan -- --nocapture +cargo test -p bat-infrastructure --bin bat-official-sync -- --nocapture ``` - diff --git a/docs/reports/current-stage-prepush.md b/docs/reports/current-stage-prepush.md index 69cb4cc..46980f4 100644 --- a/docs/reports/current-stage-prepush.md +++ b/docs/reports/current-stage-prepush.md @@ -16,7 +16,9 @@ 已确认可用的内容： - `bat-cas-engine` 的 CAS V1 已完成。 -- `bat-adapters` 的官方日服 URL 规则、官方 launcher bootstrap、官方 GameMainConfig bootstrap、官方 pull plan 已可用。 +- `bat-adapters` 的官方日服 URL 规则和官方 pull plan / update check 已可用。 +- Linux 生产资源拉取路径已明确为显式 `--auto-discover` 或已审计 metadata snapshot，不安装、不执行官方启动器。 +- 官方 launcher bootstrap、官方 GameMainConfig bootstrap 保留为底层显式开发/审计辅助路径。 - `bat-infrastructure` 的官方 bootstrap / pull integration 测试已通过。 - 官方资源链路已经明确只接受官方 host，不依赖 `bluearchive.cafe` 镜像。 diff --git a/examples/manifest_demo.rs b/examples/manifest_demo.rs index 477451f..db528a5 100644 --- a/examples/manifest_demo.rs +++ b/examples/manifest_demo.rs @@ -32,7 +32,14 @@ async fn main() -> anyhow::Result<()> { println!(" 资源数量: {}", manifest.resources.len()); println!(" 资源列表:"); for (i, resource) in manifest.resources.iter().enumerate() { - println!(" {}. {} ({})", i + 1, resource.path, resource.resource_type); + println!( + " {}. {} ({}, address={:?}, deps={})", + i + 1, + resource.path, + resource.resource_type, + resource.address, + resource.dependencies.len() + ); } } diff --git a/infrastructure/Cargo.toml b/infrastructure/Cargo.toml index 8b32234..cf41be3 100644 --- a/infrastructure/Cargo.toml +++ b/infrastructure/Cargo.toml @@ -5,6 +5,10 @@ edition.workspace = true authors.workspace = true license.workspace = true +[[bin]] +name = "bat-official-sync" +path = "src/bin/bat_official_sync.rs" + [dependencies] bat-core = { path = "../core" } bat-adapters = { path = "../adapters" } @@ -13,11 +17,13 @@ anyhow.workspace = true thiserror.workspace = true serde.workspace = true serde_json.workspace = true +blake3.workspace = true tokio.workspace = true async-trait.workspace = true md-5 = "0.10" hex = "0.4" tempfile = "3.14" +sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/infrastructure/examples/official_pull_plan.rs b/infrastructure/examples/official_pull_plan.rs index 3b8a0fb..468231c 100644 --- a/infrastructure/examples/official_pull_plan.rs +++ b/infrastructure/examples/official_pull_plan.rs @@ -22,6 +22,7 @@ struct CliArgs { app_version: Option, launcher_version: String, launcher_bootstrap: bool, + auto_discover: bool, platforms: Option>, output_root: PathBuf, curl_command: PathBuf, @@ -32,6 +33,7 @@ struct CliArgs { enum ServerInfoSource { LocalPath(PathBuf), OfficialFile(String), + OfficialUrl(String), } fn main() -> anyhow::Result<()> { @@ -41,11 +43,8 @@ fn main() -> anyhow::Result<()> { .platforms .as_deref() .unwrap_or(default_platforms.as_slice()); - let needs_official_bootstrap = args.launcher_bootstrap - || args.server_info_source.is_none() - || args.connection_group.is_none() - || args.app_version.is_none(); - let official_bootstrap = if needs_official_bootstrap { + let metadata_bootstrap_enabled = args.auto_discover || args.launcher_bootstrap; + let official_bootstrap = if metadata_bootstrap_enabled { Some( OfficialGameMainConfigBootstrapService::with_commands( &args.launcher_version, @@ -66,9 +65,7 @@ fn main() -> anyhow::Result<()> { .as_ref() .map(|bootstrap| bootstrap.game_config.game_latest_version.clone()) }) - .ok_or_else(|| { - anyhow::anyhow!("missing --app-version and official bootstrap failed") - })?; + .ok_or_else(|| anyhow::anyhow!("missing --app-version; production mode does not infer the game version from official metadata unless --auto-discover is set"))?; let connection_group = args .connection_group .clone() @@ -79,27 +76,27 @@ fn main() -> anyhow::Result<()> { }) .ok_or_else(|| { anyhow::anyhow!( - "missing --connection-group and official GameMainConfig has no default_connection_group" + "missing --connection-group; production mode does not infer the connection group from GameMainConfig unless --auto-discover is set" ) })?; let fetcher = OfficialResourcePullService::with_curl_command(&args.output_root, &args.curl_command); - if args.launcher_bootstrap { + if metadata_bootstrap_enabled { let bootstrap = official_bootstrap.as_ref().ok_or_else(|| { - anyhow::anyhow!("launcher bootstrap requested but official bootstrap is unavailable") + anyhow::anyhow!("metadata bootstrap requested but official bootstrap is unavailable") })?; println!( - "launcher_api_game_latest_version={}", + "metadata_bootstrap_game_latest_version={}", bootstrap.game_config.game_latest_version ); println!( - "launcher_api_game_latest_file_path={}", + "metadata_bootstrap_game_latest_file_path={}", bootstrap.game_config.game_latest_file_path ); println!( - "launcher_api_game_lowest_version={}", + "metadata_bootstrap_game_lowest_version={}", bootstrap .game_config .game_lowest_version @@ -107,7 +104,7 @@ fn main() -> anyhow::Result<()> { .unwrap_or("") ); println!( - "launcher_api_game_start_exe_name={}", + "metadata_bootstrap_game_start_exe_name={}", bootstrap .game_config .game_start_exe_name @@ -115,22 +112,28 @@ fn main() -> anyhow::Result<()> { .unwrap_or("") ); println!( - "launcher_api_game_start_params={:?}", + "metadata_bootstrap_game_start_params={:?}", bootstrap.game_config.game_start_params ); - println!("launcher_api_primary_cdn={}", bootstrap.cdn_config.primary_cdn); - println!("launcher_api_backup_cdn={}", bootstrap.cdn_config.back_up_cdn); - println!("launcher_api_manifest_url={}", bootstrap.manifest_url); println!( - "launcher_api_manifest_source={}", + "metadata_bootstrap_primary_cdn={}", + bootstrap.cdn_config.primary_cdn + ); + println!( + "metadata_bootstrap_backup_cdn={}", + bootstrap.cdn_config.back_up_cdn + ); + println!("metadata_bootstrap_manifest_url={}", bootstrap.manifest_url); + println!( + "metadata_bootstrap_manifest_source={}", bootstrap.manifest_source.as_deref().unwrap_or("") ); println!( - "launcher_api_manifest_file_count={}", + "metadata_bootstrap_manifest_file_count={}", bootstrap.manifest_file_count ); println!( - "launcher_api_game_main_config_server_info_url={}", + "metadata_bootstrap_game_main_config_server_info_url={}", bootstrap .game_main_config .server_info_data_url @@ -138,7 +141,7 @@ fn main() -> anyhow::Result<()> { .unwrap_or("") ); println!( - "launcher_api_game_main_config_default_connection_group={}", + "metadata_bootstrap_game_main_config_default_connection_group={}", bootstrap .game_main_config .default_connection_group @@ -148,15 +151,13 @@ fn main() -> anyhow::Result<()> { println!(); } - let server_info_source = args - .server_info_source - .as_ref(); + let server_info_source = args.server_info_source.as_ref(); let server_info_bytes = if let Some(source) = server_info_source { load_server_info(source, &fetcher)? } else { let bootstrap = official_bootstrap.as_ref().ok_or_else(|| { - anyhow::anyhow!("official bootstrap is required when no --server-info-file or --server-info-path is provided") + anyhow::anyhow!("missing server-info source; pass --server-info-url, --server-info-file, or --server-info-path, or explicitly opt in with --auto-discover") })?; let url = bootstrap .game_main_config @@ -207,8 +208,20 @@ fn main() -> anyhow::Result<()> { let downloader = OfficialResourcePullService::with_curl_command(args.output_root, args.curl_command); let report = downloader.pull(&plan).map_err(anyhow::Error::msg)?; - eprintln!("downloaded_count={}", report.items.len()); - eprintln!("downloaded_bytes={}", report.total_bytes()); + eprintln!("resource_count={}", report.items.len()); + eprintln!("downloaded_count={}", report.downloaded_count()); + eprintln!("resumed_count={}", report.resumed_count()); + eprintln!("skipped_count={}", report.skipped_count()); + eprintln!("final_bytes={}", report.total_bytes()); + eprintln!("transferred_bytes={}", report.transferred_bytes()); + eprintln!( + "official_hash_verified_count={}", + report.official_hash_verified_count() + ); + eprintln!( + "download_manifest={}", + downloader.download_manifest_path().display() + ); Ok(()) } @@ -223,6 +236,7 @@ fn load_server_info( let url = server_info_url(file_name).map_err(anyhow::Error::msg)?; fetcher.fetch_bytes(&url).map_err(anyhow::Error::msg) } + ServerInfoSource::OfficialUrl(url) => fetcher.fetch_bytes(url).map_err(anyhow::Error::msg), } } @@ -331,7 +345,10 @@ fn required_platform(endpoint: &YostarJpResourceEndpoint) -> anyhow::Result anyhow::Result { - let raw_args = env::args().collect::>(); + parse_args_from(env::args()) +} + +fn parse_args_from(raw_args: impl IntoIterator) -> anyhow::Result { let mut args = raw_args.into_iter(); let binary = args .next() @@ -342,6 +359,7 @@ fn parse_args() -> anyhow::Result { app_version: None, launcher_version: "1.7.2".to_string(), launcher_bootstrap: false, + auto_discover: false, platforms: None, output_root: PathBuf::from("official_pull_output"), curl_command: PathBuf::from("curl"), @@ -350,6 +368,11 @@ fn parse_args() -> anyhow::Result { while let Some(flag) = args.next() { match flag.as_str() { + "--server-info-url" => { + parsed.server_info_source = Some(ServerInfoSource::OfficialUrl(next_option_value( + &mut args, &flag, + )?)); + } "--server-info-file" => { parsed.server_info_source = Some(ServerInfoSource::OfficialFile( next_option_value(&mut args, &flag)?, @@ -372,6 +395,9 @@ fn parse_args() -> anyhow::Result { "--launcher-bootstrap" => { parsed.launcher_bootstrap = true; } + "--auto-discover" => { + parsed.auto_discover = true; + } "--platforms" => { let value = next_option_value(&mut args, &flag)?; parsed.platforms = Some(parse_platforms(&value).map_err(anyhow::Error::msg)?); @@ -406,9 +432,9 @@ fn next_option_value( fn print_usage(binary: &str) { eprintln!( - "usage: {binary} [--server-info-file | --server-info-path ] \ + "usage: {binary} [--server-info-url | --server-info-file | --server-info-path ] \ [--app-version 1.70.0] \ - [--launcher-bootstrap] [--launcher-version 1.7.2] [--connection-group ] \ + [--connection-group ] [--auto-discover] [--launcher-bootstrap] [--launcher-version 1.7.2] \ [--platforms Windows,Android] [--output official_pull_output] [--curl curl] [--dry-run]" ); } @@ -429,3 +455,58 @@ fn parse_platform(value: &str) -> Result { _ => Err(format!("unsupported platform: {value}")), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(values: &[&str]) -> anyhow::Result { + parse_args_from(values.iter().map(|value| value.to_string())) + } + + #[test] + fn parses_explicit_official_server_info_url_without_launcher_bootstrap() { + let args = parse(&[ + "official_pull_plan", + "--server-info-url", + "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json", + "--connection-group", + "Prod-Audit", + "--app-version", + "1.70.0", + "--dry-run", + ]) + .unwrap(); + + assert!(!args.launcher_bootstrap); + assert_eq!(args.connection_group.as_deref(), Some("Prod-Audit")); + assert_eq!(args.app_version.as_deref(), Some("1.70.0")); + assert!(matches!( + args.server_info_source, + Some(ServerInfoSource::OfficialUrl(ref url)) + if url == "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json" + )); + } + + #[test] + fn auto_discover_is_explicit_opt_in() { + let args = parse(&["official_pull_plan", "--auto-discover", "--dry-run"]).unwrap(); + + assert!(args.auto_discover); + assert!(!args.launcher_bootstrap); + assert!(args.server_info_source.is_none()); + assert!(args.connection_group.is_none()); + assert!(args.app_version.is_none()); + } + + #[test] + fn launcher_bootstrap_alias_is_explicit_opt_in() { + let args = parse(&["official_pull_plan", "--launcher-bootstrap", "--dry-run"]).unwrap(); + + assert!(!args.auto_discover); + assert!(args.launcher_bootstrap); + assert!(args.server_info_source.is_none()); + assert!(args.connection_group.is_none()); + assert!(args.app_version.is_none()); + } +} diff --git a/infrastructure/examples/official_update_check.rs b/infrastructure/examples/official_update_check.rs new file mode 100644 index 0000000..35974b9 --- /dev/null +++ b/infrastructure/examples/official_update_check.rs @@ -0,0 +1,1133 @@ +use bat_adapters::official::game_main_config::YostarJpGameMainConfig; +use bat_adapters::official::inventory::{ + YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, +}; +use bat_adapters::official::yostar_jp::{ + server_info_url, PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, + YostarJpResourceEndpointKind, YostarJpServerInfo, YostarJpSyncSnapshot, +}; +use bat_infrastructure::{ + build_official_pull_plan_for_platform_inventory, build_official_sync_plan, + changed_endpoint_urls, default_official_platforms, OfficialGameMainConfigBootstrapService, + OfficialLauncherBootstrapService, OfficialResourcePullPlan, OfficialResourcePullService, + YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +const OFFICIAL_UPDATE_SNAPSHOT_VERSION: u32 = 2; +const OFFICIAL_BOOTSTRAP_CACHE_VERSION: u32 = 1; + +#[derive(Debug)] +struct CliArgs { + server_info_source: Option, + connection_group: Option, + app_version: Option, + launcher_version: String, + auto_discover: bool, + platforms: Option>, + output_root: PathBuf, + snapshot_path: Option, + curl_command: PathBuf, + dry_run: bool, + plan: bool, + force: bool, + audit_local: bool, + repair: bool, +} + +#[derive(Debug)] +enum ServerInfoSource { + LocalPath(PathBuf), + OfficialFile(String), + OfficialUrl(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialUpdateSnapshot { + #[serde(default = "default_update_snapshot_version")] + snapshot_version: u32, + connection_group_name: String, + app_version: String, + bundle_version: Option, + addressables_root: String, + endpoints: Vec, + #[serde(default)] + endpoint_markers: Vec, + #[serde(default)] + launcher_metadata: Option, + #[serde(default)] + game_main_config_bootstrap: Option, +} + +impl OfficialUpdateSnapshot { + fn new( + base: YostarJpSyncSnapshot, + endpoint_markers: Vec, + bootstrap: Option<&ResolvedBootstrap>, + ) -> Self { + Self { + snapshot_version: OFFICIAL_UPDATE_SNAPSHOT_VERSION, + connection_group_name: base.connection_group_name, + app_version: base.app_version, + bundle_version: base.bundle_version, + addressables_root: base.addressables_root, + endpoints: base.endpoints, + endpoint_markers, + launcher_metadata: bootstrap.map(|bootstrap| bootstrap.launcher_metadata.clone()), + game_main_config_bootstrap: bootstrap + .map(|bootstrap| bootstrap.game_main_config.clone()), + } + } + + fn base_snapshot(&self) -> YostarJpSyncSnapshot { + YostarJpSyncSnapshot { + connection_group_name: self.connection_group_name.clone(), + app_version: self.app_version.clone(), + bundle_version: self.bundle_version.clone(), + addressables_root: self.addressables_root.clone(), + endpoints: self.endpoints.clone(), + } + } + + fn addressables_marker_checked_count(&self) -> usize { + self.endpoint_markers + .iter() + .filter(|marker| marker.role == OfficialEndpointMarkerRole::AddressablesCatalogMarker) + .count() + } + + fn unverified_marker_count(&self) -> usize { + self.addressables_marker_checked_count() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialEndpointMarkerSnapshot { + kind: YostarJpResourceEndpointKind, + platform: Option, + url: String, + role: OfficialEndpointMarkerRole, + value: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +enum OfficialEndpointMarkerRole { + OfficialSeedHash, + AddressablesCatalogMarker, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct LauncherMetadataSnapshot { + launcher_version: String, + game_latest_version: String, + game_latest_file_path: String, + game_lowest_version: Option, + game_start_exe_name: Option, + game_start_params: Vec, + manifest_url: String, + manifest_source: Option, + manifest_file_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct GameMainConfigSnapshot { + server_info_data_url: Option, + default_connection_group: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialBootstrapCache { + #[serde(default = "default_bootstrap_cache_version")] + cache_version: u32, + launcher_metadata: LauncherMetadataSnapshot, + game_main_config: GameMainConfigSnapshot, +} + +#[derive(Debug, Clone)] +struct ResolvedBootstrap { + launcher_metadata: LauncherMetadataSnapshot, + game_main_config: GameMainConfigSnapshot, + cache_hit: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExtendedSnapshotDelta { + endpoint_markers_changed: bool, + launcher_metadata_changed: bool, + game_main_config_changed: bool, +} + +impl ExtendedSnapshotDelta { + fn has_changes(self) -> bool { + self.endpoint_markers_changed + || self.launcher_metadata_changed + || self.game_main_config_changed + } +} + +fn main() -> anyhow::Result<()> { + let args = parse_args()?; + let default_platforms = default_official_platforms(); + let platforms = args + .platforms + .as_deref() + .unwrap_or(default_platforms.as_slice()); + let fetcher = + OfficialResourcePullService::with_curl_command(&args.output_root, &args.curl_command); + let snapshot_path = args + .snapshot_path + .clone() + .unwrap_or_else(|| args.output_root.join("official-sync-snapshot.json")); + let bootstrap_cache_path = args.output_root.join("official-bootstrap-cache.json"); + + let bootstrap = if args.auto_discover { + Some(resolve_bootstrap( + &args.launcher_version, + &args.curl_command, + &bootstrap_cache_path, + !args.dry_run, + )?) + } else { + None + }; + + let app_version = args + .app_version + .clone() + .or_else(|| { + bootstrap + .as_ref() + .map(|bootstrap| bootstrap.launcher_metadata.game_latest_version.clone()) + }) + .ok_or_else(|| { + anyhow::anyhow!("missing --app-version; pass it explicitly or use --auto-discover") + })?; + let connection_group = args + .connection_group + .clone() + .or_else(|| { + bootstrap + .as_ref() + .and_then(|bootstrap| bootstrap.game_main_config.default_connection_group.clone()) + }) + .ok_or_else(|| { + anyhow::anyhow!("missing --connection-group; pass it explicitly or use --auto-discover") + })?; + + let server_info_bytes = if let Some(source) = args.server_info_source.as_ref() { + load_server_info(source, &fetcher)? + } else { + let bootstrap = bootstrap.as_ref().ok_or_else(|| { + anyhow::anyhow!("missing server-info source; pass --server-info-url, --server-info-file, --server-info-path, or use --auto-discover") + })?; + let url = bootstrap + .game_main_config + .server_info_data_url + .as_deref() + .ok_or_else(|| anyhow::anyhow!("official GameMainConfig has no ServerInfoDataUrl"))?; + fetcher.fetch_bytes(url).map_err(anyhow::Error::msg)? + }; + + let server_info = YostarJpServerInfo::from_slice(&server_info_bytes)?; + let current_snapshot = server_info + .sync_snapshot(&connection_group, &app_version, platforms) + .map_err(anyhow::Error::msg)?; + let endpoint_markers = collect_endpoint_markers(&fetcher, ¤t_snapshot)?; + let current_update_snapshot = OfficialUpdateSnapshot::new( + current_snapshot.clone(), + endpoint_markers, + bootstrap.as_ref(), + ); + let previous_snapshot = read_snapshot(&snapshot_path)?; + let previous_base_snapshot = previous_snapshot + .as_ref() + .map(OfficialUpdateSnapshot::base_snapshot); + let sync_plan = + build_official_sync_plan(current_snapshot.clone(), previous_base_snapshot.as_ref()); + let extended_delta = + diff_extended_snapshot(¤t_update_snapshot, previous_snapshot.as_ref()); + let changed_urls = changed_endpoint_urls(&sync_plan.delta); + let remote_should_download = + args.force || sync_plan.should_download() || extended_delta.has_changes(); + let pull_plan = if args.audit_local || remote_should_download || args.plan { + Some(build_pull_plan( + &server_info, + &connection_group, + &app_version, + platforms, + &fetcher, + )?) + } else { + None + }; + let local_audit = if args.audit_local { + Some( + fetcher + .audit_local_manifest( + pull_plan + .as_ref() + .ok_or_else(|| anyhow::anyhow!("local audit requires a pull plan"))?, + ) + .map_err(anyhow::Error::msg)?, + ) + } else { + None + }; + let repair_needed = args.repair + && local_audit + .as_ref() + .map(|audit| !audit.is_clean()) + .unwrap_or(false); + let should_download = remote_should_download || repair_needed; + + println!( + "connection_group={}", + current_snapshot.connection_group_name + ); + println!("app_version={}", current_snapshot.app_version); + println!( + "bundle_version={}", + current_snapshot + .bundle_version + .as_deref() + .unwrap_or("") + ); + println!("addressables_root={}", current_snapshot.addressables_root); + println!("platforms={:?}", platforms); + println!("snapshot_path={}", snapshot_path.display()); + println!( + "previous_snapshot={}", + if previous_snapshot.is_some() { + "present" + } else { + "missing" + } + ); + println!("decision={:?}", sync_plan.decision); + println!("force={}", args.force); + println!("audit_local={}", args.audit_local); + println!("repair={}", args.repair); + println!("should_download={}", should_download); + println!("is_initial={}", sync_plan.delta.is_initial); + println!("changed_endpoint_count={}", changed_urls.len()); + for url in &changed_urls { + println!("changed_endpoint_url={url}"); + } + println!( + "endpoint_markers_changed={}", + extended_delta.endpoint_markers_changed + ); + println!( + "launcher_metadata_changed={}", + extended_delta.launcher_metadata_changed + ); + println!( + "game_main_config_changed={}", + extended_delta.game_main_config_changed + ); + println!( + "addressables_marker_checked_count={}", + current_update_snapshot.addressables_marker_checked_count() + ); + println!( + "unverified_marker_count={}", + current_update_snapshot.unverified_marker_count() + ); + if let Some(bootstrap) = bootstrap.as_ref() { + println!("bootstrap_cache_hit={}", bootstrap.cache_hit); + println!("bootstrap_cache_path={}", bootstrap_cache_path.display()); + } + if let Some(audit) = local_audit.as_ref() { + println!("local_manifest_verified_count={}", audit.verified_count()); + println!( + "local_manifest_repair_needed_count={}", + audit.repair_needed_count() + ); + } else { + println!("local_manifest_verified_count=0"); + println!("local_manifest_repair_needed_count=0"); + } + + if !should_download { + println!("official_seed_hash_verified_count=0"); + println!("update_status=up_to_date"); + return Ok(()); + } + + println!("dry_run={}", args.dry_run); + + if args.dry_run { + if args.plan { + let pull_plan = pull_plan + .as_ref() + .ok_or_else(|| anyhow::anyhow!("dry-run plan requested but no pull plan exists"))?; + let all_urls = pull_plan.all_urls().map_err(anyhow::Error::msg)?; + println!("download_url_count={}", all_urls.len()); + for url in &all_urls { + println!("{url}"); + } + } + println!("official_seed_hash_verified_count=0"); + println!("update_status=would_download"); + return Ok(()); + } + + let pull_plan = pull_plan + .as_ref() + .ok_or_else(|| anyhow::anyhow!("download requested but no pull plan exists"))?; + let report = fetcher.pull(&pull_plan).map_err(anyhow::Error::msg)?; + let final_audit = fetcher + .audit_local_manifest(pull_plan) + .map_err(anyhow::Error::msg)?; + write_snapshot(&snapshot_path, ¤t_update_snapshot)?; + + println!("update_status=downloaded"); + println!("resource_count={}", report.items.len()); + println!("downloaded_count={}", report.downloaded_count()); + println!("resumed_count={}", report.resumed_count()); + println!("skipped_count={}", report.skipped_count()); + println!("final_bytes={}", report.total_bytes()); + println!("transferred_bytes={}", report.transferred_bytes()); + println!( + "official_seed_hash_verified_count={}", + report.official_hash_verified_count() + ); + println!( + "local_manifest_verified_count={}", + final_audit.verified_count() + ); + println!( + "local_manifest_repair_needed_count={}", + final_audit.repair_needed_count() + ); + println!( + "addressables_marker_checked_count={}", + current_update_snapshot.addressables_marker_checked_count() + ); + println!( + "unverified_marker_count={}", + current_update_snapshot.unverified_marker_count() + ); + println!( + "download_manifest={}", + fetcher.download_manifest_path().display() + ); + println!("snapshot_written={}", snapshot_path.display()); + + Ok(()) +} + +fn build_pull_plan( + server_info: &YostarJpServerInfo, + connection_group: &str, + app_version: &str, + platforms: &[PatchPlatform], + fetcher: &OfficialResourcePullService, +) -> anyhow::Result { + let discovery = server_info + .discovery_plan(connection_group, app_version, platforms) + .map_err(anyhow::Error::msg)?; + let seed_catalogs = fetch_seed_catalogs(fetcher, &discovery)?; + let inventory = build_inventory_from_seed_catalogs(&seed_catalogs, platforms)?; + Ok(build_official_pull_plan_for_platform_inventory( + discovery, inventory, platforms, + )) +} + +fn load_server_info( + source: &ServerInfoSource, + fetcher: &OfficialResourcePullService, +) -> anyhow::Result> { + match source { + ServerInfoSource::LocalPath(path) => Ok(fs::read(path)?), + ServerInfoSource::OfficialFile(file_name) => { + let url = server_info_url(file_name).map_err(anyhow::Error::msg)?; + fetcher.fetch_bytes(&url).map_err(anyhow::Error::msg) + } + ServerInfoSource::OfficialUrl(url) => fetcher.fetch_bytes(url).map_err(anyhow::Error::msg), + } +} + +fn default_update_snapshot_version() -> u32 { + OFFICIAL_UPDATE_SNAPSHOT_VERSION +} + +fn default_bootstrap_cache_version() -> u32 { + OFFICIAL_BOOTSTRAP_CACHE_VERSION +} + +fn collect_endpoint_markers( + fetcher: &OfficialResourcePullService, + snapshot: &YostarJpSyncSnapshot, +) -> anyhow::Result> { + let mut markers = Vec::new(); + + for endpoint in &snapshot.endpoints { + let Some(role) = marker_role(endpoint.kind) else { + continue; + }; + let bytes = fetcher + .fetch_bytes(&endpoint.url) + .map_err(anyhow::Error::msg)?; + let value = String::from_utf8_lossy(&bytes).trim().to_string(); + markers.push(OfficialEndpointMarkerSnapshot { + kind: endpoint.kind, + platform: endpoint.platform, + url: endpoint.url.clone(), + role, + value, + }); + } + + Ok(markers) +} + +fn marker_role(kind: YostarJpResourceEndpointKind) -> Option { + match kind { + YostarJpResourceEndpointKind::TableCatalogHash + | YostarJpResourceEndpointKind::BundlePackingInfoHash + | YostarJpResourceEndpointKind::MediaCatalogHash => { + Some(OfficialEndpointMarkerRole::OfficialSeedHash) + } + YostarJpResourceEndpointKind::AddressablesCatalogHash => { + Some(OfficialEndpointMarkerRole::AddressablesCatalogMarker) + } + _ => None, + } +} + +fn diff_extended_snapshot( + current: &OfficialUpdateSnapshot, + previous: Option<&OfficialUpdateSnapshot>, +) -> ExtendedSnapshotDelta { + let Some(previous) = previous else { + return ExtendedSnapshotDelta { + endpoint_markers_changed: true, + launcher_metadata_changed: current.launcher_metadata.is_some(), + game_main_config_changed: current.game_main_config_bootstrap.is_some(), + }; + }; + + ExtendedSnapshotDelta { + endpoint_markers_changed: current.endpoint_markers != previous.endpoint_markers, + launcher_metadata_changed: current.launcher_metadata != previous.launcher_metadata, + game_main_config_changed: current.game_main_config_bootstrap + != previous.game_main_config_bootstrap, + } +} + +fn read_snapshot(path: &Path) -> anyhow::Result> { + if !path.exists() { + return Ok(None); + } + + let data = fs::read(path)?; + match serde_json::from_slice::(&data) { + Ok(snapshot) => Ok(Some(snapshot)), + Err(update_error) => { + let legacy = serde_json::from_slice::(&data).map_err(|legacy_error| { + anyhow::anyhow!( + "failed to parse official update snapshot as v2 ({update_error}) or legacy v1 ({legacy_error})" + ) + })?; + Ok(Some(OfficialUpdateSnapshot::new(legacy, Vec::new(), None))) + } + } +} + +fn write_snapshot(path: &Path, snapshot: &OfficialUpdateSnapshot) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let data = serde_json::to_vec_pretty(snapshot)?; + fs::write(path, data)?; + Ok(()) +} + +fn read_bootstrap_cache(path: &Path) -> anyhow::Result> { + if !path.exists() { + return Ok(None); + } + + let data = fs::read(path)?; + Ok(Some(serde_json::from_slice(&data)?)) +} + +fn write_bootstrap_cache(path: &Path, cache: &OfficialBootstrapCache) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let data = serde_json::to_vec_pretty(cache)?; + fs::write(path, data)?; + Ok(()) +} + +fn launcher_metadata_from_parts( + launcher_version: &str, + game_config: &YostarJpLauncherGameConfig, + manifest_url: &YostarJpLauncherManifestUrl, + manifest: &YostarJpLauncherRemoteManifest, +) -> LauncherMetadataSnapshot { + LauncherMetadataSnapshot { + launcher_version: launcher_version.to_string(), + game_latest_version: game_config.game_latest_version.clone(), + game_latest_file_path: game_config.game_latest_file_path.clone(), + game_lowest_version: game_config.game_lowest_version.clone(), + game_start_exe_name: game_config.game_start_exe_name.clone(), + game_start_params: game_config.game_start_params.clone(), + manifest_url: manifest_url.url.clone(), + manifest_source: manifest.source.clone().filter(|value| !value.is_empty()), + manifest_file_count: manifest.files.len(), + } +} + +fn game_main_config_snapshot(config: &YostarJpGameMainConfig) -> GameMainConfigSnapshot { + GameMainConfigSnapshot { + server_info_data_url: config.server_info_data_url.clone(), + default_connection_group: config.default_connection_group.clone(), + } +} + +fn resolve_bootstrap( + launcher_version: &str, + curl_command: &Path, + cache_path: &Path, + write_cache: bool, +) -> anyhow::Result { + let launcher = OfficialLauncherBootstrapService::with_curl_command( + launcher_version, + curl_command.to_string_lossy().to_string(), + ); + let (game_config, manifest_url, manifest) = launcher + .fetch_latest_remote_manifest() + .map_err(anyhow::Error::msg)?; + let launcher_metadata = + launcher_metadata_from_parts(launcher_version, &game_config, &manifest_url, &manifest); + + if let Some(cache) = read_bootstrap_cache(cache_path)? { + if let Some(game_main_config) = + cached_game_main_config_for_metadata(&cache, &launcher_metadata) + { + return Ok(ResolvedBootstrap { + launcher_metadata, + game_main_config, + cache_hit: true, + }); + } + } + + let bootstrapper = OfficialGameMainConfigBootstrapService::with_commands( + launcher_version, + curl_command.to_path_buf(), + "unzip", + ); + let bootstrap = bootstrapper.fetch_bootstrap().map_err(anyhow::Error::msg)?; + let launcher_metadata = LauncherMetadataSnapshot { + launcher_version: launcher_version.to_string(), + game_latest_version: bootstrap.game_config.game_latest_version.clone(), + game_latest_file_path: bootstrap.game_config.game_latest_file_path.clone(), + game_lowest_version: bootstrap.game_config.game_lowest_version.clone(), + game_start_exe_name: bootstrap.game_config.game_start_exe_name.clone(), + game_start_params: bootstrap.game_config.game_start_params.clone(), + manifest_url: bootstrap.manifest_url.clone(), + manifest_source: bootstrap.manifest_source.clone(), + manifest_file_count: bootstrap.manifest_file_count, + }; + let game_main_config = game_main_config_snapshot(&bootstrap.game_main_config); + if write_cache { + write_bootstrap_cache( + cache_path, + &OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: launcher_metadata.clone(), + game_main_config: game_main_config.clone(), + }, + )?; + } + + Ok(ResolvedBootstrap { + launcher_metadata, + game_main_config, + cache_hit: false, + }) +} + +fn cached_game_main_config_for_metadata( + cache: &OfficialBootstrapCache, + launcher_metadata: &LauncherMetadataSnapshot, +) -> Option { + if cache.cache_version == OFFICIAL_BOOTSTRAP_CACHE_VERSION + && cache.launcher_metadata == *launcher_metadata + { + Some(cache.game_main_config.clone()) + } else { + None + } +} + +fn fetch_seed_catalogs( + fetcher: &OfficialResourcePullService, + discovery: &YostarJpResourceDiscoveryPlan, +) -> anyhow::Result { + let mut catalogs = SeedCatalogs::default(); + + for endpoint in &discovery.endpoints { + if !matches!( + endpoint.kind, + YostarJpResourceEndpointKind::TableCatalog + | YostarJpResourceEndpointKind::BundlePackingInfo + | YostarJpResourceEndpointKind::MediaCatalog + ) { + continue; + } + + let bytes = fetcher + .fetch_bytes(&endpoint.url) + .map_err(anyhow::Error::msg)?; + catalogs.insert(endpoint, bytes)?; + } + + Ok(catalogs) +} + +fn build_inventory_from_seed_catalogs( + catalogs: &SeedCatalogs, + platforms: &[PatchPlatform], +) -> anyhow::Result { + let table_catalog = catalogs + .table_catalog + .as_deref() + .ok_or_else(|| anyhow::anyhow!("official discovery did not include TableCatalog.bytes"))?; + let mut platform_catalogs = Vec::new(); + + for platform in platforms { + let bundle_packing_info = catalogs.bundle_packing_infos.get(platform).ok_or_else(|| { + anyhow::anyhow!( + "official discovery did not include {} BundlePackingInfo.bytes", + platform.as_str() + ) + })?; + let media_catalog = catalogs.media_catalogs.get(platform).ok_or_else(|| { + anyhow::anyhow!( + "official discovery did not include {} MediaCatalog.bytes", + platform.as_str() + ) + })?; + + platform_catalogs.push(YostarJpPlatformCatalogInventory::from_catalog_bytes( + *platform, + bundle_packing_info, + media_catalog, + )); + } + + Ok(YostarJpPlatformDownloadInventory::from_catalog_bytes( + table_catalog, + platform_catalogs, + )) +} + +#[derive(Debug, Default)] +struct SeedCatalogs { + table_catalog: Option>, + bundle_packing_infos: HashMap>, + media_catalogs: HashMap>, +} + +impl SeedCatalogs { + fn insert( + &mut self, + endpoint: &YostarJpResourceEndpoint, + bytes: Vec, + ) -> anyhow::Result<()> { + match endpoint.kind { + YostarJpResourceEndpointKind::TableCatalog => { + self.table_catalog = Some(bytes); + } + YostarJpResourceEndpointKind::BundlePackingInfo => { + let platform = required_platform(endpoint)?; + self.bundle_packing_infos.insert(platform, bytes); + } + YostarJpResourceEndpointKind::MediaCatalog => { + let platform = required_platform(endpoint)?; + self.media_catalogs.insert(platform, bytes); + } + _ => {} + } + + Ok(()) + } +} + +fn required_platform(endpoint: &YostarJpResourceEndpoint) -> anyhow::Result { + endpoint.platform.ok_or_else(|| { + anyhow::anyhow!( + "discovery endpoint {:?} has no platform: {}", + endpoint.kind, + endpoint.url + ) + }) +} + +fn parse_args() -> anyhow::Result { + parse_args_from(env::args()) +} + +fn parse_args_from(raw_args: impl IntoIterator) -> anyhow::Result { + let mut args = raw_args.into_iter(); + let binary = args + .next() + .unwrap_or_else(|| "official_update_check".to_string()); + let mut parsed = CliArgs { + server_info_source: None, + connection_group: None, + app_version: None, + launcher_version: "1.7.2".to_string(), + auto_discover: false, + platforms: None, + output_root: PathBuf::from("official_update_output"), + snapshot_path: None, + curl_command: PathBuf::from("curl"), + dry_run: false, + plan: false, + force: false, + audit_local: true, + repair: true, + }; + + while let Some(flag) = args.next() { + match flag.as_str() { + "--auto-discover" => { + parsed.auto_discover = true; + } + "--server-info-url" => { + parsed.server_info_source = Some(ServerInfoSource::OfficialUrl(next_option_value( + &mut args, &flag, + )?)); + } + "--server-info-file" => { + parsed.server_info_source = Some(ServerInfoSource::OfficialFile( + next_option_value(&mut args, &flag)?, + )); + } + "--server-info-path" => { + parsed.server_info_source = Some(ServerInfoSource::LocalPath(PathBuf::from( + next_option_value(&mut args, &flag)?, + ))); + } + "--connection-group" => { + parsed.connection_group = Some(next_option_value(&mut args, &flag)?); + } + "--app-version" => { + parsed.app_version = Some(next_option_value(&mut args, &flag)?); + } + "--launcher-version" => { + parsed.launcher_version = next_option_value(&mut args, &flag)?; + } + "--platforms" => { + let value = next_option_value(&mut args, &flag)?; + parsed.platforms = Some(parse_platforms(&value).map_err(anyhow::Error::msg)?); + } + "--output" => { + parsed.output_root = PathBuf::from(next_option_value(&mut args, &flag)?); + } + "--snapshot" => { + parsed.snapshot_path = Some(PathBuf::from(next_option_value(&mut args, &flag)?)); + } + "--curl" => { + parsed.curl_command = PathBuf::from(next_option_value(&mut args, &flag)?); + } + "--dry-run" => { + parsed.dry_run = true; + } + "--plan" => { + parsed.plan = true; + } + "--force" => { + parsed.force = true; + } + "--audit-local" => { + parsed.audit_local = true; + } + "--no-audit-local" => { + parsed.audit_local = false; + } + "--repair" => { + parsed.repair = true; + } + "--no-repair" => { + parsed.repair = false; + } + "--help" | "-h" => { + print_usage(&binary); + std::process::exit(0); + } + _ => return Err(anyhow::anyhow!("unknown argument: {flag}")), + } + } + + Ok(parsed) +} + +fn next_option_value( + args: &mut impl Iterator, + flag: &str, +) -> anyhow::Result { + args.next() + .ok_or_else(|| anyhow::anyhow!("missing value for {flag}")) +} + +fn print_usage(binary: &str) { + eprintln!( + "usage: {binary} [--auto-discover | --server-info-url | --server-info-file | --server-info-path ] \ + [--app-version 1.70.0] [--connection-group ] [--launcher-version 1.7.2] \ + [--platforms Windows,Android] [--output official_update_output] [--snapshot official-sync-snapshot.json] \ + [--curl curl] [--dry-run] [--plan] [--force] [--audit-local|--no-audit-local] [--repair|--no-repair]" + ); +} + +fn parse_platforms(value: &str) -> Result, String> { + value + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(parse_platform) + .collect() +} + +fn parse_platform(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "windows" | "win" => Ok(PatchPlatform::Windows), + "android" => Ok(PatchPlatform::Android), + _ => Err(format!("unsupported platform: {value}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(values: &[&str]) -> anyhow::Result { + parse_args_from(values.iter().map(|value| value.to_string())) + } + + fn fixture_base_snapshot() -> YostarJpSyncSnapshot { + YostarJpSyncSnapshot { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: Some("bundle".to_string()), + addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/root".to_string(), + endpoints: vec![ + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.bytes".to_string(), + }, + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalogHash, + platform: None, + url: "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.hash".to_string(), + }, + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::AddressablesCatalogHash, + platform: Some(PatchPlatform::Windows), + url: "https://prod-clientpatch.bluearchiveyostar.com/root/Windows_PatchPack/catalog_StandaloneWindows64.hash".to_string(), + }, + ], + } + } + + fn fixture_marker(value: &str) -> OfficialEndpointMarkerSnapshot { + OfficialEndpointMarkerSnapshot { + kind: YostarJpResourceEndpointKind::TableCatalogHash, + platform: None, + url: + "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.hash" + .to_string(), + role: OfficialEndpointMarkerRole::OfficialSeedHash, + value: value.to_string(), + } + } + + fn fixture_bootstrap() -> ResolvedBootstrap { + ResolvedBootstrap { + launcher_metadata: LauncherMetadataSnapshot { + launcher_version: "1.7.2".to_string(), + game_latest_version: "1.70.0".to_string(), + game_latest_file_path: "BAJP_1.70.0.zip".to_string(), + game_lowest_version: Some("1.69.0".to_string()), + game_start_exe_name: Some("BlueArchive".to_string()), + game_start_params: vec!["--prod".to_string()], + manifest_url: "https://launcher-pkg-ba-jp.yo-star.com/manifest.json".to_string(), + manifest_source: Some("BAJP_1.70.0.zip".to_string()), + manifest_file_count: 42, + }, + game_main_config: GameMainConfigSnapshot { + server_info_data_url: Some( + "https://yostar-serverinfo.bluearchiveyostar.com/prod.json".to_string(), + ), + default_connection_group: Some("Prod-Audit".to_string()), + }, + cache_hit: false, + } + } + + #[test] + fn parses_auto_discover_update_check_args() { + let args = parse(&[ + "official_update_check", + "--auto-discover", + "--platforms", + "Windows,Android", + "--snapshot", + "/tmp/snapshot.json", + "--dry-run", + "--plan", + ]) + .unwrap(); + + assert!(args.auto_discover); + assert_eq!( + args.platforms.as_deref(), + Some([PatchPlatform::Windows, PatchPlatform::Android].as_slice()) + ); + assert_eq!( + args.snapshot_path, + Some(PathBuf::from("/tmp/snapshot.json")) + ); + assert!(args.dry_run); + assert!(args.plan); + assert!(args.audit_local); + assert!(args.repair); + } + + #[test] + fn parses_explicit_snapshot_source_args() { + let args = parse(&[ + "official_update_check", + "--server-info-url", + "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json", + "--connection-group", + "Prod-Audit", + "--app-version", + "1.70.0", + "--no-audit-local", + "--no-repair", + ]) + .unwrap(); + + assert!(!args.auto_discover); + assert_eq!(args.connection_group.as_deref(), Some("Prod-Audit")); + assert_eq!(args.app_version.as_deref(), Some("1.70.0")); + assert!(!args.audit_local); + assert!(!args.repair); + assert!(matches!( + args.server_info_source, + Some(ServerInfoSource::OfficialUrl(ref url)) + if url == "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json" + )); + } + + #[test] + fn persists_snapshot_json_round_trip() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("snapshot.json"); + let bootstrap = fixture_bootstrap(); + let snapshot = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("1234")], + Some(&bootstrap), + ); + + write_snapshot(&path, &snapshot).unwrap(); + assert_eq!(read_snapshot(&path).unwrap(), Some(snapshot)); + } + + #[test] + fn reads_legacy_snapshot_json() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("snapshot.json"); + let legacy = fixture_base_snapshot(); + fs::write(&path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + let snapshot = read_snapshot(&path).unwrap().unwrap(); + assert_eq!(snapshot.base_snapshot(), legacy); + assert!(snapshot.endpoint_markers.is_empty()); + assert_eq!(snapshot.snapshot_version, OFFICIAL_UPDATE_SNAPSHOT_VERSION); + } + + #[test] + fn marker_content_change_requires_download() { + let previous = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("1111")], + Some(&fixture_bootstrap()), + ); + let current = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("2222")], + Some(&fixture_bootstrap()), + ); + + let delta = diff_extended_snapshot(¤t, Some(&previous)); + assert!(delta.endpoint_markers_changed); + assert!(delta.has_changes()); + assert!(!delta.launcher_metadata_changed); + assert!(!delta.game_main_config_changed); + } + + #[test] + fn persists_bootstrap_cache_json_round_trip() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("official-bootstrap-cache.json"); + let bootstrap = fixture_bootstrap(); + let cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: bootstrap.launcher_metadata, + game_main_config: bootstrap.game_main_config, + }; + + write_bootstrap_cache(&path, &cache).unwrap(); + assert_eq!(read_bootstrap_cache(&path).unwrap(), Some(cache)); + } + + #[test] + fn bootstrap_cache_hit_requires_same_metadata_and_version() { + let bootstrap = fixture_bootstrap(); + let cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: bootstrap.launcher_metadata.clone(), + game_main_config: bootstrap.game_main_config.clone(), + }; + + assert_eq!( + cached_game_main_config_for_metadata(&cache, &bootstrap.launcher_metadata), + Some(bootstrap.game_main_config.clone()) + ); + + let mut changed_metadata = bootstrap.launcher_metadata.clone(); + changed_metadata.game_latest_version = "1.71.0".to_string(); + assert_eq!( + cached_game_main_config_for_metadata(&cache, &changed_metadata), + None + ); + + let stale_version_cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION + 1, + ..cache + }; + assert_eq!( + cached_game_main_config_for_metadata( + &stale_version_cache, + &bootstrap.launcher_metadata + ), + None + ); + } +} diff --git a/infrastructure/src/bin/bat_official_sync.rs b/infrastructure/src/bin/bat_official_sync.rs new file mode 100644 index 0000000..50157e4 --- /dev/null +++ b/infrastructure/src/bin/bat_official_sync.rs @@ -0,0 +1,447 @@ +use bat_adapters::official::yostar_jp::PatchPlatform; +use bat_infrastructure::{ + OfficialServerInfoSource, OfficialUpdateConfig, OfficialUpdateService, OfficialUpdateStatus, +}; +use serde::Serialize; +use std::env; +use std::path::PathBuf; +use std::thread; +use std::time::Duration; + +const EXIT_ERROR: i32 = 1; +const EXIT_LOCKED: i32 = 75; +const DEFAULT_WATCH_INTERVAL_SECONDS: u64 = 60 * 60; + +fn main() { + match run() { + Ok(()) => {} + Err(error) => { + let exit_code = if error.to_string().contains("locked") { + EXIT_LOCKED + } else { + EXIT_ERROR + }; + let payload = ErrorReport { + status: "error", + exit_code, + error: error.to_string(), + next_retry_seconds: None, + }; + eprintln!( + "{}", + serde_json::to_string_pretty(&payload) + .unwrap_or_else(|_| "{\"status\":\"error\"}".to_string()) + ); + std::process::exit(exit_code); + } + } +} + +fn run() -> anyhow::Result<()> { + let options = parse_args()?; + if options.watch { + run_watch(options)?; + } else { + let report = OfficialUpdateService::new().run(&options.config)?; + if should_print_status(report.update_status, options.quiet_up_to_date) { + println!("{}", serde_json::to_string_pretty(&report)?); + } + } + Ok(()) +} + +#[derive(Debug, Serialize)] +struct ErrorReport<'a> { + status: &'a str, + exit_code: i32, + error: String, + #[serde(skip_serializing_if = "Option::is_none")] + next_retry_seconds: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CliOptions { + config: OfficialUpdateConfig, + watch: bool, + interval: Duration, + quiet_up_to_date: bool, + quiet_up_to_date_explicit: bool, +} + +impl Default for CliOptions { + fn default() -> Self { + Self { + config: OfficialUpdateConfig::default(), + watch: false, + interval: Duration::from_secs(DEFAULT_WATCH_INTERVAL_SECONDS), + quiet_up_to_date: false, + quiet_up_to_date_explicit: false, + } + } +} + +fn run_watch(options: CliOptions) -> anyhow::Result<()> { + if options.config.dry_run { + return Err(anyhow::anyhow!("watch mode cannot be used with --dry-run")); + } + if options.interval.is_zero() { + return Err(anyhow::anyhow!("watch interval must be greater than zero")); + } + let service = OfficialUpdateService::new(); + loop { + match service.run(&options.config) { + Ok(report) => { + if should_print_status(report.update_status, options.quiet_up_to_date) { + println!("{}", serde_json::to_string_pretty(&report)?); + } + } + Err(error) => { + let payload = ErrorReport { + status: "error", + exit_code: EXIT_ERROR, + error: error.to_string(), + next_retry_seconds: Some(options.interval.as_secs()), + }; + eprintln!("{}", serde_json::to_string_pretty(&payload)?); + } + } + + thread::sleep(options.interval); + } +} + +fn should_print_status(status: OfficialUpdateStatus, quiet_up_to_date: bool) -> bool { + !(quiet_up_to_date && status == OfficialUpdateStatus::UpToDate) +} + +fn parse_args() -> anyhow::Result { + parse_args_from(env::args()) +} + +fn parse_args_from(raw_args: impl IntoIterator) -> anyhow::Result { + let mut args = raw_args.into_iter(); + let binary = args + .next() + .unwrap_or_else(|| "bat-official-sync".to_string()); + let mut options = CliOptions::default(); + + while let Some(flag) = args.next() { + match flag.as_str() { + "--auto-discover" => { + options.config.auto_discover = true; + } + "--server-info-url" => { + options.config.server_info_source = Some(OfficialServerInfoSource::OfficialUrl( + next_option_value(&mut args, &flag)?, + )); + } + "--server-info-file" => { + options.config.server_info_source = Some(OfficialServerInfoSource::OfficialFile( + next_option_value(&mut args, &flag)?, + )); + } + "--server-info-path" => { + options.config.server_info_source = Some(OfficialServerInfoSource::LocalPath( + PathBuf::from(next_option_value(&mut args, &flag)?), + )); + } + "--connection-group" => { + options.config.connection_group = Some(next_option_value(&mut args, &flag)?); + } + "--app-version" => { + options.config.app_version = Some(next_option_value(&mut args, &flag)?); + } + "--launcher-version" => { + options.config.launcher_version = next_option_value(&mut args, &flag)?; + } + "--platforms" => { + let value = next_option_value(&mut args, &flag)?; + options.config.platforms = + Some(parse_platforms(&value).map_err(anyhow::Error::msg)?); + } + "--output" => { + options.config.output_root = PathBuf::from(next_option_value(&mut args, &flag)?); + } + "--snapshot" => { + options.config.snapshot_path = + Some(PathBuf::from(next_option_value(&mut args, &flag)?)); + } + "--curl" => { + options.config.curl_command = PathBuf::from(next_option_value(&mut args, &flag)?); + } + "--unzip" => { + options.config.unzip_command = PathBuf::from(next_option_value(&mut args, &flag)?); + } + "--dry-run" => { + options.config.dry_run = true; + } + "--plan" => { + options.config.plan = true; + } + "--force" => { + options.config.force = true; + } + "--audit-local" => { + options.config.audit_local = true; + } + "--no-audit-local" => { + options.config.audit_local = false; + } + "--repair" => { + options.config.repair = true; + } + "--no-repair" => { + options.config.repair = false; + } + "--watch" => { + options.watch = true; + } + "--interval" => { + options.interval = parse_duration(&next_option_value(&mut args, &flag)?)?; + } + "--interval-seconds" => { + let seconds = next_option_value(&mut args, &flag)? + .parse::() + .map_err(|error| anyhow::anyhow!("invalid seconds for {flag}: {error}"))?; + options.interval = Duration::from_secs(seconds); + } + "--quiet-up-to-date" => { + options.quiet_up_to_date = true; + options.quiet_up_to_date_explicit = true; + } + "--no-quiet-up-to-date" => { + options.quiet_up_to_date = false; + options.quiet_up_to_date_explicit = true; + } + "--help" | "-h" => { + print_usage(&binary); + std::process::exit(0); + } + _ => return Err(anyhow::anyhow!("unknown argument: {flag}")), + } + } + + if options.watch && options.config.dry_run { + return Err(anyhow::anyhow!("watch mode cannot be used with --dry-run")); + } + if options.interval.is_zero() { + return Err(anyhow::anyhow!("watch interval must be greater than zero")); + } + if options.watch && !options.quiet_up_to_date_explicit { + options.quiet_up_to_date = true; + } + + Ok(options) +} + +fn next_option_value( + args: &mut impl Iterator, + flag: &str, +) -> anyhow::Result { + args.next() + .ok_or_else(|| anyhow::anyhow!("missing value for {flag}")) +} + +fn print_usage(binary: &str) { + eprintln!( + "usage: {binary} [--auto-discover | --server-info-url | --server-info-file | --server-info-path ] \ + [--app-version 1.70.0] [--connection-group ] [--launcher-version 1.7.2] \ + [--platforms Windows,Android] [--output official_update_output] [--snapshot official-sync-snapshot.json] \ + [--curl curl] [--unzip unzip] [--dry-run] [--plan] [--force] [--audit-local|--no-audit-local] [--repair|--no-repair] \ + [--watch] [--interval 1h|60m|3600s] [--quiet-up-to-date|--no-quiet-up-to-date]" + ); +} + +fn parse_duration(value: &str) -> anyhow::Result { + let value = value.trim(); + if value.is_empty() { + return Err(anyhow::anyhow!("duration must not be empty")); + } + + let (number, multiplier) = if let Some(number) = value.strip_suffix("ms") { + let millis = number + .parse::() + .map_err(|error| anyhow::anyhow!("invalid millisecond duration {value}: {error}"))?; + return Ok(Duration::from_millis(millis)); + } else if let Some(number) = value.strip_suffix('s') { + (number, 1) + } else if let Some(number) = value.strip_suffix('m') { + (number, 60) + } else if let Some(number) = value.strip_suffix('h') { + (number, 60 * 60) + } else { + (value, 1) + }; + + let amount = number + .parse::() + .map_err(|error| anyhow::anyhow!("invalid duration {value}: {error}"))?; + Ok(Duration::from_secs(amount.saturating_mul(multiplier))) +} + +fn parse_platforms(value: &str) -> Result, String> { + value + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(parse_platform) + .collect() +} + +fn parse_platform(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "windows" | "win" => Ok(PatchPlatform::Windows), + "android" => Ok(PatchPlatform::Android), + _ => Err(format!("unsupported platform: {value}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(values: &[&str]) -> anyhow::Result { + parse_args_from(values.iter().map(|value| value.to_string())) + } + + #[test] + fn parses_auto_discover_sync_args() { + let options = parse(&[ + "bat-official-sync", + "--auto-discover", + "--platforms", + "Windows,Android", + "--snapshot", + "/tmp/snapshot.json", + "--dry-run", + "--plan", + ]) + .unwrap(); + let config = options.config; + + assert!(config.auto_discover); + assert_eq!( + config.platforms.as_deref(), + Some([PatchPlatform::Windows, PatchPlatform::Android].as_slice()) + ); + assert_eq!( + config.snapshot_path, + Some(PathBuf::from("/tmp/snapshot.json")) + ); + assert!(config.dry_run); + assert!(config.plan); + assert!(config.audit_local); + assert!(config.repair); + assert!(!options.watch); + assert_eq!( + options.interval, + Duration::from_secs(DEFAULT_WATCH_INTERVAL_SECONDS) + ); + assert!(!options.quiet_up_to_date); + assert!(!options.quiet_up_to_date_explicit); + } + + #[test] + fn parses_explicit_source_and_disable_repair() { + let options = parse(&[ + "bat-official-sync", + "--server-info-url", + "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json", + "--connection-group", + "Prod-Audit", + "--app-version", + "1.70.0", + "--no-audit-local", + "--no-repair", + "--unzip", + "/usr/bin/unzip", + ]) + .unwrap(); + let config = options.config; + + assert!(!config.auto_discover); + assert_eq!(config.connection_group.as_deref(), Some("Prod-Audit")); + assert_eq!(config.app_version.as_deref(), Some("1.70.0")); + assert!(!config.audit_local); + assert!(!config.repair); + assert_eq!(config.unzip_command, PathBuf::from("/usr/bin/unzip")); + assert!(matches!( + config.server_info_source, + Some(OfficialServerInfoSource::OfficialUrl(ref url)) + if url == "https://yostar-serverinfo.bluearchiveyostar.com/r93_fixture.json" + )); + } + + #[test] + fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() { + let options = parse(&[ + "bat-official-sync", + "--auto-discover", + "--watch", + "--interval", + "30m", + ]) + .unwrap(); + + assert!(options.watch); + assert_eq!(options.interval, Duration::from_secs(30 * 60)); + assert!(options.quiet_up_to_date); + assert!(!options.quiet_up_to_date_explicit); + } + + #[test] + fn explicit_no_quiet_up_to_date_overrides_watch_default() { + let options = parse(&[ + "bat-official-sync", + "--watch", + "--no-quiet-up-to-date", + "--interval", + "1h", + ]) + .unwrap(); + + assert!(options.watch); + assert!(!options.quiet_up_to_date); + assert!(options.quiet_up_to_date_explicit); + } + + #[test] + fn rejects_watch_dry_run_and_zero_interval() { + let error = parse(&["bat-official-sync", "--watch", "--dry-run"]).unwrap_err(); + assert!(error.to_string().contains("watch mode")); + + let error = + parse(&["bat-official-sync", "--watch", "--interval-seconds", "0"]).unwrap_err(); + assert!(error.to_string().contains("greater than zero")); + } + + #[test] + fn parses_duration_units() { + assert_eq!(parse_duration("1h").unwrap(), Duration::from_secs(3600)); + assert_eq!(parse_duration("60m").unwrap(), Duration::from_secs(3600)); + assert_eq!(parse_duration("90s").unwrap(), Duration::from_secs(90)); + assert_eq!(parse_duration("250ms").unwrap(), Duration::from_millis(250)); + assert_eq!(parse_duration("3600").unwrap(), Duration::from_secs(3600)); + } + + #[test] + fn quiet_up_to_date_suppresses_only_clean_reports() { + assert!(!should_print_status(OfficialUpdateStatus::UpToDate, true)); + assert!(should_print_status(OfficialUpdateStatus::Downloaded, true)); + assert!(should_print_status( + OfficialUpdateStatus::WouldDownload, + true + )); + assert!(should_print_status(OfficialUpdateStatus::UpToDate, false)); + } + + #[test] + fn status_string_is_stable() { + assert_eq!(OfficialUpdateStatus::UpToDate.as_str(), "up_to_date"); + assert_eq!( + OfficialUpdateStatus::WouldDownload.as_str(), + "would_download" + ); + assert_eq!(OfficialUpdateStatus::Downloaded.as_str(), "downloaded"); + } +} diff --git a/infrastructure/src/import.rs b/infrastructure/src/import.rs index fe18a83..c774ab0 100644 --- a/infrastructure/src/import.rs +++ b/infrastructure/src/import.rs @@ -276,12 +276,16 @@ mod tests { hash: "synthetic-manifest-hash".to_string(), size: 99, resource_type: ResourceType::AssetBundle, + address: None, + dependencies: Vec::new(), }, ResourceEntry { path: "synthetic/catalog.json".to_string(), hash: "synthetic-catalog-hash".to_string(), size: 1, resource_type: ResourceType::Manifest, + address: None, + dependencies: Vec::new(), }, ], metadata: ManifestMetadata { diff --git a/infrastructure/src/lib.rs b/infrastructure/src/lib.rs index 12eb32e..103abd6 100644 --- a/infrastructure/src/lib.rs +++ b/infrastructure/src/lib.rs @@ -12,17 +12,20 @@ pub mod cas; pub mod import; -pub mod official_game_main_config; pub mod official_download; +pub mod official_game_main_config; pub mod official_launcher; pub mod official_pull; pub mod official_sync; +pub mod official_update; pub mod resources; pub use cas::FileSystemCasRepository; pub use import::{BundleSource, ImportedResource, ResourceImportReport, ResourceImportService}; pub use official_download::{ - OfficialResourcePullItem, OfficialResourcePullReport, OfficialResourcePullService, + OfficialLocalManifestAuditItem, OfficialLocalManifestAuditReport, + OfficialLocalManifestAuditStatus, OfficialResourcePullItem, OfficialResourcePullReport, + OfficialResourcePullService, }; pub use official_game_main_config::OfficialGameMainConfigBootstrapService; pub use official_launcher::{ @@ -39,7 +42,15 @@ pub use official_sync::{ build_official_sync_plan, changed_endpoint_urls, classify_sync_decision, default_official_platforms, OfficialSyncDecision, OfficialSyncPlan, }; -pub use resources::InMemoryResourceRepository; +pub use official_update::{ + cached_game_main_config_for_metadata, diff_extended_snapshot, read_bootstrap_cache, + read_snapshot, write_bootstrap_cache, write_snapshot, ExtendedSnapshotDelta, + GameMainConfigSnapshot, LauncherMetadataSnapshot, OfficialBootstrapCache, + OfficialEndpointMarkerRole, OfficialEndpointMarkerSnapshot, OfficialServerInfoSource, + OfficialUpdateConfig, OfficialUpdateReport, OfficialUpdateService, OfficialUpdateSnapshot, + OfficialUpdateStatus, ResolvedBootstrap, +}; +pub use resources::{InMemoryResourceRepository, SqliteResourceRepository}; /// Infrastructure 版本号 pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/infrastructure/src/official_download.rs b/infrastructure/src/official_download.rs index bca81a3..bee1ef1 100644 --- a/infrastructure/src/official_download.rs +++ b/infrastructure/src/official_download.rs @@ -1,11 +1,60 @@ //! Official JP resource download execution. use crate::official_pull::OfficialResourcePullPlan; -use bat_adapters::official::yostar_jp::is_official_yostar_jp_url; -use std::fs; +use bat_adapters::official::yostar_jp::{is_official_yostar_jp_url, YostarJpResourceEndpointKind}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashSet}; +use std::fs::{self, File}; +use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; +const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json"; +const DOWNLOAD_MANIFEST_VERSION: u32 = 1; +const DEFAULT_RETRY_ATTEMPTS: usize = 3; + +/// Outcome for one official resource pull item. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OfficialResourcePullStatus { + /// The destination already existed and passed local manifest validation. + SkippedExisting, + /// A partial `.part` file was resumed. + Resumed, + /// The resource was downloaded from scratch. + Downloaded, +} + +/// Official hash algorithm used by a verified resource sidecar. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OfficialResourceHashAlgorithm { + /// Decimal text form of `xxHash32` with seed `0`. + XxHash32Decimal, +} + +impl OfficialResourceHashAlgorithm { + /// Returns a stable algorithm label for reports and logs. + pub fn as_str(self) -> &'static str { + match self { + Self::XxHash32Decimal => "xxhash32_decimal", + } + } +} + +/// Successful official hash verification for one downloaded resource. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OfficialResourceHashVerification { + /// Downloaded data URL that was verified. + pub data_url: String, + /// Official `.hash` URL used as the verification source. + pub hash_url: String, + /// Official hash algorithm. + pub algorithm: OfficialResourceHashAlgorithm, + /// Expected hash value read from the official `.hash` file. + pub expected: String, + /// Actual hash value computed from the downloaded data file. + pub actual: String, +} + /// One downloaded official resource. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialResourcePullItem { @@ -13,8 +62,12 @@ pub struct OfficialResourcePullItem { pub url: String, /// Local destination path under the output root. pub destination: PathBuf, - /// Downloaded byte count. + /// Final local file size. pub bytes: u64, + /// Bytes transferred during this pull execution. + pub transferred_bytes: u64, + /// Pull outcome. + pub status: OfficialResourcePullStatus, } /// Download report for one executed pull plan. @@ -22,6 +75,8 @@ pub struct OfficialResourcePullItem { pub struct OfficialResourcePullReport { /// Downloaded resources in execution order. pub items: Vec, + /// Official hash sidecars successfully verified after download. + pub verified_hashes: Vec, } impl OfficialResourcePullReport { @@ -29,6 +84,114 @@ impl OfficialResourcePullReport { pub fn total_bytes(&self) -> u64 { self.items.iter().map(|item| item.bytes).sum() } + + /// Returns bytes transferred by this pull execution. + pub fn transferred_bytes(&self) -> u64 { + self.items.iter().map(|item| item.transferred_bytes).sum() + } + + /// Returns the number of resources reused from the local output tree. + pub fn skipped_count(&self) -> usize { + self.items + .iter() + .filter(|item| item.status == OfficialResourcePullStatus::SkippedExisting) + .count() + } + + /// Returns the number of resources resumed from `.part` files. + pub fn resumed_count(&self) -> usize { + self.items + .iter() + .filter(|item| item.status == OfficialResourcePullStatus::Resumed) + .count() + } + + /// Returns the number of resources downloaded from scratch. + pub fn downloaded_count(&self) -> usize { + self.items + .iter() + .filter(|item| item.status == OfficialResourcePullStatus::Downloaded) + .count() + } + + /// Returns the number of official hash sidecars verified after download. + pub fn official_hash_verified_count(&self) -> usize { + self.verified_hashes.len() + } +} + +/// Local download-manifest audit status for one expected official URL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OfficialLocalManifestAuditStatus { + /// The local file exists and matches the recorded manifest size and BLAKE3. + Verified, + /// No manifest entry exists for this URL. + MissingManifestEntry, + /// The manifest entry URL does not match the expected URL. + UrlMismatch, + /// The manifest destination does not match the URL-derived destination. + DestinationMismatch, + /// The manifest entry exists but the local file is missing. + MissingFile, + /// The local file size differs from the manifest entry. + SizeMismatch, + /// The local file BLAKE3 differs from the manifest entry. + Blake3Mismatch, +} + +impl OfficialLocalManifestAuditStatus { + /// Returns true when this status proves the local file can be reused. + pub fn is_verified(self) -> bool { + self == Self::Verified + } +} + +/// Local download-manifest audit result for one expected official URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OfficialLocalManifestAuditItem { + /// Expected official URL from the pull plan. + pub url: String, + /// URL-derived local destination. + pub destination: PathBuf, + /// Audit status. + pub status: OfficialLocalManifestAuditStatus, + /// Manifest-recorded destination, when an entry exists. + pub manifest_destination: Option, + /// Manifest-recorded byte size, when an entry exists. + pub expected_bytes: Option, + /// Actual local byte size, when the destination exists. + pub actual_bytes: Option, + /// Manifest-recorded BLAKE3, when an entry exists. + pub expected_blake3: Option, + /// Actual local BLAKE3, when it could be computed. + pub actual_blake3: Option, +} + +/// Local download-manifest audit result for a pull plan. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OfficialLocalManifestAuditReport { + /// Per-URL audit items in pull-plan order. + pub items: Vec, +} + +impl OfficialLocalManifestAuditReport { + /// Returns true when every expected URL was locally verified. + pub fn is_clean(&self) -> bool { + self.items.iter().all(|item| item.status.is_verified()) + } + + /// Returns the number of locally verified files. + pub fn verified_count(&self) -> usize { + self.items + .iter() + .filter(|item| item.status.is_verified()) + .count() + } + + /// Returns the number of files that require repair. + pub fn repair_needed_count(&self) -> usize { + self.items.len().saturating_sub(self.verified_count()) + } } /// Executes an official pull plan with the system `curl` command. @@ -36,6 +199,7 @@ impl OfficialResourcePullReport { pub struct OfficialResourcePullService { output_root: PathBuf, curl_command: PathBuf, + retry_attempts: usize, } impl OfficialResourcePullService { @@ -52,14 +216,26 @@ impl OfficialResourcePullService { Self { output_root: output_root.into(), curl_command: curl_command.into(), + retry_attempts: DEFAULT_RETRY_ATTEMPTS, } } + /// Sets the number of attempts for each `curl` transfer. + pub fn with_retry_attempts(mut self, retry_attempts: usize) -> Self { + self.retry_attempts = retry_attempts.max(1); + self + } + /// Returns the output root used for downloaded files. pub fn output_root(&self) -> &Path { &self.output_root } + /// Returns the local manifest path used to validate completed downloads. + pub fn download_manifest_path(&self) -> PathBuf { + self.output_root.join(DOWNLOAD_MANIFEST_FILE) + } + /// Executes the plan and downloads every official URL to disk. pub fn pull( &self, @@ -72,6 +248,9 @@ impl OfficialResourcePullService { ) })?; + let official_hash_pairs = official_seed_hash_pairs(plan); + let force_refresh_urls = official_hash_refresh_urls(&official_hash_pairs); + let mut manifest = self.read_download_manifest()?; let mut items = Vec::new(); for url in plan.all_urls()? { if !is_official_yostar_jp_url(&url) { @@ -88,24 +267,56 @@ impl OfficialResourcePullService { })?; } - self.download_one(&url, &destination)?; - let bytes = fs::metadata(&destination) - .map_err(|error| { - format!( - "Downloaded file missing at {}: {error}", - destination.display() - ) - })? - .len(); + let force_refresh = force_refresh_urls.contains(&url); + let result = if force_refresh { + None + } else { + self.validated_existing_file(&url, &destination, &manifest)? + }; + let result = if let Some(result) = result { + result + } else { + let result = self.pull_one(&url, &destination)?; + self.record_download_manifest_entry(&mut manifest, &url, &destination)?; + self.write_download_manifest(&manifest)?; + result + }; items.push(OfficialResourcePullItem { url, destination, - bytes, + bytes: result.bytes, + transferred_bytes: result.transferred_bytes, + status: result.status, }); } - Ok(OfficialResourcePullReport { items }) + let verified_hashes = self.verify_downloaded_official_hashes(&official_hash_pairs)?; + + Ok(OfficialResourcePullReport { + items, + verified_hashes, + }) + } + + /// Audits every URL in a pull plan against the local download manifest. + /// + /// This performs no network I/O. It checks that each URL has a manifest + /// entry, that the entry maps to the URL-derived destination, and that the + /// local file still matches the recorded size and BLAKE3 digest. + pub fn audit_local_manifest( + &self, + plan: &OfficialResourcePullPlan, + ) -> Result { + let manifest = self.read_download_manifest()?; + let mut items = Vec::new(); + + for url in plan.all_urls()? { + let destination = self.destination_for_url(&url)?; + items.push(self.audit_one(&url, destination, &manifest)?); + } + + Ok(OfficialLocalManifestAuditReport { items }) } /// Fetches an official URL into memory. @@ -141,23 +352,102 @@ impl OfficialResourcePullService { Ok(output.stdout) } - fn download_one(&self, url: &str, destination: &Path) -> Result<(), String> { - let output = Command::new(&self.curl_command) + fn pull_one(&self, url: &str, destination: &Path) -> Result { + let partial = partial_path_for(destination); + let partial_len_before = file_len_if_exists(&partial)?.unwrap_or_default(); + let resumed = partial_len_before > 0; + + if resumed { + if let Err(error) = self.download_one(url, &partial, true) { + let _ = fs::remove_file(&partial); + self.download_one(url, &partial, false) + .map_err(|retry_error| { + format!( + "resume failed for {url}: {error}; clean retry also failed: {retry_error}" + ) + })?; + } + } else { + self.download_one(url, &partial, false)?; + } + + if file_len_if_exists(destination)?.is_some() { + fs::remove_file(destination).map_err(|error| { + format!( + "Failed to replace existing destination {}: {error}", + destination.display() + ) + })?; + } + + fs::rename(&partial, destination).map_err(|error| { + format!( + "Failed to move partial download {} -> {}: {error}", + partial.display(), + destination.display() + ) + })?; + + let bytes = fs::metadata(destination) + .map_err(|error| { + format!( + "Downloaded file missing at {}: {error}", + destination.display() + ) + })? + .len(); + + Ok(PullOneResult { + bytes, + transferred_bytes: bytes.saturating_sub(partial_len_before), + status: if resumed { + OfficialResourcePullStatus::Resumed + } else { + OfficialResourcePullStatus::Downloaded + }, + }) + } + + fn download_one(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> { + let attempts = self.retry_attempts.max(1); + let mut last_error = None; + for attempt in 1..=attempts { + match self.download_one_once(url, destination, resume) { + Ok(()) => return Ok(()), + Err(error) => { + last_error = Some(format!("attempt {attempt}/{attempts}: {error}")); + } + } + } + + Err(last_error.unwrap_or_else(|| { + format!( + "curl failed for {url} -> {} without an attempt", + destination.display() + ) + })) + } + + fn download_one_once(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> { + let mut command = Command::new(&self.curl_command); + command .arg("--fail") .arg("--location") .arg("--silent") .arg("--show-error") .arg("--output") - .arg(destination) - .arg("--url") - .arg(url) - .output() - .map_err(|error| { - format!( - "Failed to launch curl command {}: {error}", - self.curl_command.display() - ) - })?; + .arg(destination); + if resume { + command.arg("--continue-at").arg("-"); + } + command.arg("--url").arg(url); + + let output = command.output().map_err(|error| { + format!( + "Failed to launch curl command {}: {error}", + self.curl_command.display() + ) + })?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -171,6 +461,293 @@ impl OfficialResourcePullService { Ok(()) } + fn verify_downloaded_official_hashes( + &self, + pairs: &[OfficialSeedHashPair], + ) -> Result, String> { + let mut verified = Vec::new(); + + for pair in pairs { + let data_path = self.destination_for_url(&pair.data_url)?; + let hash_path = self.destination_for_url(&pair.hash_url)?; + let data = fs::read(&data_path).map_err(|error| { + format!( + "Failed to read official hash data file {}: {error}", + data_path.display() + ) + })?; + let hash = fs::read(&hash_path).map_err(|error| { + format!( + "Failed to read official hash sidecar {}: {error}", + hash_path.display() + ) + })?; + + verified.push(verify_official_seed_catalog_hash( + &pair.data_url, + &data, + &pair.hash_url, + &hash, + )?); + } + + Ok(verified) + } + + fn read_download_manifest(&self) -> Result { + let path = self.download_manifest_path(); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(OfficialDownloadManifest::default()); + } + Err(error) => { + return Err(format!( + "Failed to read download manifest {}: {error}", + path.display() + )); + } + }; + + let manifest: OfficialDownloadManifest = + serde_json::from_slice(&bytes).map_err(|error| { + format!( + "Failed to parse download manifest {}: {error}", + path.display() + ) + })?; + + if manifest.version != DOWNLOAD_MANIFEST_VERSION { + return Err(format!( + "Unsupported download manifest version {} in {}", + manifest.version, + path.display() + )); + } + + Ok(manifest) + } + + fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> { + let path = self.download_manifest_path(); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|error| { + format!( + "Failed to create download manifest directory {}: {error}", + parent.display() + ) + })?; + } + + let temporary = path.with_extension("json.tmp"); + let bytes = serde_json::to_vec_pretty(manifest).map_err(|error| { + format!( + "Failed to serialize download manifest {}: {error}", + path.display() + ) + })?; + fs::write(&temporary, bytes).map_err(|error| { + format!( + "Failed to write temporary download manifest {}: {error}", + temporary.display() + ) + })?; + fs::rename(&temporary, &path).map_err(|error| { + format!( + "Failed to move download manifest {} -> {}: {error}", + temporary.display(), + path.display() + ) + })?; + + Ok(()) + } + + fn validated_existing_file( + &self, + url: &str, + destination: &Path, + manifest: &OfficialDownloadManifest, + ) -> Result, String> { + let Some(entry) = manifest.entries.get(url) else { + return Ok(None); + }; + + if entry.url != url { + return Ok(None); + } + + if entry.destination != self.relative_destination(destination)? { + return Ok(None); + } + + let Some(bytes) = file_len_if_exists(destination)? else { + return Ok(None); + }; + + if bytes != entry.bytes { + return Ok(None); + } + + let digest = blake3_file_hex(destination)?; + if digest != entry.blake3 { + return Ok(None); + } + + Ok(Some(PullOneResult { + bytes, + transferred_bytes: 0, + status: OfficialResourcePullStatus::SkippedExisting, + })) + } + + fn record_download_manifest_entry( + &self, + manifest: &mut OfficialDownloadManifest, + url: &str, + destination: &Path, + ) -> Result<(), String> { + let bytes = fs::metadata(destination) + .map_err(|error| { + format!( + "Failed to inspect downloaded file {}: {error}", + destination.display() + ) + })? + .len(); + let digest = blake3_file_hex(destination)?; + let relative_destination = self.relative_destination(destination)?; + + manifest.entries.insert( + url.to_string(), + OfficialDownloadManifestEntry { + url: url.to_string(), + destination: relative_destination, + bytes, + blake3: digest, + }, + ); + + Ok(()) + } + + fn relative_destination(&self, destination: &Path) -> Result { + let relative = destination + .strip_prefix(&self.output_root) + .map_err(|error| { + format!( + "Destination {} is not under output root {}: {error}", + destination.display(), + self.output_root.display() + ) + })?; + + Ok(relative.to_string_lossy().replace('\\', "/")) + } + + fn audit_one( + &self, + url: &str, + destination: PathBuf, + manifest: &OfficialDownloadManifest, + ) -> Result { + let Some(entry) = manifest.entries.get(url) else { + let actual_bytes = file_len_if_exists(&destination)?; + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::MissingManifestEntry, + manifest_destination: None, + expected_bytes: None, + actual_bytes, + expected_blake3: None, + actual_blake3: None, + }); + }; + + let manifest_destination = Some(entry.destination.clone()); + let expected_bytes = Some(entry.bytes); + let expected_blake3 = Some(entry.blake3.clone()); + + if entry.url != url { + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::UrlMismatch, + manifest_destination, + expected_bytes, + actual_bytes: None, + expected_blake3, + actual_blake3: None, + }); + } + + let expected_destination = self.relative_destination(&destination)?; + if entry.destination != expected_destination { + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::DestinationMismatch, + manifest_destination, + expected_bytes, + actual_bytes: None, + expected_blake3, + actual_blake3: None, + }); + } + + let Some(actual_bytes) = file_len_if_exists(&destination)? else { + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::MissingFile, + manifest_destination, + expected_bytes, + actual_bytes: None, + expected_blake3, + actual_blake3: None, + }); + }; + + if actual_bytes != entry.bytes { + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::SizeMismatch, + manifest_destination, + expected_bytes, + actual_bytes: Some(actual_bytes), + expected_blake3, + actual_blake3: None, + }); + } + + let actual_blake3 = blake3_file_hex(&destination)?; + if actual_blake3 != entry.blake3 { + return Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::Blake3Mismatch, + manifest_destination, + expected_bytes, + actual_bytes: Some(actual_bytes), + expected_blake3, + actual_blake3: Some(actual_blake3), + }); + } + + Ok(OfficialLocalManifestAuditItem { + url: url.to_string(), + destination, + status: OfficialLocalManifestAuditStatus::Verified, + manifest_destination, + expected_bytes, + actual_bytes: Some(actual_bytes), + expected_blake3, + actual_blake3: Some(actual_blake3), + }) + } + fn destination_for_url(&self, url: &str) -> Result { if !is_official_yostar_jp_url(url) { return Err(format!("URL is not an official JP host: {url}")); @@ -203,6 +780,250 @@ impl OfficialResourcePullService { } } +/// Verifies an official seed catalog `.bytes` payload against its official +/// `.hash` sidecar. +/// +/// The currently verified seed catalogs are `TableCatalog.bytes`, +/// `BundlePackingInfo.bytes`, and `MediaCatalog.bytes`; their `.hash` files are +/// decimal text `xxHash32` values with seed `0`. +pub fn verify_official_seed_catalog_hash( + data_url: &str, + data: &[u8], + hash_url: &str, + hash_bytes: &[u8], +) -> Result { + let expected = parse_official_xxhash32_decimal(hash_url, hash_bytes)?; + let actual = xxhash32(data); + + if actual != expected { + return Err(format!( + "official hash mismatch for {data_url}: expected {expected} from {hash_url}, actual {actual}" + )); + } + + Ok(OfficialResourceHashVerification { + data_url: data_url.to_string(), + hash_url: hash_url.to_string(), + algorithm: OfficialResourceHashAlgorithm::XxHash32Decimal, + expected: expected.to_string(), + actual: actual.to_string(), + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct OfficialSeedHashPair { + data_url: String, + hash_url: String, +} + +fn official_seed_hash_pairs(plan: &OfficialResourcePullPlan) -> Vec { + plan.discovery + .endpoints + .iter() + .filter_map(|endpoint| { + let hash_kind = strong_seed_hash_kind(endpoint.kind)?; + let hash_endpoint = plan.discovery.endpoints.iter().find(|candidate| { + candidate.kind == hash_kind && candidate.platform == endpoint.platform + })?; + + Some(OfficialSeedHashPair { + data_url: endpoint.url.clone(), + hash_url: hash_endpoint.url.clone(), + }) + }) + .collect() +} + +fn official_hash_refresh_urls(pairs: &[OfficialSeedHashPair]) -> HashSet { + let mut urls = HashSet::new(); + for pair in pairs { + urls.insert(pair.data_url.clone()); + urls.insert(pair.hash_url.clone()); + } + urls +} + +fn strong_seed_hash_kind( + kind: YostarJpResourceEndpointKind, +) -> Option { + match kind { + YostarJpResourceEndpointKind::TableCatalog => { + Some(YostarJpResourceEndpointKind::TableCatalogHash) + } + YostarJpResourceEndpointKind::BundlePackingInfo => { + Some(YostarJpResourceEndpointKind::BundlePackingInfoHash) + } + YostarJpResourceEndpointKind::MediaCatalog => { + Some(YostarJpResourceEndpointKind::MediaCatalogHash) + } + YostarJpResourceEndpointKind::AddressablesCatalog + | YostarJpResourceEndpointKind::TableCatalogHash + | YostarJpResourceEndpointKind::AddressablesCatalogHash + | YostarJpResourceEndpointKind::BundlePackingInfoHash + | YostarJpResourceEndpointKind::MediaCatalogHash => None, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialDownloadManifest { + #[serde(default = "default_download_manifest_version")] + version: u32, + #[serde(default)] + entries: BTreeMap, +} + +impl Default for OfficialDownloadManifest { + fn default() -> Self { + Self { + version: DOWNLOAD_MANIFEST_VERSION, + entries: BTreeMap::new(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialDownloadManifestEntry { + url: String, + destination: String, + bytes: u64, + blake3: String, +} + +fn default_download_manifest_version() -> u32 { + DOWNLOAD_MANIFEST_VERSION +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PullOneResult { + bytes: u64, + transferred_bytes: u64, + status: OfficialResourcePullStatus, +} + +fn file_len_if_exists(path: &Path) -> Result, String> { + match fs::metadata(path) { + Ok(metadata) if metadata.is_file() => Ok(Some(metadata.len())), + Ok(_) => Err(format!( + "Expected file path but found non-file: {}", + path.display() + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(format!("Failed to inspect {}: {error}", path.display())), + } +} + +fn partial_path_for(destination: &Path) -> PathBuf { + let mut partial = destination.as_os_str().to_owned(); + partial.push(".part"); + PathBuf::from(partial) +} + +fn blake3_file_hex(path: &Path) -> Result { + let mut file = + File::open(path).map_err(|error| format!("Failed to open {}: {error}", path.display()))?; + let mut hasher = blake3::Hasher::new(); + let mut buffer = [0u8; 64 * 1024]; + + loop { + let read = file + .read(&mut buffer) + .map_err(|error| format!("Failed to read {}: {error}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + + Ok(hasher.finalize().to_hex().to_string()) +} + +fn parse_official_xxhash32_decimal(hash_url: &str, bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes) + .map_err(|error| format!("Official hash sidecar is not UTF-8 at {hash_url}: {error}"))? + .trim(); + text.parse::().map_err(|error| { + format!("Official hash sidecar is not decimal xxHash32 at {hash_url}: {error}") + }) +} + +fn xxhash32(bytes: &[u8]) -> u32 { + const PRIME1: u32 = 0x9E37_79B1; + const PRIME2: u32 = 0x85EB_CA77; + const PRIME3: u32 = 0xC2B2_AE3D; + const PRIME4: u32 = 0x27D4_EB2F; + const PRIME5: u32 = 0x1656_67B1; + + let len = bytes.len(); + let mut index = 0usize; + let mut hash = if len >= 16 { + let mut v1 = PRIME1.wrapping_add(PRIME2); + let mut v2 = PRIME2; + let mut v3 = 0; + let mut v4 = 0u32.wrapping_sub(PRIME1); + + while index + 16 <= len { + v1 = xxhash32_round(v1, read_u32_le(bytes, index)); + v2 = xxhash32_round(v2, read_u32_le(bytes, index + 4)); + v3 = xxhash32_round(v3, read_u32_le(bytes, index + 8)); + v4 = xxhash32_round(v4, read_u32_le(bytes, index + 12)); + index += 16; + } + + v1.rotate_left(1) + .wrapping_add(v2.rotate_left(7)) + .wrapping_add(v3.rotate_left(12)) + .wrapping_add(v4.rotate_left(18)) + } else { + PRIME5 + } + .wrapping_add(len as u32); + + while index + 4 <= len { + hash = hash + .wrapping_add(read_u32_le(bytes, index).wrapping_mul(PRIME3)) + .rotate_left(17) + .wrapping_mul(PRIME4); + index += 4; + } + + while index < len { + hash = hash + .wrapping_add((bytes[index] as u32).wrapping_mul(PRIME5)) + .rotate_left(11) + .wrapping_mul(PRIME1); + index += 1; + } + + xxhash32_avalanche(hash) +} + +fn xxhash32_round(acc: u32, input: u32) -> u32 { + const PRIME2: u32 = 0x85EB_CA77; + const PRIME1: u32 = 0x9E37_79B1; + + acc.wrapping_add(input.wrapping_mul(PRIME2)) + .rotate_left(13) + .wrapping_mul(PRIME1) +} + +fn xxhash32_avalanche(mut hash: u32) -> u32 { + hash ^= hash >> 15; + hash = hash.wrapping_mul(0x85EB_CA6B); + hash ^= hash >> 13; + hash = hash.wrapping_mul(0xC2B2_AE35); + hash ^= hash >> 16; + hash +} + +fn read_u32_le(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + fn sanitize_segment(segment: &str) -> String { segment .chars() @@ -269,7 +1090,155 @@ mod tests { ) } - fn fake_curl_script() -> &'static str { + fn fake_curl_script() -> String { + format!( + r#"#!/bin/sh +set -eu +out="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + --url) + url="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [ -n "$out" ]; then + mkdir -p "$(dirname "$out")" + case "$url" in + */TableBundles/TableCatalog.hash) + printf '%s' '{table_catalog_hash}' > "$out" + ;; + */TableBundles/TableCatalog.bytes) + printf '%s' 'ExcelDB.db' > "$out" + ;; + */Windows_PatchPack/BundlePackingInfo.hash) + printf '%s' '{windows_bundle_hash}' > "$out" + ;; + */Android_PatchPack/BundlePackingInfo.hash) + printf '%s' '{android_bundle_hash}' > "$out" + ;; + */Windows_PatchPack/BundlePackingInfo.bytes) + printf '%s' 'FullPatch_000.zip' > "$out" + ;; + */Android_PatchPack/BundlePackingInfo.bytes) + printf '%s' 'FullPatch_001.zip' > "$out" + ;; + */MediaResources-Windows/Catalog/MediaCatalog.hash) + printf '%s' '{windows_media_hash}' > "$out" + ;; + */MediaResources/Catalog/MediaCatalog.hash) + printf '%s' '{android_media_hash}' > "$out" + ;; + */MediaResources-Windows/Catalog/MediaCatalog.bytes) + printf '%s' 'JP_Airi_Win.zip' > "$out" + ;; + */MediaResources/Catalog/MediaCatalog.bytes) + printf '%s' 'JP_Airi_Android.zip' > "$out" + ;; + *) + printf '%s' "$url" > "$out" + ;; + esac +else + printf '%s' "$url" +fi +"#, + table_catalog_hash = xxhash32(b"ExcelDB.db"), + windows_bundle_hash = xxhash32(b"FullPatch_000.zip"), + android_bundle_hash = xxhash32(b"FullPatch_001.zip"), + windows_media_hash = xxhash32(b"JP_Airi_Win.zip"), + android_media_hash = xxhash32(b"JP_Airi_Android.zip"), + ) + } + + fn fake_resume_curl_script() -> &'static str { + r#"#!/bin/sh +set -eu +out="" +url="" +resume=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + --continue-at) + resume=1 + shift 2 + ;; + --url) + url="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [ -n "$out" ]; then + mkdir -p "$(dirname "$out")" + if [ "$resume" -eq 1 ]; then + printf '%s' "-resumed" >> "$out" + else + printf '%s' "$url" > "$out" + fi +else + printf '%s' "$url" +fi +"# + } + + fn fake_flaky_curl_script() -> &'static str { + r#"#!/bin/sh +set -eu +out="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + --url) + url="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +state="$0.state" +count=0 +if [ -f "$state" ]; then + count="$(cat "$state")" +fi +count=$((count + 1)) +printf '%s' "$count" > "$state" +if [ "$count" -eq 1 ]; then + printf '%s\n' "temporary failure" >&2 + exit 22 +fi +if [ -n "$out" ]; then + mkdir -p "$(dirname "$out")" + printf '%s' "$url" > "$out" +else + printf '%s' "$url" +fi +"# + } + + fn fake_bad_hash_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" @@ -291,15 +1260,25 @@ while [ "$#" -gt 0 ]; do done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" - printf '%s' "$url" > "$out" + case "$url" in + */TableBundles/TableCatalog.hash) + printf '%s' '1' > "$out" + ;; + */TableBundles/TableCatalog.bytes) + printf '%s' 'ExcelDB.db' > "$out" + ;; + *) + printf '%s' "$url" > "$out" + ;; + esac else printf '%s' "$url" fi "# } - fn write_fake_curl(path: &Path) { - fs::write(path, fake_curl_script()).unwrap(); + fn write_shell_script(path: &Path, script: &str) { + fs::write(path, script).unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -309,6 +1288,22 @@ fi } } + fn write_fake_curl(path: &Path) { + write_shell_script(path, &fake_curl_script()); + } + + fn write_fake_resume_curl(path: &Path) { + write_shell_script(path, fake_resume_curl_script()); + } + + fn write_fake_flaky_curl(path: &Path) { + write_shell_script(path, fake_flaky_curl_script()); + } + + fn write_fake_bad_hash_curl(path: &Path) { + write_shell_script(path, fake_bad_hash_curl_script()); + } + #[test] fn downloads_verified_pull_plan_to_disk() { let out_dir = TempDir::new().unwrap(); @@ -322,11 +1317,325 @@ fi assert_eq!(report.items.len(), 7); assert!(report.total_bytes() > 0); + assert_eq!(report.downloaded_count(), report.items.len()); + assert_eq!(report.skipped_count(), 0); + assert_eq!(report.resumed_count(), 0); + assert_eq!(report.transferred_bytes(), report.total_bytes()); assert_eq!(report.items[0].url, plan.discovery.endpoints[0].url); for item in &report.items { assert!(item.destination.exists()); - assert_eq!(fs::read_to_string(&item.destination).unwrap(), item.url); + assert_eq!(item.status, OfficialResourcePullStatus::Downloaded); + assert_eq!(item.bytes, item.transferred_bytes); } + assert_eq!(report.official_hash_verified_count(), 1); + assert_eq!( + report.verified_hashes[0].algorithm, + OfficialResourceHashAlgorithm::XxHash32Decimal + ); + assert_eq!(report.verified_hashes[0].expected, "3223500437"); + assert_eq!(report.verified_hashes[0].actual, "3223500437"); + + let manifest = service.read_download_manifest().unwrap(); + assert_eq!(manifest.version, DOWNLOAD_MANIFEST_VERSION); + assert_eq!(manifest.entries.len(), report.items.len()); + for item in &report.items { + let entry = manifest.entries.get(&item.url).unwrap(); + assert_eq!(entry.url, item.url); + assert_eq!( + entry.destination, + service.relative_destination(&item.destination).unwrap() + ); + assert_eq!(entry.bytes, item.bytes); + assert_eq!(entry.blake3, blake3_file_hex(&item.destination).unwrap()); + } + } + + #[test] + fn local_manifest_audit_detects_and_repairs_corrupted_file() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan(discovery_plan(), inventory()); + let first_report = service.pull(&plan).unwrap(); + let clean_audit = service.audit_local_manifest(&plan).unwrap(); + assert!(clean_audit.is_clean()); + assert_eq!(clean_audit.verified_count(), first_report.items.len()); + + let corrupted = first_report + .items + .iter() + .find(|item| item.url.ends_with("/Windows_PatchPack/FullPatch_000.zip")) + .unwrap(); + let mut corrupted_bytes = fs::read(&corrupted.destination).unwrap(); + corrupted_bytes[0] ^= 0xff; + fs::write(&corrupted.destination, corrupted_bytes).unwrap(); + + let corrupted_audit = service.audit_local_manifest(&plan).unwrap(); + assert!(!corrupted_audit.is_clean()); + assert_eq!(corrupted_audit.repair_needed_count(), 1); + assert!(corrupted_audit.items.iter().any(|item| { + item.url == corrupted.url + && item.status == OfficialLocalManifestAuditStatus::Blake3Mismatch + })); + + let repair_report = service.pull(&plan).unwrap(); + assert!(repair_report.downloaded_count() >= 1); + assert!(service.audit_local_manifest(&plan).unwrap().is_clean()); + } + + #[test] + fn refreshes_official_hash_pairs_and_skips_manifest_verified_content() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan(discovery_plan(), inventory()); + let mut manifest = OfficialDownloadManifest::default(); + + for url in plan.all_urls().unwrap() { + let destination = service.destination_for_url(&url).unwrap(); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + let bytes = if url.ends_with("/TableBundles/TableCatalog.bytes") { + b"ExcelDB.db".as_slice() + } else if url.ends_with("/TableBundles/TableCatalog.hash") { + b"3223500437".as_slice() + } else { + url.as_bytes() + }; + fs::write(&destination, bytes).unwrap(); + service + .record_download_manifest_entry(&mut manifest, &url, &destination) + .unwrap(); + } + service.write_download_manifest(&manifest).unwrap(); + + let report = service.pull(&plan).unwrap(); + + assert_eq!(report.items.len(), 7); + assert_eq!(report.skipped_count(), 5); + assert_eq!(report.downloaded_count(), 2); + assert_eq!(report.resumed_count(), 0); + assert!(report.transferred_bytes() > 0); + assert_eq!(report.official_hash_verified_count(), 1); + assert_eq!( + fs::read_to_string( + service + .destination_for_url( + "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", + ) + .unwrap() + ) + .unwrap(), + "ExcelDB.db" + ); + } + + #[test] + fn redownloads_existing_files_without_manifest() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan(discovery_plan(), inventory()); + + for url in plan.all_urls().unwrap() { + let destination = service.destination_for_url(&url).unwrap(); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + fs::write(destination, b"stale-local-file").unwrap(); + } + + let report = service.pull(&plan).unwrap(); + + assert_eq!(report.items.len(), 7); + assert_eq!(report.skipped_count(), 0); + assert_eq!(report.downloaded_count(), 7); + assert_eq!(report.resumed_count(), 0); + assert_eq!(report.official_hash_verified_count(), 1); + assert!(report.transferred_bytes() > 0); + assert!(report + .items + .iter() + .all(|item| item.status == OfficialResourcePullStatus::Downloaded)); + } + + #[test] + fn redownloads_existing_file_when_manifest_hash_mismatches() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan_for_platforms( + discovery_plan(), + inventory(), + &[PatchPlatform::Windows], + ); + + let first_report = service.pull(&plan).unwrap(); + assert_eq!(first_report.downloaded_count(), first_report.items.len()); + + let first_item = &first_report.items[0]; + fs::write(&first_item.destination, b"corrupted").unwrap(); + + let second_report = service + .pull(&OfficialResourcePullPlan { + discovery: YostarJpResourceDiscoveryPlan { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: None, + addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" + .to_string(), + endpoints: vec![YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: first_item.url.clone(), + }], + }, + inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( + YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), + &[], + ), + platforms: Vec::new(), + }) + .unwrap(); + + assert_eq!(second_report.items.len(), 1); + assert_eq!(second_report.skipped_count(), 0); + assert_eq!(second_report.downloaded_count(), 1); + assert_eq!( + fs::read_to_string(&first_item.destination).unwrap(), + "ExcelDB.db" + ); + } + + #[test] + fn rejects_seed_catalog_when_official_hash_mismatches() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_bad_hash_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = OfficialResourcePullPlan { + discovery: discovery_plan(), + inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( + YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), + &[], + ), + platforms: Vec::new(), + }; + + let error = service.pull(&plan).unwrap_err(); + assert!(error.contains("official hash mismatch")); + } + + #[test] + fn verifies_known_real_seed_catalog_hash_samples() { + assert_eq!(xxhash32(b"ExcelDB.db"), 3223500437); + assert_eq!(xxhash32(b"FullPatch_000.zip"), 3628251239); + assert_eq!(xxhash32(b"JP_Airi_Win.zip"), 876690720); + } + + #[test] + fn resumes_partial_files_before_atomic_completion() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_resume_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan_for_platforms( + discovery_plan(), + inventory(), + &[PatchPlatform::Windows], + ); + let first_url = plan.all_urls().unwrap().remove(0); + let destination = service.destination_for_url(&first_url).unwrap(); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + let partial = partial_path_for(&destination); + fs::write(&partial, b"partial").unwrap(); + + let report = service + .pull(&OfficialResourcePullPlan { + discovery: YostarJpResourceDiscoveryPlan { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: None, + addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" + .to_string(), + endpoints: vec![YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: first_url, + }], + }, + inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( + YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), + &[], + ), + platforms: Vec::new(), + }) + .unwrap(); + + assert_eq!(report.items.len(), 1); + assert_eq!(report.resumed_count(), 1); + assert_eq!(report.items[0].status, OfficialResourcePullStatus::Resumed); + assert_eq!(fs::read_to_string(&destination).unwrap(), "partial-resumed"); + assert!(!partial.exists()); + + let manifest = service.read_download_manifest().unwrap(); + assert_eq!(manifest.entries.len(), 1); + assert!(manifest.entries.contains_key(&report.items[0].url)); + } + + #[test] + fn retries_transient_download_failures() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_flaky_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) + .with_retry_attempts(2); + let plan = OfficialResourcePullPlan { + discovery: YostarJpResourceDiscoveryPlan { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: None, + addressables_root: + "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), + endpoints: vec![YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(), + }], + }, + inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( + YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), + &[], + ), + platforms: Vec::new(), + }; + + let report = service.pull(&plan).unwrap(); + + assert_eq!(report.items.len(), 1); + assert_eq!(report.downloaded_count(), 1); + assert_eq!( + fs::read_to_string(&report.items[0].destination).unwrap(), + report.items[0].url + ); + assert_eq!( + fs::read_to_string(curl_path.with_extension("state")).unwrap(), + "2" + ); } #[test] diff --git a/infrastructure/src/official_game_main_config.rs b/infrastructure/src/official_game_main_config.rs index 6153ee7..81747f0 100644 --- a/infrastructure/src/official_game_main_config.rs +++ b/infrastructure/src/official_game_main_config.rs @@ -1,9 +1,13 @@ //! Official launcher package bootstrap for `GameMainConfig`. +//! +//! This is an explicit metadata discovery helper. It downloads official HTTP +//! artifacts into a temporary directory, extracts only the data needed to read +//! `GameMainConfig`, and does not install or execute the official launcher. -use crate::official_launcher::OfficialLauncherBootstrapService; use crate::official_launcher::launcher_package_url; -use bat_adapters::official::game_main_config::YostarJpGameMainConfig; +use crate::official_launcher::OfficialLauncherBootstrapService; use crate::official_launcher::{YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig}; +use bat_adapters::official::game_main_config::YostarJpGameMainConfig; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -72,14 +76,15 @@ impl OfficialGameMainConfigBootstrapService { .filter(|value| !value.is_empty()) .ok_or_else(|| "Official launcher remote manifest has no source".to_string())?; let cdn_config = self.launcher.fetch_cdn_config()?; - let game_zip_url = launcher_package_url(&cdn_config.primary_cdn, &manifest_source)?; + let package_path = select_game_package_path(&game_config, &manifest_source)?; + let game_zip_url = launcher_package_url(&cdn_config.primary_cdn, package_path)?; let temp_dir = TempDir::new() .map_err(|error| format!("Failed to create temporary directory: {error}"))?; let archive_path = temp_dir.path().join("official-game.zip"); self.download_file_with_fallback( &game_zip_url, &cdn_config.back_up_cdn, - &manifest_source, + package_path, &archive_path, )?; let extract_root = temp_dir.path().join("extract"); @@ -104,7 +109,8 @@ impl OfficialGameMainConfigBootstrapService { /// Fetches the latest official game ZIP, extracts `resources.assets`, and /// decrypts `GameMainConfig`. pub fn fetch_game_main_config(&self) -> Result { - self.fetch_bootstrap().map(|bootstrap| bootstrap.game_main_config) + self.fetch_bootstrap() + .map(|bootstrap| bootstrap.game_main_config) } fn download_file(&self, url: &str, destination: &Path) -> Result<(), String> { @@ -180,6 +186,36 @@ impl OfficialGameMainConfigBootstrapService { } } +fn select_game_package_path<'a>( + game_config: &'a YostarJpLauncherGameConfig, + manifest_source: &'a str, +) -> Result<&'a str, String> { + if is_launcher_archive_path(manifest_source) { + return Ok(manifest_source); + } + + if is_launcher_archive_path(&game_config.game_latest_file_path) { + return Ok(&game_config.game_latest_file_path); + } + + Err(format!( + "Official launcher metadata has no usable game archive path: source={manifest_source}, game_latest_file_path={}", + game_config.game_latest_file_path + )) +} + +fn is_launcher_archive_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.starts_with('\\') + && path.ends_with(".zip") + && !path.contains('?') + && !path.contains('#') + && !path.split('/').any(|segment| { + segment.is_empty() || segment == "." || segment == ".." || segment.contains('\\') + }) +} + fn find_resources_assets(root: &Path) -> Option { let mut stack = vec![root.to_path_buf()]; while let Some(dir) = stack.pop() { diff --git a/infrastructure/src/official_update.rs b/infrastructure/src/official_update.rs new file mode 100644 index 0000000..0b78601 --- /dev/null +++ b/infrastructure/src/official_update.rs @@ -0,0 +1,1193 @@ +//! Official JP resource update orchestration. +//! +//! This module is the production-facing form of the official update flow. It +//! keeps the one-shot semantics required by external schedulers and Go CLI +//! callers: discover current official state, compare remote markers, audit the +//! local manifest, repair/download when needed, then return a structured report. + +use crate::{ + build_official_pull_plan_for_platform_inventory, build_official_sync_plan, + changed_endpoint_urls, default_official_platforms, OfficialGameMainConfigBootstrapService, + OfficialLauncherBootstrapService, OfficialResourcePullPlan, OfficialResourcePullService, + YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest, +}; +use bat_adapters::official::game_main_config::YostarJpGameMainConfig; +use bat_adapters::official::inventory::{ + YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, +}; +use bat_adapters::official::yostar_jp::{ + server_info_url, PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, + YostarJpResourceEndpointKind, YostarJpServerInfo, YostarJpSyncSnapshot, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::path::{Path, PathBuf}; + +/// Current official update snapshot schema version. +pub const OFFICIAL_UPDATE_SNAPSHOT_VERSION: u32 = 2; +/// Current official bootstrap cache schema version. +pub const OFFICIAL_BOOTSTRAP_CACHE_VERSION: u32 = 1; + +/// Server-info input for an official update run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum OfficialServerInfoSource { + /// Read a local, already-audited official server-info JSON file. + LocalPath(PathBuf), + /// Fetch a server-info file by official file name. + OfficialFile(String), + /// Fetch a full official server-info URL. + OfficialUrl(String), +} + +/// Configuration for one official update execution. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OfficialUpdateConfig { + /// Optional explicit server-info source. + pub server_info_source: Option, + /// Optional explicit connection group. + pub connection_group: Option, + /// Optional explicit app version. + pub app_version: Option, + /// Official launcher version used for signed metadata discovery. + pub launcher_version: String, + /// Whether to discover app version, server-info URL, and connection group + /// from official metadata. + pub auto_discover: bool, + /// Platforms to update. Defaults to verified official platforms. + pub platforms: Option>, + /// Output root for resources and state files. + pub output_root: PathBuf, + /// Optional explicit sync snapshot path. + pub snapshot_path: Option, + /// Curl command used by the current infrastructure downloader. + pub curl_command: PathBuf, + /// Unzip command used when a metadata change requires GameMainConfig parsing. + pub unzip_command: PathBuf, + /// Dry run reports decisions and optional plan URLs without writing sync state. + pub dry_run: bool, + /// Include full download URLs when dry-running. + pub plan: bool, + /// Force download even when remote and local state look clean. + pub force: bool, + /// Audit the local download manifest before deciding up-to-date. + pub audit_local: bool, + /// Repair local files when the local manifest audit fails. + pub repair: bool, +} + +impl Default for OfficialUpdateConfig { + fn default() -> Self { + Self { + server_info_source: None, + connection_group: None, + app_version: None, + launcher_version: "1.7.2".to_string(), + auto_discover: false, + platforms: None, + output_root: PathBuf::from("official_update_output"), + snapshot_path: None, + curl_command: PathBuf::from("curl"), + unzip_command: PathBuf::from("unzip"), + dry_run: false, + plan: false, + force: false, + audit_local: true, + repair: true, + } + } +} + +impl OfficialUpdateConfig { + /// Returns the effective sync snapshot path for this config. + pub fn effective_snapshot_path(&self) -> PathBuf { + self.snapshot_path + .clone() + .unwrap_or_else(|| self.output_root.join("official-sync-snapshot.json")) + } + + /// Returns the bootstrap cache path for this config. + pub fn bootstrap_cache_path(&self) -> PathBuf { + self.output_root.join("official-bootstrap-cache.json") + } + + /// Returns the lock path used for non-dry-run executions. + pub fn lock_path(&self) -> PathBuf { + self.output_root.join(".official-sync.lock") + } +} + +/// Status of one official update execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OfficialUpdateStatus { + /// Remote and local resources are already clean. + UpToDate, + /// Dry-run detected that a download would occur. + WouldDownload, + /// Resources were downloaded or repaired. + Downloaded, +} + +impl OfficialUpdateStatus { + /// Returns a stable string label for CLI and JSON callers. + pub fn as_str(self) -> &'static str { + match self { + Self::UpToDate => "up_to_date", + Self::WouldDownload => "would_download", + Self::Downloaded => "downloaded", + } + } +} + +/// Snapshot of the official JP update state observed at a point in time. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialUpdateSnapshot { + /// Snapshot schema version. + #[serde(default = "default_update_snapshot_version")] + pub snapshot_version: u32, + /// Selected server-info connection group. + pub connection_group_name: String, + /// App version used for server-info override selection. + pub app_version: String, + /// Bundle version from server-info, when present. + pub bundle_version: Option, + /// Selected official Addressables root URL. + pub addressables_root: String, + /// Seed endpoints observed for the selected platform set. + pub endpoints: Vec, + /// Small remote marker contents fetched for change detection. + #[serde(default)] + pub endpoint_markers: Vec, + /// Launcher metadata summary used for auto-discovery, when available. + #[serde(default)] + pub launcher_metadata: Option, + /// Parsed GameMainConfig summary used for auto-discovery, when available. + #[serde(default)] + pub game_main_config_bootstrap: Option, +} + +impl OfficialUpdateSnapshot { + /// Creates an update snapshot from a base sync snapshot and extended data. + pub fn new( + base: YostarJpSyncSnapshot, + endpoint_markers: Vec, + bootstrap: Option<&ResolvedBootstrap>, + ) -> Self { + Self { + snapshot_version: OFFICIAL_UPDATE_SNAPSHOT_VERSION, + connection_group_name: base.connection_group_name, + app_version: base.app_version, + bundle_version: base.bundle_version, + addressables_root: base.addressables_root, + endpoints: base.endpoints, + endpoint_markers, + launcher_metadata: bootstrap.map(|bootstrap| bootstrap.launcher_metadata.clone()), + game_main_config_bootstrap: bootstrap + .map(|bootstrap| bootstrap.game_main_config.clone()), + } + } + + /// Returns the legacy base snapshot used by the existing sync planner. + pub fn base_snapshot(&self) -> YostarJpSyncSnapshot { + YostarJpSyncSnapshot { + connection_group_name: self.connection_group_name.clone(), + app_version: self.app_version.clone(), + bundle_version: self.bundle_version.clone(), + addressables_root: self.addressables_root.clone(), + endpoints: self.endpoints.clone(), + } + } + + /// Returns the number of Addressables catalog markers checked. + pub fn addressables_marker_checked_count(&self) -> usize { + self.endpoint_markers + .iter() + .filter(|marker| marker.role == OfficialEndpointMarkerRole::AddressablesCatalogMarker) + .count() + } + + /// Returns marker count not used as strong content validation. + pub fn unverified_marker_count(&self) -> usize { + self.addressables_marker_checked_count() + } +} + +/// Captured value of one small remote marker endpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialEndpointMarkerSnapshot { + /// Endpoint kind. + pub kind: YostarJpResourceEndpointKind, + /// Platform for platform-specific endpoints. + pub platform: Option, + /// Official marker URL. + pub url: String, + /// How this marker should be interpreted. + pub role: OfficialEndpointMarkerRole, + /// Trimmed marker content. + pub value: String, +} + +/// Role of a fetched marker endpoint. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum OfficialEndpointMarkerRole { + /// Official seed `.hash`, interpreted as xxHash32 decimal by download verification. + OfficialSeedHash, + /// Unity Addressables/SBP Hash128 marker, not a strong content validation. + AddressablesCatalogMarker, +} + +/// Summary of launcher metadata used by auto-discovery. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LauncherMetadataSnapshot { + /// Launcher version used for signed requests. + pub launcher_version: String, + /// Latest game version reported by the launcher API. + pub game_latest_version: String, + /// Latest game file path reported by the launcher API. + pub game_latest_file_path: String, + /// Lowest game version accepted by the launcher API. + pub game_lowest_version: Option, + /// Game executable name. + pub game_start_exe_name: Option, + /// Game executable arguments. + pub game_start_params: Vec, + /// Remote launcher manifest URL. + pub manifest_url: String, + /// Remote launcher manifest source path. + pub manifest_source: Option, + /// Number of files in the remote launcher manifest. + pub manifest_file_count: usize, +} + +/// Summary of the decrypted GameMainConfig used by auto-discovery. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GameMainConfigSnapshot { + /// Official server-info URL from GameMainConfig. + pub server_info_data_url: Option, + /// Default connection group from GameMainConfig. + pub default_connection_group: Option, +} + +/// Cached GameMainConfig summary keyed by launcher metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialBootstrapCache { + /// Cache schema version. + #[serde(default = "default_bootstrap_cache_version")] + pub cache_version: u32, + /// Launcher metadata that produced this GameMainConfig summary. + pub launcher_metadata: LauncherMetadataSnapshot, + /// Cached GameMainConfig summary. + pub game_main_config: GameMainConfigSnapshot, +} + +/// Resolved bootstrap data for one run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedBootstrap { + /// Launcher metadata summary. + pub launcher_metadata: LauncherMetadataSnapshot, + /// GameMainConfig summary. + pub game_main_config: GameMainConfigSnapshot, + /// Whether GameMainConfig came from the local cache. + pub cache_hit: bool, +} + +/// Extended snapshot delta not represented by the legacy sync planner. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExtendedSnapshotDelta { + /// Whether fetched endpoint marker contents changed. + pub endpoint_markers_changed: bool, + /// Whether launcher metadata summary changed. + pub launcher_metadata_changed: bool, + /// Whether GameMainConfig summary changed. + pub game_main_config_changed: bool, +} + +impl ExtendedSnapshotDelta { + /// Returns true when any extended snapshot field changed. + pub fn has_changes(self) -> bool { + self.endpoint_markers_changed + || self.launcher_metadata_changed + || self.game_main_config_changed + } +} + +/// Structured report returned by an official update run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialUpdateReport { + /// Final update status. + pub update_status: OfficialUpdateStatus, + /// Selected connection group. + pub connection_group: String, + /// Selected app version. + pub app_version: String, + /// Selected bundle version. + pub bundle_version: Option, + /// Selected Addressables root. + pub addressables_root: String, + /// Platform set used by the run. + pub platforms: Vec, + /// Sync snapshot path. + pub snapshot_path: PathBuf, + /// Whether a previous snapshot existed. + pub previous_snapshot_present: bool, + /// Legacy sync decision label. + pub decision: String, + /// Whether force was enabled. + pub force: bool, + /// Whether local audit was enabled. + pub audit_local: bool, + /// Whether local repair was enabled. + pub repair: bool, + /// Whether a download was required. + pub should_download: bool, + /// Whether this was the first observed snapshot. + pub is_initial: bool, + /// Changed endpoint URLs from legacy diff. + pub changed_endpoint_urls: Vec, + /// Extended marker/metadata delta. + pub extended_delta: ExtendedSnapshotDelta, + /// Count of Addressables marker files checked. + pub addressables_marker_checked_count: usize, + /// Count of marker files not used for strong content validation. + pub unverified_marker_count: usize, + /// Whether bootstrap cache was hit. + pub bootstrap_cache_hit: Option, + /// Bootstrap cache path. + pub bootstrap_cache_path: Option, + /// Local manifest verified file count. + pub local_manifest_verified_count: usize, + /// Local manifest repair-needed file count. + pub local_manifest_repair_needed_count: usize, + /// Dry-run flag. + pub dry_run: bool, + /// Download URL count when plan output was requested. + pub download_url_count: Option, + /// Download URLs when plan output was requested. + pub download_urls: Vec, + /// Pull item count after a real download. + pub resource_count: Option, + /// Number of resources downloaded from scratch. + pub downloaded_count: usize, + /// Number of resources resumed. + pub resumed_count: usize, + /// Number of resources skipped using the local manifest. + pub skipped_count: usize, + /// Final local byte count in the pull report. + pub final_bytes: u64, + /// Bytes transferred in this run. + pub transferred_bytes: u64, + /// Number of official seed hashes verified. + pub official_seed_hash_verified_count: usize, + /// Download manifest path. + pub download_manifest: PathBuf, + /// Snapshot path written after success. + pub snapshot_written: Option, +} + +/// Official update runner. +#[derive(Debug, Clone, Default)] +pub struct OfficialUpdateService; + +impl OfficialUpdateService { + /// Creates an official update runner. + pub fn new() -> Self { + Self + } + + /// Executes one official update run. + pub fn run(&self, config: &OfficialUpdateConfig) -> anyhow::Result { + let _lock = if config.dry_run { + None + } else { + Some(OfficialUpdateLock::acquire(config)?) + }; + + let default_platforms = default_official_platforms(); + let platforms = config + .platforms + .as_deref() + .unwrap_or(default_platforms.as_slice()); + let fetcher = OfficialResourcePullService::with_curl_command( + &config.output_root, + &config.curl_command, + ); + let snapshot_path = config.effective_snapshot_path(); + let bootstrap_cache_path = config.bootstrap_cache_path(); + + let bootstrap = if config.auto_discover { + Some(resolve_bootstrap( + &config.launcher_version, + &config.curl_command, + &config.unzip_command, + &bootstrap_cache_path, + !config.dry_run, + )?) + } else { + None + }; + + let app_version = config + .app_version + .clone() + .or_else(|| { + bootstrap + .as_ref() + .map(|bootstrap| bootstrap.launcher_metadata.game_latest_version.clone()) + }) + .ok_or_else(|| { + anyhow::anyhow!("missing app version; pass it explicitly or enable auto-discover") + })?; + let connection_group = config + .connection_group + .clone() + .or_else(|| { + bootstrap.as_ref().and_then(|bootstrap| { + bootstrap.game_main_config.default_connection_group.clone() + }) + }) + .ok_or_else(|| { + anyhow::anyhow!( + "missing connection group; pass it explicitly or enable auto-discover" + ) + })?; + + let server_info_bytes = if let Some(source) = config.server_info_source.as_ref() { + load_server_info(source, &fetcher)? + } else { + let bootstrap = bootstrap.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "missing server-info source; pass one explicitly or enable auto-discover" + ) + })?; + let url = bootstrap + .game_main_config + .server_info_data_url + .as_deref() + .ok_or_else(|| { + anyhow::anyhow!("official GameMainConfig has no ServerInfoDataUrl") + })?; + fetcher.fetch_bytes(url).map_err(anyhow::Error::msg)? + }; + + let server_info = YostarJpServerInfo::from_slice(&server_info_bytes)?; + let current_snapshot = server_info + .sync_snapshot(&connection_group, &app_version, platforms) + .map_err(anyhow::Error::msg)?; + let endpoint_markers = collect_endpoint_markers(&fetcher, ¤t_snapshot)?; + let current_update_snapshot = OfficialUpdateSnapshot::new( + current_snapshot.clone(), + endpoint_markers, + bootstrap.as_ref(), + ); + let previous_snapshot = read_snapshot(&snapshot_path)?; + let previous_base_snapshot = previous_snapshot + .as_ref() + .map(OfficialUpdateSnapshot::base_snapshot); + let sync_plan = + build_official_sync_plan(current_snapshot.clone(), previous_base_snapshot.as_ref()); + let extended_delta = + diff_extended_snapshot(¤t_update_snapshot, previous_snapshot.as_ref()); + let changed_urls = changed_endpoint_urls(&sync_plan.delta); + let remote_should_download = + config.force || sync_plan.should_download() || extended_delta.has_changes(); + let pull_plan = if config.audit_local || remote_should_download || config.plan { + Some(build_pull_plan( + &server_info, + &connection_group, + &app_version, + platforms, + &fetcher, + )?) + } else { + None + }; + let local_audit = if config.audit_local { + Some( + fetcher + .audit_local_manifest( + pull_plan + .as_ref() + .ok_or_else(|| anyhow::anyhow!("local audit requires a pull plan"))?, + ) + .map_err(anyhow::Error::msg)?, + ) + } else { + None + }; + let local_manifest_verified_count = local_audit + .as_ref() + .map(|audit| audit.verified_count()) + .unwrap_or(0); + let local_manifest_repair_needed_count = local_audit + .as_ref() + .map(|audit| audit.repair_needed_count()) + .unwrap_or(0); + let repair_needed = config.repair + && local_audit + .as_ref() + .map(|audit| !audit.is_clean()) + .unwrap_or(false); + let should_download = remote_should_download || repair_needed; + let mut report = OfficialUpdateReport { + update_status: if should_download { + OfficialUpdateStatus::WouldDownload + } else { + OfficialUpdateStatus::UpToDate + }, + connection_group: current_snapshot.connection_group_name.clone(), + app_version: current_snapshot.app_version.clone(), + bundle_version: current_snapshot.bundle_version.clone(), + addressables_root: current_snapshot.addressables_root.clone(), + platforms: platforms.to_vec(), + snapshot_path: snapshot_path.clone(), + previous_snapshot_present: previous_snapshot.is_some(), + decision: format!("{:?}", sync_plan.decision), + force: config.force, + audit_local: config.audit_local, + repair: config.repair, + should_download, + is_initial: sync_plan.delta.is_initial, + changed_endpoint_urls: changed_urls, + extended_delta, + addressables_marker_checked_count: current_update_snapshot + .addressables_marker_checked_count(), + unverified_marker_count: current_update_snapshot.unverified_marker_count(), + bootstrap_cache_hit: bootstrap.as_ref().map(|bootstrap| bootstrap.cache_hit), + bootstrap_cache_path: bootstrap.as_ref().map(|_| bootstrap_cache_path.clone()), + local_manifest_verified_count, + local_manifest_repair_needed_count, + dry_run: config.dry_run, + download_url_count: None, + download_urls: Vec::new(), + resource_count: None, + downloaded_count: 0, + resumed_count: 0, + skipped_count: 0, + final_bytes: 0, + transferred_bytes: 0, + official_seed_hash_verified_count: 0, + download_manifest: fetcher.download_manifest_path(), + snapshot_written: None, + }; + + if !should_download { + return Ok(report); + } + + if config.dry_run { + if config.plan { + let urls = pull_plan + .as_ref() + .ok_or_else(|| { + anyhow::anyhow!("dry-run plan requested but no pull plan exists") + })? + .all_urls() + .map_err(anyhow::Error::msg)?; + report.download_url_count = Some(urls.len()); + report.download_urls = urls; + } + report.update_status = OfficialUpdateStatus::WouldDownload; + return Ok(report); + } + + let pull_plan = pull_plan + .as_ref() + .ok_or_else(|| anyhow::anyhow!("download requested but no pull plan exists"))?; + let pull_report = fetcher.pull(pull_plan).map_err(anyhow::Error::msg)?; + let final_audit = fetcher + .audit_local_manifest(pull_plan) + .map_err(anyhow::Error::msg)?; + write_snapshot(&snapshot_path, ¤t_update_snapshot)?; + + report.update_status = OfficialUpdateStatus::Downloaded; + report.resource_count = Some(pull_report.items.len()); + report.downloaded_count = pull_report.downloaded_count(); + report.resumed_count = pull_report.resumed_count(); + report.skipped_count = pull_report.skipped_count(); + report.final_bytes = pull_report.total_bytes(); + report.transferred_bytes = pull_report.transferred_bytes(); + report.official_seed_hash_verified_count = pull_report.official_hash_verified_count(); + report.local_manifest_verified_count = final_audit.verified_count(); + report.local_manifest_repair_needed_count = final_audit.repair_needed_count(); + report.snapshot_written = Some(snapshot_path); + + Ok(report) + } +} + +fn build_pull_plan( + server_info: &YostarJpServerInfo, + connection_group: &str, + app_version: &str, + platforms: &[PatchPlatform], + fetcher: &OfficialResourcePullService, +) -> anyhow::Result { + let discovery = server_info + .discovery_plan(connection_group, app_version, platforms) + .map_err(anyhow::Error::msg)?; + let seed_catalogs = fetch_seed_catalogs(fetcher, &discovery)?; + let inventory = build_inventory_from_seed_catalogs(&seed_catalogs, platforms)?; + Ok(build_official_pull_plan_for_platform_inventory( + discovery, inventory, platforms, + )) +} + +fn load_server_info( + source: &OfficialServerInfoSource, + fetcher: &OfficialResourcePullService, +) -> anyhow::Result> { + match source { + OfficialServerInfoSource::LocalPath(path) => Ok(fs::read(path)?), + OfficialServerInfoSource::OfficialFile(file_name) => { + let url = server_info_url(file_name).map_err(anyhow::Error::msg)?; + fetcher.fetch_bytes(&url).map_err(anyhow::Error::msg) + } + OfficialServerInfoSource::OfficialUrl(url) => { + fetcher.fetch_bytes(url).map_err(anyhow::Error::msg) + } + } +} + +fn collect_endpoint_markers( + fetcher: &OfficialResourcePullService, + snapshot: &YostarJpSyncSnapshot, +) -> anyhow::Result> { + let mut markers = Vec::new(); + + for endpoint in &snapshot.endpoints { + let Some(role) = marker_role(endpoint.kind) else { + continue; + }; + let bytes = fetcher + .fetch_bytes(&endpoint.url) + .map_err(anyhow::Error::msg)?; + let value = String::from_utf8_lossy(&bytes).trim().to_string(); + markers.push(OfficialEndpointMarkerSnapshot { + kind: endpoint.kind, + platform: endpoint.platform, + url: endpoint.url.clone(), + role, + value, + }); + } + + Ok(markers) +} + +fn marker_role(kind: YostarJpResourceEndpointKind) -> Option { + match kind { + YostarJpResourceEndpointKind::TableCatalogHash + | YostarJpResourceEndpointKind::BundlePackingInfoHash + | YostarJpResourceEndpointKind::MediaCatalogHash => { + Some(OfficialEndpointMarkerRole::OfficialSeedHash) + } + YostarJpResourceEndpointKind::AddressablesCatalogHash => { + Some(OfficialEndpointMarkerRole::AddressablesCatalogMarker) + } + _ => None, + } +} + +/// Computes the extended snapshot delta. +pub fn diff_extended_snapshot( + current: &OfficialUpdateSnapshot, + previous: Option<&OfficialUpdateSnapshot>, +) -> ExtendedSnapshotDelta { + let Some(previous) = previous else { + return ExtendedSnapshotDelta { + endpoint_markers_changed: true, + launcher_metadata_changed: current.launcher_metadata.is_some(), + game_main_config_changed: current.game_main_config_bootstrap.is_some(), + }; + }; + + ExtendedSnapshotDelta { + endpoint_markers_changed: current.endpoint_markers != previous.endpoint_markers, + launcher_metadata_changed: current.launcher_metadata != previous.launcher_metadata, + game_main_config_changed: current.game_main_config_bootstrap + != previous.game_main_config_bootstrap, + } +} + +/// Reads an official update snapshot from disk, accepting legacy v1 snapshots. +pub fn read_snapshot(path: &Path) -> anyhow::Result> { + if !path.exists() { + return Ok(None); + } + + let data = fs::read(path)?; + match serde_json::from_slice::(&data) { + Ok(snapshot) => Ok(Some(snapshot)), + Err(update_error) => { + let legacy = + serde_json::from_slice::(&data).map_err(|legacy_error| { + anyhow::anyhow!( + "failed to parse official update snapshot as v2 ({update_error}) or legacy v1 ({legacy_error})" + ) + })?; + Ok(Some(OfficialUpdateSnapshot::new(legacy, Vec::new(), None))) + } + } +} + +/// Writes an official update snapshot to disk. +pub fn write_snapshot(path: &Path, snapshot: &OfficialUpdateSnapshot) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let data = serde_json::to_vec_pretty(snapshot)?; + fs::write(path, data)?; + Ok(()) +} + +/// Reads an official bootstrap cache from disk. +pub fn read_bootstrap_cache(path: &Path) -> anyhow::Result> { + if !path.exists() { + return Ok(None); + } + + let data = fs::read(path)?; + Ok(Some(serde_json::from_slice(&data)?)) +} + +/// Writes an official bootstrap cache to disk. +pub fn write_bootstrap_cache(path: &Path, cache: &OfficialBootstrapCache) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let data = serde_json::to_vec_pretty(cache)?; + fs::write(path, data)?; + Ok(()) +} + +/// Returns cached GameMainConfig when cache version and metadata match. +pub fn cached_game_main_config_for_metadata( + cache: &OfficialBootstrapCache, + launcher_metadata: &LauncherMetadataSnapshot, +) -> Option { + if cache.cache_version == OFFICIAL_BOOTSTRAP_CACHE_VERSION + && cache.launcher_metadata == *launcher_metadata + { + Some(cache.game_main_config.clone()) + } else { + None + } +} + +fn default_update_snapshot_version() -> u32 { + OFFICIAL_UPDATE_SNAPSHOT_VERSION +} + +fn default_bootstrap_cache_version() -> u32 { + OFFICIAL_BOOTSTRAP_CACHE_VERSION +} + +fn launcher_metadata_from_parts( + launcher_version: &str, + game_config: &YostarJpLauncherGameConfig, + manifest_url: &YostarJpLauncherManifestUrl, + manifest: &YostarJpLauncherRemoteManifest, +) -> LauncherMetadataSnapshot { + LauncherMetadataSnapshot { + launcher_version: launcher_version.to_string(), + game_latest_version: game_config.game_latest_version.clone(), + game_latest_file_path: game_config.game_latest_file_path.clone(), + game_lowest_version: game_config.game_lowest_version.clone(), + game_start_exe_name: game_config.game_start_exe_name.clone(), + game_start_params: game_config.game_start_params.clone(), + manifest_url: manifest_url.url.clone(), + manifest_source: manifest.source.clone().filter(|value| !value.is_empty()), + manifest_file_count: manifest.files.len(), + } +} + +fn game_main_config_snapshot(config: &YostarJpGameMainConfig) -> GameMainConfigSnapshot { + GameMainConfigSnapshot { + server_info_data_url: config.server_info_data_url.clone(), + default_connection_group: config.default_connection_group.clone(), + } +} + +fn resolve_bootstrap( + launcher_version: &str, + curl_command: &Path, + unzip_command: &Path, + cache_path: &Path, + write_cache: bool, +) -> anyhow::Result { + let launcher = OfficialLauncherBootstrapService::with_curl_command( + launcher_version, + curl_command.to_string_lossy().to_string(), + ); + let (game_config, manifest_url, manifest) = launcher + .fetch_latest_remote_manifest() + .map_err(anyhow::Error::msg)?; + let launcher_metadata = + launcher_metadata_from_parts(launcher_version, &game_config, &manifest_url, &manifest); + + if let Some(cache) = read_bootstrap_cache(cache_path)? { + if let Some(game_main_config) = + cached_game_main_config_for_metadata(&cache, &launcher_metadata) + { + return Ok(ResolvedBootstrap { + launcher_metadata, + game_main_config, + cache_hit: true, + }); + } + } + + let bootstrapper = OfficialGameMainConfigBootstrapService::with_commands( + launcher_version, + curl_command.to_path_buf(), + unzip_command.to_path_buf(), + ); + let bootstrap = bootstrapper.fetch_bootstrap().map_err(anyhow::Error::msg)?; + let launcher_metadata = LauncherMetadataSnapshot { + launcher_version: launcher_version.to_string(), + game_latest_version: bootstrap.game_config.game_latest_version.clone(), + game_latest_file_path: bootstrap.game_config.game_latest_file_path.clone(), + game_lowest_version: bootstrap.game_config.game_lowest_version.clone(), + game_start_exe_name: bootstrap.game_config.game_start_exe_name.clone(), + game_start_params: bootstrap.game_config.game_start_params.clone(), + manifest_url: bootstrap.manifest_url.clone(), + manifest_source: bootstrap.manifest_source.clone(), + manifest_file_count: bootstrap.manifest_file_count, + }; + let game_main_config = game_main_config_snapshot(&bootstrap.game_main_config); + if write_cache { + write_bootstrap_cache( + cache_path, + &OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: launcher_metadata.clone(), + game_main_config: game_main_config.clone(), + }, + )?; + } + + Ok(ResolvedBootstrap { + launcher_metadata, + game_main_config, + cache_hit: false, + }) +} + +fn fetch_seed_catalogs( + fetcher: &OfficialResourcePullService, + discovery: &YostarJpResourceDiscoveryPlan, +) -> anyhow::Result { + let mut catalogs = SeedCatalogs::default(); + + for endpoint in &discovery.endpoints { + if !matches!( + endpoint.kind, + YostarJpResourceEndpointKind::TableCatalog + | YostarJpResourceEndpointKind::BundlePackingInfo + | YostarJpResourceEndpointKind::MediaCatalog + ) { + continue; + } + + let bytes = fetcher + .fetch_bytes(&endpoint.url) + .map_err(anyhow::Error::msg)?; + catalogs.insert(endpoint, bytes)?; + } + + Ok(catalogs) +} + +fn build_inventory_from_seed_catalogs( + catalogs: &SeedCatalogs, + platforms: &[PatchPlatform], +) -> anyhow::Result { + let table_catalog = catalogs + .table_catalog + .as_deref() + .ok_or_else(|| anyhow::anyhow!("official discovery did not include TableCatalog.bytes"))?; + let mut platform_catalogs = Vec::new(); + + for platform in platforms { + let bundle_packing_info = catalogs.bundle_packing_infos.get(platform).ok_or_else(|| { + anyhow::anyhow!( + "official discovery did not include {} BundlePackingInfo.bytes", + platform.as_str() + ) + })?; + let media_catalog = catalogs.media_catalogs.get(platform).ok_or_else(|| { + anyhow::anyhow!( + "official discovery did not include {} MediaCatalog.bytes", + platform.as_str() + ) + })?; + + platform_catalogs.push(YostarJpPlatformCatalogInventory::from_catalog_bytes( + *platform, + bundle_packing_info, + media_catalog, + )); + } + + Ok(YostarJpPlatformDownloadInventory::from_catalog_bytes( + table_catalog, + platform_catalogs, + )) +} + +#[derive(Debug, Default)] +struct SeedCatalogs { + table_catalog: Option>, + bundle_packing_infos: HashMap>, + media_catalogs: HashMap>, +} + +impl SeedCatalogs { + fn insert( + &mut self, + endpoint: &YostarJpResourceEndpoint, + bytes: Vec, + ) -> anyhow::Result<()> { + match endpoint.kind { + YostarJpResourceEndpointKind::TableCatalog => { + self.table_catalog = Some(bytes); + } + YostarJpResourceEndpointKind::BundlePackingInfo => { + let platform = required_platform(endpoint)?; + self.bundle_packing_infos.insert(platform, bytes); + } + YostarJpResourceEndpointKind::MediaCatalog => { + let platform = required_platform(endpoint)?; + self.media_catalogs.insert(platform, bytes); + } + _ => {} + } + + Ok(()) + } +} + +fn required_platform(endpoint: &YostarJpResourceEndpoint) -> anyhow::Result { + endpoint.platform.ok_or_else(|| { + anyhow::anyhow!( + "discovery endpoint {:?} has no platform: {}", + endpoint.kind, + endpoint.url + ) + }) +} + +#[derive(Debug)] +struct OfficialUpdateLock { + path: PathBuf, +} + +impl OfficialUpdateLock { + fn acquire(config: &OfficialUpdateConfig) -> anyhow::Result { + fs::create_dir_all(&config.output_root)?; + let path = config.lock_path(); + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => { + let pid = std::process::id().to_string(); + use std::io::Write; + let mut file = file; + file.write_all(pid.as_bytes())?; + Ok(Self { path }) + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Err( + anyhow::anyhow!("official update output is locked: {}", path.display()), + ), + Err(error) => Err(anyhow::anyhow!( + "failed to acquire official update lock {}: {error}", + path.display() + )), + } + } +} + +impl Drop for OfficialUpdateLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_base_snapshot() -> YostarJpSyncSnapshot { + YostarJpSyncSnapshot { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: Some("bundle".to_string()), + addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/root".to_string(), + endpoints: vec![ + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.bytes".to_string(), + }, + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalogHash, + platform: None, + url: "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.hash".to_string(), + }, + YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::AddressablesCatalogHash, + platform: Some(PatchPlatform::Windows), + url: "https://prod-clientpatch.bluearchiveyostar.com/root/Windows_PatchPack/catalog_StandaloneWindows64.hash".to_string(), + }, + ], + } + } + + fn fixture_marker(value: &str) -> OfficialEndpointMarkerSnapshot { + OfficialEndpointMarkerSnapshot { + kind: YostarJpResourceEndpointKind::TableCatalogHash, + platform: None, + url: + "https://prod-clientpatch.bluearchiveyostar.com/root/TableBundles/TableCatalog.hash" + .to_string(), + role: OfficialEndpointMarkerRole::OfficialSeedHash, + value: value.to_string(), + } + } + + fn fixture_bootstrap() -> ResolvedBootstrap { + ResolvedBootstrap { + launcher_metadata: LauncherMetadataSnapshot { + launcher_version: "1.7.2".to_string(), + game_latest_version: "1.70.0".to_string(), + game_latest_file_path: "BAJP_1.70.0.zip".to_string(), + game_lowest_version: Some("1.69.0".to_string()), + game_start_exe_name: Some("BlueArchive".to_string()), + game_start_params: vec!["--prod".to_string()], + manifest_url: "https://launcher-pkg-ba-jp.yo-star.com/manifest.json".to_string(), + manifest_source: Some("BAJP_1.70.0.zip".to_string()), + manifest_file_count: 42, + }, + game_main_config: GameMainConfigSnapshot { + server_info_data_url: Some( + "https://yostar-serverinfo.bluearchiveyostar.com/prod.json".to_string(), + ), + default_connection_group: Some("Prod-Audit".to_string()), + }, + cache_hit: false, + } + } + + #[test] + fn persists_snapshot_json_round_trip() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("snapshot.json"); + let bootstrap = fixture_bootstrap(); + let snapshot = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("1234")], + Some(&bootstrap), + ); + + write_snapshot(&path, &snapshot).unwrap(); + assert_eq!(read_snapshot(&path).unwrap(), Some(snapshot)); + } + + #[test] + fn reads_legacy_snapshot_json() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("snapshot.json"); + let legacy = fixture_base_snapshot(); + fs::write(&path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + let snapshot = read_snapshot(&path).unwrap().unwrap(); + assert_eq!(snapshot.base_snapshot(), legacy); + assert!(snapshot.endpoint_markers.is_empty()); + assert_eq!(snapshot.snapshot_version, OFFICIAL_UPDATE_SNAPSHOT_VERSION); + } + + #[test] + fn marker_content_change_requires_download() { + let previous = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("1111")], + Some(&fixture_bootstrap()), + ); + let current = OfficialUpdateSnapshot::new( + fixture_base_snapshot(), + vec![fixture_marker("2222")], + Some(&fixture_bootstrap()), + ); + + let delta = diff_extended_snapshot(¤t, Some(&previous)); + assert!(delta.endpoint_markers_changed); + assert!(delta.has_changes()); + assert!(!delta.launcher_metadata_changed); + assert!(!delta.game_main_config_changed); + } + + #[test] + fn persists_bootstrap_cache_json_round_trip() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("official-bootstrap-cache.json"); + let bootstrap = fixture_bootstrap(); + let cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: bootstrap.launcher_metadata, + game_main_config: bootstrap.game_main_config, + }; + + write_bootstrap_cache(&path, &cache).unwrap(); + assert_eq!(read_bootstrap_cache(&path).unwrap(), Some(cache)); + } + + #[test] + fn bootstrap_cache_hit_requires_same_metadata_and_version() { + let bootstrap = fixture_bootstrap(); + let cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION, + launcher_metadata: bootstrap.launcher_metadata.clone(), + game_main_config: bootstrap.game_main_config.clone(), + }; + + assert_eq!( + cached_game_main_config_for_metadata(&cache, &bootstrap.launcher_metadata), + Some(bootstrap.game_main_config.clone()) + ); + + let mut changed_metadata = bootstrap.launcher_metadata.clone(); + changed_metadata.game_latest_version = "1.71.0".to_string(); + assert_eq!( + cached_game_main_config_for_metadata(&cache, &changed_metadata), + None + ); + + let stale_version_cache = OfficialBootstrapCache { + cache_version: OFFICIAL_BOOTSTRAP_CACHE_VERSION + 1, + ..cache + }; + assert_eq!( + cached_game_main_config_for_metadata( + &stale_version_cache, + &bootstrap.launcher_metadata + ), + None + ); + } + + #[test] + fn non_dry_run_lock_rejects_concurrent_writer() { + let temp = tempfile::TempDir::new().unwrap(); + let config = OfficialUpdateConfig { + output_root: temp.path().to_path_buf(), + ..OfficialUpdateConfig::default() + }; + + let first = OfficialUpdateLock::acquire(&config).unwrap(); + let second = OfficialUpdateLock::acquire(&config).unwrap_err(); + assert!(second.to_string().contains("locked")); + drop(first); + assert!(OfficialUpdateLock::acquire(&config).is_ok()); + } +} diff --git a/infrastructure/src/resources.rs b/infrastructure/src/resources.rs index 1ceeb3c..c5d2a37 100644 --- a/infrastructure/src/resources.rs +++ b/infrastructure/src/resources.rs @@ -1,9 +1,13 @@ //! 内存资源仓储实现。 use async_trait::async_trait; -use bat_core::domain::Resource; +use bat_core::domain::{Resource, ResourceEntry, ResourceType}; use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository}; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult}; +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::str::FromStr; use tokio::sync::RwLock; /// 以内存 `HashMap` 保存资源索引的仓储实现。 @@ -83,6 +87,359 @@ impl ResourceRepository for InMemoryResourceRepository { } } +/// SQLite 资源仓储实现。 +#[derive(Debug, Clone)] +pub struct SqliteResourceRepository { + pool: SqlitePool, +} + +impl SqliteResourceRepository { + /// 打开或创建 SQLite 资源仓储。 + pub async fn new(path: impl AsRef) -> bat_core::Result { + if let Some(parent) = path.as_ref().parent() { + tokio::fs::create_dir_all(parent).await?; + } + + let options = + SqliteConnectOptions::from_str(&format!("sqlite://{}", path.as_ref().display())) + .map_err(|error| bat_core::Error::Other(error.into()))? + .create_if_missing(true); + + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + + let repository = Self { pool }; + repository.init_schema().await?; + Ok(repository) + } + + async fn init_schema(&self) -> bat_core::Result<()> { + Self::execute_query( + &self.pool, + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS resources ( + id TEXT PRIMARY KEY NOT NULL, + path TEXT NOT NULL, + hash TEXT NOT NULL, + size INTEGER NOT NULL CHECK(size >= 0), + resource_type TEXT NOT NULL, + local_path TEXT NOT NULL, + address TEXT, + dependencies_json TEXT NOT NULL DEFAULT '[]' + ) + "#, + ), + ) + .await?; + + Self::execute_query( + &self.pool, + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_resources_hash + ON resources(hash) + "#, + ), + ) + .await?; + + Self::execute_query( + &self.pool, + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_resources_type + ON resources(resource_type) + "#, + ), + ) + .await?; + + Self::execute_query( + &self.pool, + sqlx::query( + r#" + CREATE INDEX IF NOT EXISTS idx_resources_path + ON resources(path) + "#, + ), + ) + .await?; + + Ok(()) + } + + async fn execute_query<'q>( + pool: &SqlitePool, + query: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>, + ) -> bat_core::Result { + query + .execute(pool) + .await + .map_err(|error| bat_core::Error::Other(error.into())) + } + + fn resource_type_to_str(resource_type: ResourceType) -> &'static str { + match resource_type { + ResourceType::AssetBundle => "AssetBundle", + ResourceType::Manifest => "Manifest", + ResourceType::TableBundle => "TableBundle", + ResourceType::Other => "Other", + } + } + + fn resource_type_from_str(value: &str) -> bat_core::Result { + match value { + "AssetBundle" => Ok(ResourceType::AssetBundle), + "Manifest" => Ok(ResourceType::Manifest), + "TableBundle" => Ok(ResourceType::TableBundle), + "Other" => Ok(ResourceType::Other), + other => Err(bat_core::Error::Serialization(format!( + "Unknown resource type: {}", + other + ))), + } + } + + fn dependencies_to_json(dependencies: &[String]) -> bat_core::Result { + serde_json::to_string(dependencies) + .map_err(|error| bat_core::Error::Serialization(error.to_string())) + } + + fn dependencies_from_json(value: &str) -> bat_core::Result> { + serde_json::from_str(value) + .map_err(|error| bat_core::Error::Serialization(error.to_string())) + } + + fn resource_from_row(row: ResourceRow) -> bat_core::Result { + let (id, path, hash, size, resource_type, local_path, address, dependencies_json) = row; + Ok(Resource { + id, + local_path: PathBuf::from(local_path), + entry: ResourceEntry { + path, + hash, + size: size as u64, + resource_type: Self::resource_type_from_str(&resource_type)?, + address, + dependencies: Self::dependencies_from_json(&dependencies_json)?, + }, + }) + } + + fn apply_filters<'a>( + builder: &mut QueryBuilder<'a, Sqlite>, + query: &'a ResourceQuery, + ) -> bat_core::Result<()> { + let mut has_where = false; + + if let Some(resource_type) = query.resource_type { + push_condition_prefix(builder, &mut has_where); + builder.push("resource_type = "); + builder.push_bind(Self::resource_type_to_str(resource_type)); + } + + if let Some(hash) = &query.hash { + push_condition_prefix(builder, &mut has_where); + builder.push("hash = "); + builder.push_bind(hash); + } + + if let Some(pattern) = &query.path_pattern { + push_condition_prefix(builder, &mut has_where); + builder.push("path LIKE "); + builder + .push_bind(glob_to_like(pattern)) + .push(" ESCAPE '\\'"); + } + + Ok(()) + } + + async fn fetch_resources( + &self, + query: &ResourceQuery, + limit: Option, + ) -> bat_core::Result> { + let mut builder = QueryBuilder::::new( + "SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json FROM resources", + ); + Self::apply_filters(&mut builder, query)?; + builder.push(" ORDER BY id"); + if let Some(limit) = limit { + builder.push(" LIMIT ").push_bind(limit as i64); + } + + let rows: Vec = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + + rows.into_iter().map(Self::resource_from_row).collect() + } + + async fn count_resources(&self, query: &ResourceQuery) -> bat_core::Result { + let mut builder = QueryBuilder::::new("SELECT COUNT(*) FROM resources"); + Self::apply_filters(&mut builder, query)?; + + let count: i64 = builder + .build_query_scalar() + .fetch_one(&self.pool) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + Ok(count as u64) + } +} + +#[async_trait] +impl ResourceRepository for SqliteResourceRepository { + async fn add(&self, resource: Resource) -> bat_core::Result { + let dependencies = Self::dependencies_to_json(&resource.entry.dependencies)?; + Self::execute_query( + &self.pool, + sqlx::query( + r#" + INSERT INTO resources ( + id, path, hash, size, resource_type, local_path, address, dependencies_json + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(id) DO UPDATE SET + path = excluded.path, + hash = excluded.hash, + size = excluded.size, + resource_type = excluded.resource_type, + local_path = excluded.local_path, + address = excluded.address, + dependencies_json = excluded.dependencies_json + "#, + ) + .bind(resource.id.clone()) + .bind(resource.entry.path.clone()) + .bind(resource.entry.hash.clone()) + .bind(resource.entry.size as i64) + .bind(Self::resource_type_to_str(resource.entry.resource_type)) + .bind(resource.local_path.to_string_lossy().to_string()) + .bind(resource.entry.address.clone()) + .bind(dependencies), + ) + .await?; + + Ok(resource.id) + } + + async fn find_by_id(&self, id: &str) -> bat_core::Result { + let row: Option = sqlx::query_as( + r#" + SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json + FROM resources + WHERE id = ?1 + "#, + ) + .bind(id) + .fetch_optional(&self.pool) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + + row.map(Self::resource_from_row) + .transpose()? + .ok_or_else(|| bat_core::Error::NotFound(id.to_string())) + } + + async fn find_by_hash(&self, hash: &str) -> bat_core::Result { + let row: Option = sqlx::query_as( + r#" + SELECT id, path, hash, size, resource_type, local_path, address, dependencies_json + FROM resources + WHERE hash = ?1 + ORDER BY id + LIMIT 1 + "#, + ) + .bind(hash) + .fetch_optional(&self.pool) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + + row.map(Self::resource_from_row) + .transpose()? + .ok_or_else(|| bat_core::Error::NotFound(hash.to_string())) + } + + async fn list(&self, query: ResourceQuery) -> bat_core::Result> { + self.fetch_resources(&query, None).await + } + + async fn update(&self, resource: Resource) -> bat_core::Result<()> { + let _existing = self.find_by_id(&resource.id).await?; + self.add(resource).await?; + Ok(()) + } + + async fn delete(&self, id: &str) -> bat_core::Result<()> { + let result = Self::execute_query( + &self.pool, + sqlx::query("DELETE FROM resources WHERE id = ?1").bind(id), + ) + .await?; + if result.rows_affected() == 0 { + return Err(bat_core::Error::NotFound(id.to_string())); + } + Ok(()) + } + + async fn count(&self, query: ResourceQuery) -> bat_core::Result { + self.count_resources(&query).await + } +} + +fn push_condition_prefix<'a>(builder: &mut QueryBuilder<'a, Sqlite>, has_where: &mut bool) { + if *has_where { + builder.push(" AND "); + } else { + builder.push(" WHERE "); + *has_where = true; + } +} + +type ResourceRow = ( + String, + String, + String, + i64, + String, + String, + Option, + String, +); + +fn glob_to_like(pattern: &str) -> String { + let mut escaped = String::new(); + let mut chars = pattern.chars().peekable(); + + while let Some(ch) = chars.next() { + match ch { + '*' => { + if matches!(chars.peek(), Some('*')) { + chars.next(); + } + escaped.push('%'); + } + '?' => escaped.push('_'), + '%' | '_' | '\\' => { + escaped.push('\\'); + escaped.push(ch); + } + other => escaped.push(other), + } + } + + escaped +} + fn query_matches(query: &ResourceQuery, resource: &Resource) -> bool { if let Some(resource_type) = query.resource_type { if resource.entry.resource_type != resource_type { @@ -142,6 +499,8 @@ mod tests { hash: hash.to_string(), size: 7, resource_type, + address: None, + dependencies: Vec::new(), }, } } @@ -214,4 +573,56 @@ mod tests { ); assert_eq!(repository.count(ResourceQuery::all()).await.unwrap(), 2); } + + async fn sqlite_repository() -> (tempfile::TempDir, SqliteResourceRepository) { + let temp_dir = tempfile::tempdir().unwrap(); + let repository = SqliteResourceRepository::new(temp_dir.path().join("resources.sqlite")) + .await + .unwrap(); + (temp_dir, repository) + } + + #[tokio::test] + async fn sqlite_repository_persists_and_filters_resources() { + let (_temp_dir, repository) = sqlite_repository().await; + let mut resource = resource( + "resource/sqlite-a", + "assets/model.bundle", + "hash-sqlite-a", + ResourceType::AssetBundle, + ); + resource.entry.address = Some("Character_001".to_string()); + resource + .entry + .dependencies + .push("assets/shared.bundle".to_string()); + + repository.add(resource.clone()).await.unwrap(); + + let by_id = repository.find_by_id(&resource.id).await.unwrap(); + assert_eq!(by_id.entry.address.as_deref(), Some("Character_001")); + assert_eq!( + by_id.entry.dependencies, + vec!["assets/shared.bundle".to_string()] + ); + assert_eq!( + repository.find_by_hash("hash-sqlite-a").await.unwrap().id, + resource.id + ); + + let bundles = repository + .list(ResourceQuery::by_type(ResourceType::AssetBundle)) + .await + .unwrap(); + assert_eq!(bundles.len(), 1); + + let count = repository.count(ResourceQuery::all()).await.unwrap(); + assert_eq!(count, 1); + + repository.delete(&resource.id).await.unwrap(); + assert!(matches!( + repository.find_by_id(&resource.id).await, + Err(bat_core::Error::NotFound(_)) + )); + } } diff --git a/infrastructure/tests/official_game_main_config_bootstrap.rs b/infrastructure/tests/official_game_main_config_bootstrap.rs index 4f941f3..b05f856 100644 --- a/infrastructure/tests/official_game_main_config_bootstrap.rs +++ b/infrastructure/tests/official_game_main_config_bootstrap.rs @@ -3,8 +3,8 @@ use bat_adapters::official::inventory::{ YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, }; use bat_adapters::official::yostar_jp::{ - is_official_yostar_jp_url, verified_official_platforms, PatchPlatform, YostarJpResourceEndpointKind, - YostarJpResourceDiscoveryPlan, YostarJpServerInfo, + is_official_yostar_jp_url, verified_official_platforms, PatchPlatform, + YostarJpResourceDiscoveryPlan, YostarJpResourceEndpointKind, YostarJpServerInfo, }; use bat_infrastructure::{ build_official_pull_plan_from_platform_inventory, OfficialGameMainConfigBootstrapService, @@ -40,8 +40,14 @@ fn fetch_bootstrap_uses_official_manifest_source_for_latest_zip() { let harness = TestHarness::new(); let bootstrap = harness.fetch_bootstrap(); - assert_eq!(bootstrap.game_config.game_latest_version, TEST_LAUNCHER_LATEST_VERSION); - assert_eq!(bootstrap.game_config.game_latest_file_path, TEST_LAUNCHER_LATEST_FILE_PATH); + assert_eq!( + bootstrap.game_config.game_latest_version, + TEST_LAUNCHER_LATEST_VERSION + ); + assert_eq!( + bootstrap.game_config.game_latest_file_path, + TEST_LAUNCHER_LATEST_FILE_PATH + ); assert_eq!(bootstrap.manifest_url, TEST_LAUNCHER_MANIFEST_URL); assert_eq!( bootstrap.manifest_source.as_deref(), @@ -54,7 +60,10 @@ fn fetch_bootstrap_uses_official_manifest_source_for_latest_zip() { Some(TEST_SERVER_INFO_URL) ); assert_eq!( - bootstrap.game_main_config.default_connection_group.as_deref(), + bootstrap + .game_main_config + .default_connection_group + .as_deref(), Some(TEST_CONNECTION_GROUP) ); assert_ne!( @@ -66,6 +75,28 @@ fn fetch_bootstrap_uses_official_manifest_source_for_latest_zip() { ); } +#[test] +fn fetch_bootstrap_falls_back_to_latest_file_path_when_manifest_source_is_not_zip() { + let harness = TestHarness::new_with_manifest_source("/BlueArchive_JP-1.70.436321-game"); + let bootstrap = harness.fetch_bootstrap(); + + assert_eq!( + bootstrap.manifest_source.as_deref(), + Some("/BlueArchive_JP-1.70.436321-game") + ); + assert_eq!( + bootstrap.game_zip_url, + format!( + "https://launcher-pkg-ba-jp.yo-star.com/{}", + TEST_LAUNCHER_LATEST_FILE_PATH + ) + ); + + let curl_log = fs::read_to_string(&harness.curl_log).unwrap(); + assert!(curl_log.contains(TEST_LAUNCHER_LATEST_FILE_PATH)); + assert!(!curl_log.contains(" /BlueArchive_JP-1.70.436321-game")); +} + #[test] fn official_pipeline_can_dry_run_and_pull_without_local_client() { let harness = TestHarness::new(); @@ -100,9 +131,7 @@ fn official_pipeline_can_dry_run_and_pull_without_local_client() { assert_eq!(plan.platforms, verified_official_platforms()); assert_eq!(all_urls.len(), 19); assert!(all_urls.iter().all(|url| is_official_yostar_jp_url(url))); - assert!(all_urls - .iter() - .all(|url| !url.contains("bluearchive.cafe"))); + assert!(all_urls.iter().all(|url| !url.contains("bluearchive.cafe"))); assert!(all_urls .iter() .any(|url| url.ends_with("/TableBundles/ExcelDB.db"))); @@ -132,6 +161,10 @@ struct TestHarness { impl TestHarness { fn new() -> Self { + Self::new_with_manifest_source(TEST_LAUNCHER_MANIFEST_SOURCE) + } + + fn new_with_manifest_source(manifest_source: &str) -> Self { let temp = TempDir::new().unwrap(); let resources_assets = synthetic_resources_assets(); @@ -145,11 +178,14 @@ impl TestHarness { let curl_script = temp.path().join("fake-curl"); write_executable( &curl_script, - &official_curl_script(&curl_log, &zip_fixture), + &official_curl_script(&curl_log, &zip_fixture, manifest_source), ); let unzip_script = temp.path().join("fake-unzip"); - write_executable(&unzip_script, &launcher_unzip_script(&resources_assets_path)); + write_executable( + &unzip_script, + &launcher_unzip_script(&resources_assets_path), + ); Self { temp, @@ -159,7 +195,9 @@ impl TestHarness { } } - fn fetch_bootstrap(&self) -> bat_infrastructure::official_game_main_config::OfficialGameMainConfigBootstrap { + fn fetch_bootstrap( + &self, + ) -> bat_infrastructure::official_game_main_config::OfficialGameMainConfigBootstrap { OfficialGameMainConfigBootstrapService::with_commands( TEST_LAUNCHER_VERSION, &self.curl_script, @@ -170,11 +208,17 @@ impl TestHarness { } fn fetcher(&self) -> OfficialResourcePullService { - OfficialResourcePullService::with_curl_command(self.temp.path().join("official-pull"), &self.curl_script) + OfficialResourcePullService::with_curl_command( + self.temp.path().join("official-pull"), + &self.curl_script, + ) } fn pull_service(&self) -> OfficialResourcePullService { - OfficialResourcePullService::with_curl_command(self.temp.path().join("pulled"), &self.curl_script) + OfficialResourcePullService::with_curl_command( + self.temp.path().join("pulled"), + &self.curl_script, + ) } } @@ -185,7 +229,7 @@ fn write_executable(path: &Path, content: &str) { fs::set_permissions(path, permissions).unwrap(); } -fn official_curl_script(curl_log: &Path, zip_fixture: &Path) -> String { +fn official_curl_script(curl_log: &Path, zip_fixture: &Path, manifest_source: &str) -> String { format!( r#"#!/usr/bin/env bash set -euo pipefail @@ -255,23 +299,23 @@ elif [[ "$url" == "{addressables_root}/Android_PatchPack/catalog_Android.hash" ] elif [[ "$url" == "{addressables_root}/TableBundles/TableCatalog.bytes" ]]; then emit_text "ExcelDB.db" elif [[ "$url" == "{addressables_root}/TableBundles/TableCatalog.hash" ]]; then - emit_text "tablecatalog-hash" + emit_text "{table_catalog_hash}" elif [[ "$url" == "{addressables_root}/Windows_PatchPack/BundlePackingInfo.bytes" ]]; then emit_text "FullPatch_000.zip" elif [[ "$url" == "{addressables_root}/Windows_PatchPack/BundlePackingInfo.hash" ]]; then - emit_text "bundlepackinginfo-win-hash" + emit_text "{windows_bundle_catalog_hash}" elif [[ "$url" == "{addressables_root}/MediaResources-Windows/Catalog/MediaCatalog.bytes" ]]; then emit_text "JP_Airi_Win.zip" elif [[ "$url" == "{addressables_root}/MediaResources-Windows/Catalog/MediaCatalog.hash" ]]; then - emit_text "mediacatalog-win-hash" + emit_text "{windows_media_catalog_hash}" elif [[ "$url" == "{addressables_root}/Android_PatchPack/BundlePackingInfo.bytes" ]]; then emit_text "FullPatch_001.zip" elif [[ "$url" == "{addressables_root}/Android_PatchPack/BundlePackingInfo.hash" ]]; then - emit_text "bundlepackinginfo-android-hash" + emit_text "{android_bundle_catalog_hash}" elif [[ "$url" == "{addressables_root}/MediaResources/Catalog/MediaCatalog.bytes" ]]; then emit_text "JP_Airi_Android.zip" elif [[ "$url" == "{addressables_root}/MediaResources/Catalog/MediaCatalog.hash" ]]; then - emit_text "mediacatalog-android-hash" + emit_text "{android_media_catalog_hash}" elif [[ -n "$output" && "$url" == "{addressables_root}/"* ]]; then filename="${{url##*/}}" if [[ "$filename" == *.zip ]]; then @@ -292,10 +336,15 @@ fi launcher_latest_file_path = TEST_LAUNCHER_LATEST_FILE_PATH, launcher_game_config_json_url = TEST_LAUNCHER_GAME_CONFIG_JSON_URL, launcher_manifest_url = TEST_LAUNCHER_MANIFEST_URL, - launcher_manifest_source = TEST_LAUNCHER_MANIFEST_SOURCE, + launcher_manifest_source = manifest_source, server_info_url = TEST_SERVER_INFO_URL, connection_group = TEST_CONNECTION_GROUP, - addressables_root = TEST_ADDRESSABLES_ROOT + addressables_root = TEST_ADDRESSABLES_ROOT, + table_catalog_hash = xxhash32(b"ExcelDB.db"), + windows_bundle_catalog_hash = xxhash32(b"FullPatch_000.zip"), + windows_media_catalog_hash = xxhash32(b"JP_Airi_Win.zip"), + android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"), + android_media_catalog_hash = xxhash32(b"JP_Airi_Android.zip"), ) } diff --git a/internal/ffi/ffi.go b/internal/ffi/ffi.go new file mode 100644 index 0000000..b895be4 --- /dev/null +++ b/internal/ffi/ffi.go @@ -0,0 +1,57 @@ +package ffi + +/* +#cgo CFLAGS: -I${SRCDIR}/../../target/bat-ffi +#cgo LDFLAGS: -L${SRCDIR}/../../target/debug -lbat_ffi +#include + +typedef char* ccharp; +extern const char* bat_version(); +extern void bat_free_string(char* s); +extern char* bat_manifest_inspect_json(const char* raw_json); +extern char* bat_sync_plan_json(const char* current_json, const char* previous_json); +*/ +import "C" +import ( + "errors" + "unsafe" +) + +func Version() (string, error) { + ptr := C.bat_version() + if ptr == nil { + return "", errors.New("bat_version returned nil") + } + defer C.bat_free_string((*C.char)(unsafe.Pointer(ptr))) + return C.GoString(ptr), nil +} + +func InspectManifest(rawJSON string) (string, error) { + cRaw := C.CString(rawJSON) + defer C.free(unsafe.Pointer(cRaw)) + + ptr := C.bat_manifest_inspect_json(cRaw) + if ptr == nil { + return "", errors.New("bat_manifest_inspect_json returned nil") + } + defer C.bat_free_string((*C.char)(unsafe.Pointer(ptr))) + return C.GoString(ptr), nil +} + +func BuildSyncPlan(currentJSON, previousJSON string) (string, error) { + cCurrent := C.CString(currentJSON) + defer C.free(unsafe.Pointer(cCurrent)) + + var cPrevious *C.char + if previousJSON != "" { + cPrevious = C.CString(previousJSON) + defer C.free(unsafe.Pointer(cPrevious)) + } + + ptr := C.bat_sync_plan_json(cCurrent, cPrevious) + if ptr == nil { + return "", errors.New("bat_sync_plan_json returned nil") + } + defer C.bat_free_string((*C.char)(unsafe.Pointer(ptr))) + return C.GoString(ptr), nil +}