diff --git a/CURRENT_STATUS.md b/CURRENT_STATUS.md index a13bf80..1d1b5be 100644 --- a/CURRENT_STATUS.md +++ b/CURRENT_STATUS.md @@ -20,7 +20,7 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 3. 默认平台为 `Windows + Android`。 4. 能生成官方全量 pull plan，执行真实下载，维护 release 内的 `official-download-manifest.json`。 5. 下载后使用本地 manifest 的 size + BLAKE3 校验复用文件；所有 `.zip` 在下载验收、复用、本地 audit/verify 时做 ZIP 结构校验；官方 seed `.hash` 使用标准 `xxHash32(seed=0)` 强校验（早期实现的非标准 avalanche 常量已修正）。 -6. 支持 `.part` 断点续传、失败后 clean retry、本地 manifest audit/repair、失败 staging 恢复复用、403/404/5xx 分类重试（重试带指数退避）、下载 quarantine 诊断，以及旧 launcher 包官方 primary/backup CDN 切换。启动器/server-info 先行更新但 client-patch seed marker 或必需 seed catalog 尚未开放时，会进入 `waiting_for_official_resources`，保留现有 `current`，不创建失败 staging，也不写入失败版本循环；启用 `--auto-discover` 的非 dry-run 会写入 `/official-launcher-bootstrap.pending.json` 作为维护期证据。下载执行保持顺序处理；manifest/quarantine 簿记与 seed `.hash` 校验仍逐项执行，`fail-fast` 与「不发布不完整资源」不变量不变。下载进度按已完成数量单调上报，不再使用 plan 序号计算百分比。 +6. 支持 `.part` 断点续传、失败后 clean retry、本地 manifest audit/repair、失败 staging 恢复复用、403/404/5xx 分类重试（重试带指数退避）、下载 quarantine 诊断，以及旧 launcher 包官方 primary/backup CDN 切换。启动器/server-info 先行更新但 client-patch seed marker 或必需 seed catalog 尚未开放时，会进入 `waiting_for_official_resources`，保留现有 `current`，不创建失败 staging，也不写入失败版本循环；启用 `--auto-discover` 的非 dry-run 会写入 `/official-launcher-bootstrap.pending.json` 作为维护期证据。下载默认使用 8 个独立 worker，范围为 `1..=256`；每个 worker 完成当前 URL 后立即从共享计划队列领取下一个任务，进度按实际完成顺序即时上报，最终 report 资源列表仍按计划顺序输出。manifest/quarantine 簿记与 seed `.hash` 校验仍逐项执行，`fail-fast` 与「不发布不完整资源」不变量不变。下载进度按已完成数量单调上报，不再使用 plan 序号计算百分比。 7. 支持 curl 传输层本地代理：默认自动检测 `HTTPS_PROXY` / `ALL_PROXY` / `HTTP_PROXY` 及小写环境变量（带凭据的代理推荐用环境变量配置），也可用 `--proxy ` 显式指定或 `--no-proxy` 强制直连；代理决策会写入 progress log、daemon log 和 `bat doctor` 诊断输出。代理凭据不落世界可读位置：日志/`status` 脱敏，传给 curl 经 `ALL_PROXY` 环境变量而非 argv，`--daemon` 下经环境变量下传后台子进程、不进子进程 argv 或 `bat-status.json`，复用凭据存于 `bat-proxy.secret`（`0600`）且 `clean-stable` 会清除。 8. `bat --watch` 可常驻运行，`bat --daemon` 可后台运行并用 `bat status` / `bat stop` / `bat restart` / `bat reload` / `bat logs` 管理；daemon 使用 `bat.sock` Unix socket JSON-RPC 作为 live 控制通道，PID/状态/日志文件作为快照和 fallback，`bat-events.jsonl` 记录带轮转的结构化事件日志，`bat-control.lock` 串行化控制命令；正常检查默认每 1 小时一次；远端和本地一致时默认静默，失败后默认 60 秒快速重试，官方资源端尚未开放时状态为 `waiting` 并同样按错误重试间隔探测；`resource.state` / `catalog.status` / `parse.status` / `localized.status` 会返回 `status` 与稳定 `status_code`（如 `official.up_to_date`、`official.published`、`parse.completed`、`translation.queued_offline`、`localized.published`、`distribution.ready`），供 `bat-api` 等读侧判断阶段、终态和重试属性；CLI 默认向 stdout 输出人类可读摘要，向 stderr 输出 ASCII banner、progress log、失败分类和 quarantine 状态，需要机器输出时使用 `--json --no-progress`。 9. 远端 snapshot 未变化但输出目录为空时，会按首次运行执行全量拉取；官方 seed `.hash` 校验失败时会清理对应 manifest 条目，避免失败产物被后续本地 audit 误判为可复用。 @@ -32,6 +32,11 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 仍需明确：这不是完整产品完成。完整 AssetBundle 重打包、翻译、Web、以及 `bat-api` 的服务器联调/可选业务扩展仍是后续工作；G-008（产品级 Go 同步 CLI）已决策关闭。真实官方网络全量拉取 smoke 已固化（G-018 已关闭）；真实大文件与运行报告默认在 `/tmp` 隔离目录，不纳入 Git。Go 细节见 `docs/reports/GO_STATUS.md`。 +当前翻译交接还包括 `translation-tasks.sqlite` 和版本化 `translation-handoff.json`； +`translation.tasks` 查询单项 worker 状态，`translation.handoff` 查询完整 +job/unit/provider run 状态。当前下载实现使用默认 8 个独立 worker，完成后动态领取 +任务，最终资源报告按 pull plan 顺序输出。 + --- ## 2. 权威文档入口 @@ -281,7 +286,7 @@ cargo run -p bat-infrastructure --bin bat -- \ ## 6. 当前阻塞项 -GitHub issue 状态：#1 已关闭；#17 已按 wontfix 关闭（多线程下载入口已移除，下载回归顺序执行并保留指数退避与单调进度上报，子 issue #20–#23 均已关闭）。其他 open issue 的实时标签以 GitHub 为准。 +GitHub issue 状态：#1 已关闭；#17 的历史决定不代表当前下载实现，现行默认并发为 8，范围 `1..=256`，每个独立 worker 完成后立即领取下一个任务，进度按完成事件即时统计并保持 report 计划顺序；子 issue #20–#23 均已关闭。其他 open issue 的实时标签以 GitHub 为准。 下一阶段必须优先完成： diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index 029292c..f45bfce 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -174,7 +174,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 1. Addressables Catalog 真实字段解析：**部分完成**。当前已覆盖 path、hash、size、address、dependencies、metadata 和真实形态 fixture/golden；仍需继续覆盖更多官方 catalog 结构变体。 2. 资源版本、区域、渠道、远端 URL、Hash、大小、依赖关系模型：**部分完成**。`Resource` 和官方 endpoint/snapshot 模型已扩展；仍需冻结 Go CLI/API 可见模型。 -3. Rust 官方下载器：**已完成当前生产入口需要的核心能力**。包含官方 URL 校验、`.part` 续传、重试、本地 manifest size+BLAKE3 校验、官方 seed `.hash` 校验和 repair。 +3. Rust 官方下载器：**已完成当前生产入口需要的核心能力**。包含官方 URL 校验、`.part` 续传、重试、本地 manifest size+BLAKE3 校验、官方 seed `.hash` 校验、repair，以及默认 8、范围 `1..=256` 的有界并发 scheduler；worker 动态领取任务，进度按完成数单调上报，report 保持 plan 顺序。 4. Rust 自动更新入口：**已完成当前生产入口**。`bat` 支持 snapshot、marker diff、bootstrap cache、one-shot、`--watch`、`--daemon`、默认 1 小时间隔、北京时间固定强制刷新，以及 Unix socket JSON-RPC 后台运维命令返回。 5. Go 入口边界：**已冻结**。同步命令行 = Rust `bat`（G-008 关闭）；资源分发 = `bat-api` MVP（G-009 部分完成）。详见 `docs/reports/GO_STATUS.md`。 6. 用户级 `sync`、`manifest inspect`、`cache status`：**未完成**。Rust `bat --json` 是当前稳定进程边界；`bat-ffi` 只提供可选兼容用的 Manifest inspect 和 sync plan JSON helper。 diff --git a/README.md b/README.md index 7b5a56b..9a5e798 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ - `bat-adapters` Unity、Manifest、Client 集成框架，以及当前真实形态 Addressables catalog 解析覆盖，含 `m_Crc` 提取和 UnityFS 解包/TextAsset 提取基础校验。 - `bat-cas-engine` CAS V1：原子写入、BLAKE3 校验、引用计数、GC、并发写入测试、损坏检测。 - `bat-infrastructure` CAS 适配层、SQLite Resource Repository、资源导入服务、官方资源 pull/update 服务。 -- `bat`：官方资源自动发现、全量拉取、原子发布到 `current -> versions/`、本地 manifest audit/repair、`.part` 断点续传、403/404/5xx 分类重试、指数退避、顺序下载、下载 quarantine 诊断、ZIP 结构校验、官方 seed `.hash` 校验、snapshot/cache、版本化 `official-launcher-bootstrap.json`、`--watch` 常驻更新、`--daemon` 后台运行，以及 Unix socket JSON-RPC live control/backend 方法（`daemon.status/logs/stop/restart/reload/refresh/doctor`、`resource.sync/verify/repair/state/manifest/list/index`、`parse.status/text_units/errors`、`localized.status`、`catalog.*`、`task.*`）。 +- `bat`：官方资源自动发现、全量拉取、原子发布到 `current -> versions/`、本地 manifest audit/repair、`.part` 断点续传、403/404/5xx 分类重试、指数退避、默认并发 8（可配置 `1..=256`，report 按 plan 顺序、进度按完成数单调上报）、下载 quarantine 诊断、ZIP 结构校验、官方 seed `.hash` 校验、snapshot/cache、版本化 `official-launcher-bootstrap.json`、`--watch` 常驻更新、`--daemon` 后台运行，以及 Unix socket JSON-RPC live control/backend 方法（`daemon.status/logs/stop/restart/reload/refresh/doctor`、`resource.sync/verify/repair/state/manifest/list/index`、`parse.status/text_units/errors`、`localized.status`、`catalog.*`、`task.*`）。 - `internal/backendrpc`：Go 侧 typed Unix socket JSON-RPC client，是 `bat-api` 调用 Rust daemon 的默认路径。 - `cmd/bat-api`：资源 bootstrap + 分发 HTTP MVP（issue #19 / G-009）；`/v1/bootstrap` 和 `/v1/launcher/bootstrap` 组织 `bat` 已发布 release 的启动前资源入口，launcher 形状兼容端点仅输出资源 metadata / GameMainConfig 引导，`/healthz` 暴露 RPC refresh 诊断，`/readyz` 做 release readiness，CDN path 支持 `GET`/`HEAD`/`Range`、ETag、Last-Modified 和缓存头；玩家-facing 控制面已具备 token 鉴权、限流、访问日志、反代 IP 适配、动态 JSON no-store、OpenAPI 和管理控制白名单；`.env` 配置端口/RPC socket/刷新周期；生产资源根来自 RPC，不负责自动拉取。 - Go 边界权威说明：[`docs/reports/GO_STATUS.md`](docs/reports/GO_STATUS.md)（G-008 已关闭：同步 CLI = Rust `bat`）。 - 官方同步会维护 `/official-version-state.json`，明确记录当前已完成版本、正在拉取版本、上一个可用版本和失败版本。 - 资源导入链路可配置为在官方 release 发布后写入 CAS + `ResourceRepository`，资源 metadata 会记录 release、平台、bundle path、parse status、TextAsset 名称和 TextUnit 数量/格式，TextAsset/Table/Media 会按类型分类索引；`resource.index` RPC/CLI 可按类型、hash、路径模式、官方 release ID、平台、destination、archive entry、parse status 和 TextUnit format 分页查询索引。 -- 新 release 发布后会生成 `official-resource-changes.json`、`official-parse-cache.json`、`official-textunit-index.json`、`official-textunit-tasks.json`、`crowdin-translation-handoff.json` 和 `crowdin-textunit-queue.json`；其中 TextUnit/Crowdin 队列只使用 Added/Modified 资源，不调用 Crowdin 网络 API，离线 TextUnit 翻译任务可通过 `translation.tasks` RPC/CLI 查询状态和跳过/失败原因。 +- 新 release 发布后会生成 `official-resource-changes.json`、`official-parse-cache.json`、`official-textunit-index.json`、`official-textunit-tasks.json`、`crowdin-translation-handoff.json`、`crowdin-textunit-queue.json`、`translation-tasks.sqlite` 和 `translation-handoff.json`；其中 TextUnit/Crowdin 队列只使用 Added/Modified 资源，不调用 Crowdin 网络 API，离线 TextUnit 翻译任务可通过 `translation.tasks` / `translation.handoff` RPC 或 CLI 查询状态、跳过/失败原因和 provider run 交接。 - `LocalizedPatchService` 已具备 UnityFS TextAsset patch 发布前置能力：在 `--localized-output` / `BAT_LOCALIZED_OUTPUT` 配置的独立汉化目录 staging 中复制官方 release、应用 TextAsset patch、写 `localized-patch-manifest.json`（hash、size、diff、rollback）、校验后发布到 `versions/` 并切换 `current`。 - `bat-patch` 已具备通用 Patch 基础：确定性 Binary hunk diff/apply、RFC 6902 JSON Patch apply、UTF-8 Text Patch、Patch manifest、BLAKE3/size 完整性校验和 rollback 元数据；文件级 `patch.apply` RPC / `patch-apply` CLI 与 UnityFS TextAsset / TypeTree string / TypeTree 语义字段写入入口已开放，TypeTree 语义字段支持基础标量、固定 Unity float/int/hash 值类型的 leaf/direct-child 形态、PPtr、managed-reference registry payload 字符串、object 字段组合、unknown fixed-size raw bytes 同长度替换和 TypeTree schema 支撑的 array/vector/map 整体替换；TextUnit 提取会把 managed-reference 类型信息保留为上下文而非翻译文本，汉化发布当前仍走 UnityFS TextAsset 前置链路。 - `bat-ffi` 可选无状态 C ABI 兼容层：仅保留 Manifest inspect 和官方 sync plan 的粗粒度 JSON helper，不作为 Go CLI 或生产同步的主集成边界。 diff --git a/adapters/src/client/discovery.rs b/adapters/src/client/discovery.rs index 0087aa8..e62ac2f 100644 --- a/adapters/src/client/discovery.rs +++ b/adapters/src/client/discovery.rs @@ -2,6 +2,45 @@ use async_trait::async_trait; use bat_core::domain::{GameClient, GameRegion}; +use std::path::PathBuf; + +/// Linux-first client discovery backed by explicitly supplied roots. +/// +/// The adapter never scans home directories implicitly and does not require +/// the official launcher. The roots are normally a staging/import directory +/// selected by the caller. +#[derive(Debug, Clone, Default)] +pub struct LinuxClientDiscovery { + roots: Vec, +} + +impl LinuxClientDiscovery { + /// Creates a discovery adapter for explicit candidate roots. + pub fn new(roots: Vec) -> Self { + Self { roots } + } +} + +#[async_trait] +impl ClientDiscovery for LinuxClientDiscovery { + async fn discover_all(&self) -> Result, String> { + GameClient::discover_in_roots(&self.roots).map_err(|error| error.to_string()) + } + + async fn verify_client(&self, path: &str) -> bool { + GameClient::new(PathBuf::from(path), GameRegion::Japan) + .verify_integrity() + .unwrap_or(false) + } + + async fn detect_region(&self, path: &str) -> Result { + if self.verify_client(path).await { + Ok(GameRegion::Japan) + } else { + Err(format!("不是有效的 Linux Blue Archive 客户端：{path}")) + } + } +} /// 客户端发现接口 /// @@ -49,5 +88,44 @@ pub trait ClientDiscovery: Send + Sync { #[cfg(test)] mod tests { - // 测试将在实现时添加 + use super::*; + use std::fs; + use tempfile::TempDir; + + #[tokio::test] + async fn linux_discovery_uses_explicit_roots_and_verifies_layout() { + let temp = TempDir::new().unwrap(); + let client = temp.path().join("BlueArchive_JP"); + fs::create_dir_all(client.join("BlueArchive_Data/StreamingAssets/AssetBundles")).unwrap(); + let discovery = LinuxClientDiscovery::new(vec![temp.path().to_path_buf()]); + + let clients = discovery.discover_all().await.unwrap(); + assert_eq!(clients.len(), 1); + assert!( + discovery + .verify_client(clients[0].install_path.to_str().unwrap()) + .await + ); + assert_eq!( + discovery + .detect_region(clients[0].install_path.to_str().unwrap()) + .await + .unwrap(), + GameRegion::Japan + ); + } + + #[tokio::test] + async fn linux_discovery_rejects_unrelated_path() { + let discovery = LinuxClientDiscovery::default(); + assert!( + !discovery + .verify_client("/tmp/not-a-blue-archive-client") + .await + ); + assert!(discovery + .detect_region("/tmp/not-a-blue-archive-client") + .await + .is_err()); + } } diff --git a/adapters/src/official/backend.rs b/adapters/src/official/backend.rs new file mode 100644 index 0000000..e48fa03 --- /dev/null +++ b/adapters/src/official/backend.rs @@ -0,0 +1,208 @@ +//! Official resource backend seams. +//! +//! The update pipeline consumes these small contracts instead of depending on +//! one region's URL and catalog rules everywhere. The JP implementation is +//! the only production adapter today; adding another region should implement +//! this module's contracts without changing downloader orchestration. + +use super::inventory::{YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory}; +use super::yostar_jp::{ + is_official_yostar_jp_url, server_info_url, PatchPlatform, YostarJpResourceDiscoveryPlan, + YostarJpResourceRoot, YostarJpServerInfo, +}; +use std::path::{Path, PathBuf}; + +/// Catalog bytes required to build one platform's download inventory. +#[derive(Debug, Clone, Copy)] +pub struct PlatformCatalogInput<'a> { + /// Platform represented by the catalog. + pub platform: PatchPlatform, + /// `BundlePackingInfo.bytes` payload. + pub bundle_packing_info: &'a [u8], + /// `MediaCatalog.bytes` payload. + pub media_catalog: &'a [u8], +} + +/// Region/backend contract used by official resource orchestration. +pub trait OfficialResourceBackend: Send + Sync { + /// Stable backend identifier persisted in diagnostics. + fn backend_id(&self) -> &'static str; + + /// Builds the server-info URL from an official metadata file name. + fn server_info_url(&self, file_name: &str) -> Result; + + /// Selects a discovery plan from server-info and requested platforms. + fn discovery_plan( + &self, + server_info: &YostarJpServerInfo, + connection_group: &str, + app_version: &str, + platforms: &[PatchPlatform], + ) -> Result; + + /// Builds platform-aware inventory from the verified seed catalog bytes. + fn inventory( + &self, + table_catalog: &[u8], + platform_catalogs: &[PlatformCatalogInput<'_>], + ) -> YostarJpPlatformDownloadInventory; + + /// Validates that a URL belongs to this backend's official hosts. + fn is_official_url(&self, url: &str) -> bool; +} + +/// URL-to-destination mapping contract for a resource backend. +pub trait DownloadUrlMapper: Send + Sync { + /// Maps an official HTTPS URL to a relative release destination. + fn relative_destination(&self, url: &str) -> Result; +} + +/// The currently supported official Blue Archive JP backend. +#[derive(Debug, Clone, Copy, Default)] +pub struct YostarJpBackend; + +impl OfficialResourceBackend for YostarJpBackend { + fn backend_id(&self) -> &'static str { + "bluearchive.yostar.jp" + } + + fn server_info_url(&self, file_name: &str) -> Result { + server_info_url(file_name) + } + + fn discovery_plan( + &self, + server_info: &YostarJpServerInfo, + connection_group: &str, + app_version: &str, + platforms: &[PatchPlatform], + ) -> Result { + server_info.discovery_plan(connection_group, app_version, platforms) + } + + fn inventory( + &self, + table_catalog: &[u8], + platform_catalogs: &[PlatformCatalogInput<'_>], + ) -> YostarJpPlatformDownloadInventory { + let catalogs = platform_catalogs + .iter() + .map(|catalog| { + YostarJpPlatformCatalogInventory::from_catalog_bytes( + catalog.platform, + catalog.bundle_packing_info, + catalog.media_catalog, + ) + }) + .collect(); + YostarJpPlatformDownloadInventory::from_catalog_bytes(table_catalog, catalogs) + } + + fn is_official_url(&self, url: &str) -> bool { + is_official_yostar_jp_url(url) + } +} + +impl DownloadUrlMapper for YostarJpBackend { + fn relative_destination(&self, url: &str) -> Result { + let rest = url + .strip_prefix("https://") + .ok_or_else(|| format!("官方 URL 必须使用 https：{url}"))?; + let (host, path) = rest + .split_once('/') + .ok_or_else(|| format!("官方 URL 缺少路径：{url}"))?; + let mut destination = PathBuf::from(sanitize_component(host, url)?); + for segment in path.split('/') { + if segment.is_empty() { + continue; + } + destination.push(sanitize_component(segment, url)?); + } + Ok(destination) + } +} + +impl YostarJpBackend { + /// Returns the validated resource-root builder for an official root. + pub fn resource_root(&self, addressables_root: &str) -> Result { + YostarJpResourceRoot::from_addressables_root(addressables_root) + } +} + +fn sanitize_component(component: &str, url: &str) -> Result { + if component == "." || component == ".." || component.is_empty() { + return Err(format!("官方 URL 包含不安全路径片段：{url}")); + } + if component.contains('?') || component.contains('#') || component.contains('\\') { + return Err(format!("官方资源 URL 包含不安全路径字符：{url}")); + } + Ok(component.to_string()) +} + +/// Joins a backend-relative destination below an output root. +pub fn destination_under_root(root: &Path, relative: &Path) -> Result { + if relative.is_absolute() { + return Err(format!( + "backend destination must be relative: {}", + relative.display() + )); + } + let destination = root.join(relative); + if destination + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + { + return Err(format!( + "backend destination escapes output root: {}", + relative.display() + )); + } + Ok(destination) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn jp_backend_keeps_discovery_and_inventory_rules_in_one_adapter() { + let backend = YostarJpBackend; + let server_info = YostarJpServerInfo::from_json( + r#"{"ConnectionGroups":[{"Name":"Prod","AddressablesCatalogUrlRoot":"https://prod-clientpatch.bluearchiveyostar.com/r93_fixture"}]}"#, + ) + .unwrap(); + let plan = backend + .discovery_plan(&server_info, "Prod", "1.70.0", &[PatchPlatform::Windows]) + .unwrap(); + assert_eq!(backend.backend_id(), "bluearchive.yostar.jp"); + assert!(backend.is_official_url(&plan.endpoints[0].url)); + + let inventory = backend.inventory( + b"ExcelDB.db ExcelDB.db", + &[PlatformCatalogInput { + platform: PatchPlatform::Windows, + bundle_packing_info: b"FullPatch_000.zip", + media_catalog: b"GameData/Audio/JP.zip", + }], + ); + assert_eq!(inventory.table_file_names, vec!["ExcelDB.db"]); + assert_eq!(inventory.platform_catalogs.len(), 1); + } + + #[test] + fn jp_backend_maps_and_rejects_unsafe_destinations() { + let backend = YostarJpBackend; + assert_eq!( + backend + .relative_destination( + "https://prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/a.bytes" + ) + .unwrap(), + PathBuf::from("prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/a.bytes") + ); + assert!(backend + .relative_destination("https://prod-clientpatch.bluearchiveyostar.com/r93/../secret") + .is_err()); + assert!(!backend.is_official_url("https://example.invalid/a")); + } +} diff --git a/adapters/src/official/mod.rs b/adapters/src/official/mod.rs index 3c4aa42..755873a 100644 --- a/adapters/src/official/mod.rs +++ b/adapters/src/official/mod.rs @@ -4,11 +4,16 @@ //! client endpoints. Mirror-specific layers such as `bluearchive.cafe` or //! `text=jp/voice=jp/media=jp` are intentionally excluded. +pub mod backend; pub mod game_main_config; pub mod inventory; pub mod launcher; pub mod yostar_jp; +pub use backend::{ + destination_under_root, DownloadUrlMapper, OfficialResourceBackend, PlatformCatalogInput, + YostarJpBackend, +}; pub use game_main_config::YostarJpGameMainConfig; pub use inventory::{ YostarJpDownloadInventory, YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, diff --git a/core/src/domain/game_client.rs b/core/src/domain/game_client.rs index f749664..d77bb10 100644 --- a/core/src/domain/game_client.rs +++ b/core/src/domain/game_client.rs @@ -1,6 +1,11 @@ //! 游戏客户端领域对象 -use std::path::PathBuf; +use std::collections::HashSet; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +const CLIENT_ROOTS_ENV: &str = "BAT_CLIENT_ROOTS"; /// 游戏区域 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -69,18 +74,58 @@ impl GameClient { } } - /// 发现本地安装的客户端 + /// 发现显式配置根目录下的本地客户端。 /// /// # 返回 /// - 成功：返回找到的所有客户端 /// - 失败：返回错误 /// - /// # 注意 - /// 此功能将在 Phase 3 实现 + /// 默认不扫描系统目录。调用方必须通过 `BAT_CLIENT_ROOTS` 提供一个或 + /// 多个路径；路径格式使用平台原生路径分隔符。没有配置时返回空列表。 pub fn discover() -> crate::Result> { - Err(crate::Error::NotImplemented( - "客户端发现功能将在 Phase 3 实现".to_string(), - )) + let Some(value) = env::var_os(CLIENT_ROOTS_ENV) else { + return Ok(Vec::new()); + }; + let roots = env::split_paths(&value).collect::>(); + Self::discover_in_roots(&roots) + } + + /// 在调用方明确提供的隔离根目录下发现客户端。 + /// + /// 每个根目录只检查根本身和它的直接子目录，不递归扫描用户目录。 + /// 当前核心模型的默认发现区域为日本服；其他区域应由适配器提供 + /// 专用区域识别策略。 + pub fn discover_in_roots(roots: &[PathBuf]) -> crate::Result> { + let mut candidates = Vec::new(); + let mut seen = HashSet::new(); + + for root in roots { + if !is_real_directory(root)? || has_symlink_component(root)? { + continue; + } + if seen.insert(root.clone()) { + candidates.push(root.clone()); + } + + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if !is_real_directory(&path)? || has_symlink_component(&path)? { + continue; + } + if seen.insert(path.clone()) { + candidates.push(path); + } + } + } + + let mut clients = Vec::new(); + for path in candidates { + if client_layout_is_present(&path)? { + clients.push(GameClient::new(path, GameRegion::Japan)); + } + } + Ok(clients) } /// 验证客户端完整性 @@ -89,12 +134,11 @@ impl GameClient { /// - true: 客户端完整 /// - false: 客户端损坏 /// - /// # 注意 - /// 此功能将在 Phase 3 实现 pub fn verify_integrity(&self) -> crate::Result { - Err(crate::Error::NotImplemented( - "完整性验证将在 Phase 3 实现".to_string(), - )) + if !is_real_directory(&self.install_path)? || has_symlink_component(&self.install_path)? { + return Ok(false); + } + Ok(client_layout_is_present(&self.install_path)?) } /// 获取 StreamingAssets 目录路径 @@ -113,6 +157,8 @@ impl GameClient { #[cfg(test)] mod tests { use super::*; + use std::fs; + use tempfile::TempDir; #[test] fn test_game_region_code() { @@ -153,12 +199,77 @@ mod tests { } #[test] - fn test_discover_not_implemented() { - let result = GameClient::discover(); - assert!(result.is_err()); - assert!(matches!( - result.unwrap_err(), - crate::Error::NotImplemented(_) - )); + fn test_discover_without_explicit_roots_is_empty() { + // discover() 不得因为测试机或用户 home 中存在目录而扫描它们。 + assert!(GameClient::discover_in_roots(&[]).unwrap().is_empty()); + } + + #[test] + fn test_discover_and_verify_isolated_client_layout() { + let temp = TempDir::new().unwrap(); + let client_root = temp.path().join("BlueArchive_JP"); + fs::create_dir_all(client_root.join("BlueArchive_Data/StreamingAssets/AssetBundles")) + .unwrap(); + + let clients = GameClient::discover_in_roots(&[temp.path().to_path_buf()]).unwrap(); + assert_eq!(clients.len(), 1); + assert_eq!(clients[0].install_path, client_root); + assert_eq!(clients[0].region, GameRegion::Japan); + assert!(clients[0].verify_integrity().unwrap()); + } + + #[test] + fn test_integrity_rejects_incomplete_layout() { + let temp = TempDir::new().unwrap(); + let client = GameClient::new(temp.path().join("missing"), GameRegion::Japan); + assert!(!client.verify_integrity().unwrap()); + } + + #[cfg(unix)] + #[test] + fn test_discovery_and_integrity_reject_symlinked_client() { + use std::os::unix::fs::symlink; + + let temp = TempDir::new().unwrap(); + let real = temp.path().join("real"); + fs::create_dir_all(real.join("BlueArchive_Data/StreamingAssets/AssetBundles")).unwrap(); + let link = temp.path().join("link"); + symlink(&real, &link).unwrap(); + + let clients = GameClient::discover_in_roots(&[temp.path().to_path_buf()]).unwrap(); + assert_eq!(clients.len(), 1); + assert_eq!(clients[0].install_path, real); + assert!(!GameClient::new(link, GameRegion::Japan) + .verify_integrity() + .unwrap()); } } + +fn client_layout_is_present(path: &Path) -> crate::Result { + Ok(is_real_directory(&path.join("BlueArchive_Data"))? + && is_real_directory(&path.join("BlueArchive_Data/StreamingAssets"))? + && is_real_directory(&path.join("BlueArchive_Data/StreamingAssets/AssetBundles"))? + && !has_symlink_component(path)?) +} + +fn is_real_directory(path: &Path) -> crate::Result { + match fs::symlink_metadata(path) { + Ok(metadata) => Ok(metadata.is_dir() && !metadata.file_type().is_symlink()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn has_symlink_component(path: &Path) -> crate::Result { + let mut current = PathBuf::new(); + for component in path.components() { + current.push(component.as_os_str()); + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => return Ok(true), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + } + } + Ok(false) +} diff --git a/docs/architecture/official-resource-backend.md b/docs/architecture/official-resource-backend.md index 1d5c536..5f36cc1 100644 --- a/docs/architecture/official-resource-backend.md +++ b/docs/architecture/official-resource-backend.md @@ -146,6 +146,12 @@ 17. 旧 launcher 包或 `resources.assets` 下载使用官方 launcher CDN 配置，primary CDN 失败后切换 official backup CDN；资源 patch host 不猜测非官方镜像。 18. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 19. 非官方 URL 直接拒绝。 +20. 下载调度默认并发数为 `8`，允许范围是 `1..=256`，由 + `--download-concurrency` / `BAT_DOWNLOAD_CONCURRENCY` 配置。worker 从共享 + plan 队列逐项领取任务，单个任务完成后立即领取下一个，不等待其他 worker + 的当前任务；完成结果在协调线程即时更新 manifest、hash 事件和进度计数。 + 最终 `OfficialResourcePullReport.items` 仍按 `OfficialResourcePullPlan` + 顺序排列，避免并发完成顺序泄露到发布和 API 读侧。 路径映射时会做分段清理，并在写入前做输出目录安全校验、相对路径归属校验和现有路径组件 symlink 检查，避免把不安全路径写进输出目录或通过 symlink 跳出输出目录。 @@ -171,9 +177,12 @@ 这层的意义是把“下载到磁盘的文件”变成“可查询、可复用、可去重”的资源对象。 `resource.index` RPC / CLI 只读查询现有 SQLite 索引；索引不存在时返回 -`available=false`，不会因为查询创建空库。`translation.tasks` RPC / CLI -可查询离线 TextUnit 任务状态和跳过/失败 reason；G-011 剩余工作是真实 -worker 状态写回、CAS 诊断入口和面向大索引的查询优化。 +`available=false`，不会因为查询创建空库。发布后的 TextUnit 队列还会在当前 +release 根目录写入 `translation-tasks.sqlite`，由版本化 `schema_migrations` +管理 queued/running/failed/completed/skipped 和 failure reason。`translation.tasks` +优先查询这份状态库，`translation.task.update` 供 provider worker 回写状态； +没有状态库的旧 release 才回退到 immutable JSON 队列。G-011 剩余工作是 CAS +诊断入口和面向大索引的查询优化。 对应实现主要在： @@ -226,7 +235,7 @@ worker 状态写回、CAS 诊断入口和面向大索引的查询优化。 11. 下载、manifest、本地 BLAKE3、ZIP 和官方 `.hash` 校验完成后写入新的 snapshot，并在 staging 中写入 `official-launcher-bootstrap.json`（若本轮启用 `--auto-discover`）。 12. 将 staging rename 为 `/versions/`，再原子替换 `/current` symlink 指向该 versioned 目录。 13. 发布完成后先对比上一完整 release 和当前 release 的 `official-download-manifest.json`，写出 `official-resource-changes.json` 和 `crowdin-translation-handoff.json`。同一 destination 只有 size 或 BLAKE3 变化才算 modified；新增+变更资源进入解析/翻译 handoff，删除资源只进入差异记录。当前只预留 Crowdin 本地 handoff，不发外部 API 请求。 -14. 随后刷新 active release 下的 `official-parse-cache.json` 和 `official-textunit-index.json`，并从 Added/Modified 资源、parse cache 与 TextUnit 明细索引派生 `official-textunit-tasks.json` 和 `crowdin-textunit-queue.json`；up-to-date 轮询在已有有效解析缓存、TextUnit 明细索引和 TextUnit 队列时只读取摘要，不重复解析。 +14. 随后刷新 active release 下的 `official-parse-cache.json` 和 `official-textunit-index.json`，并从 Added/Modified 资源、parse cache 与 TextUnit 明细索引派生 `official-textunit-tasks.json`、`crowdin-textunit-queue.json` 和版本化的 `translation-tasks.sqlite`；up-to-date 轮询在已有有效解析缓存、TextUnit 明细索引和 TextUnit 队列时只读取摘要，不重复解析，重新同步队列时保留已有 worker 状态。 15. 若启用 `--import-repository`，已校验 release 会被导入 CAS + `ResourceRepository`，并可经 `resource.index` 查询。 16. 官方同步报告默认给出 `localized_release_status=not_localized`，表示原版资源已发布、汉化资源未发布；UnityFS TextAsset patch 发布成功并通过 `localized-patch-manifest.json`、current symlink 和 release ID 校验后，`localized.status` 才返回 `localized`，表示原版和汉化两套资源都已发布。 diff --git a/docs/guides/development.md b/docs/guides/development.md index c9c2a4b..2fea9f4 100644 --- a/docs/guides/development.md +++ b/docs/guides/development.md @@ -219,12 +219,20 @@ cargo run -p bat-infrastructure --bin bat -- \ `official-resource-changes.json`、`crowdin-translation-handoff.json`、 `official-parse-cache.json` 和 `official-textunit-index.json`，随后从 Added/Modified 资源、parse cache 与 TextUnit 明细索引派生 -`official-textunit-tasks.json` 和 `crowdin-textunit-queue.json`。本地已有旧完整 +`official-textunit-tasks.json`、`crowdin-textunit-queue.json`、 +`translation-tasks.sqlite` 和 `translation-handoff.json`。本地已有旧完整 版本时，新版本发布后会先按 manifest destination 对比旧/新 release，只把新增和 内容变更的资源写入解析与 Crowdin handoff；删除资源只记录差异，不进入翻译队列。 up-to-date 轮询发现本地文件、解析缓存、TextUnit 明细索引和 TextUnit 队列未变时不会重复解析。 Crowdin 队列当前只落本地文件，不发网络请求。 +官方下载服务默认使用 8 个有界 worker，`--download-concurrency` / +`BAT_DOWNLOAD_CONCURRENCY` 只接受 `1..=256`。worker 完成一个 URL 后立即从共享 +队列领取下一个任务；finished 进度按实际完成顺序即时上报，完成计数单调递增， +最终 report 的资源列表仍按 pull plan 顺序。需要验证顺序模式时显式使用 +`--download-concurrency 1`。本文档中的真实资源命令仅是隔离 runbook；本地轻量 +验证应使用 fake-curl/fixture，不要在开发机执行真实下载或 smoke run。 + 需要把已校验官方 release 导入 CAS + `ResourceRepository` 时，显式启用： ```bash @@ -243,6 +251,8 @@ cargo run -p bat-infrastructure --bin bat -- parse-status cargo run -p bat-infrastructure --bin bat -- parse-text-units --limit 50 cargo run -p bat-infrastructure --bin bat -- parse-errors --limit 50 cargo run -p bat-infrastructure --bin bat -- translation-tasks --task-status skipped_parse_failed --has-reason --limit 50 +cargo run -p bat-infrastructure --bin bat -- translation-tasks --worker-status failed --has-failure-reason --limit 50 +cargo run -p bat-infrastructure --bin bat -- translation-handoff cargo run -p bat-infrastructure --bin bat -- localized-status cargo run -p bat-infrastructure --bin bat -- resource-index --limit 50 cargo run -p bat-infrastructure --bin bat -- resource-index --release-id --platform windows --archive-entry --format json --limit 50 @@ -251,8 +261,13 @@ cargo run -p bat-infrastructure --bin bat -- resource-index --release-id -- `parse-status` 会额外显示 TextUnit 明细索引和队列摘要；`parse-text-units` / `parse-errors` 可按 destination、archive entry、path id、class id、field path 和 format 分页查询当前官方 release 的 TextUnit 明细与解析错误； -`translation-tasks` 可按 release、destination、archive entry、任务状态、parse status、 -TextUnit format 和 reason presence 查询离线 TextUnit 翻译任务状态与跳过/失败原因； +`translation-tasks` 可按 release、destination、archive entry、队列任务状态、provider +worker 状态、parse status、TextUnit format、队列 reason 和 provider failure reason +查询离线 TextUnit 翻译任务状态与跳过/失败原因；发布后的状态保存在当前 release +根目录的 `translation-tasks.sqlite`，旧 release 没有状态库时回退到 JSON 队列； +`translation-handoff` / `translation.handoff` 会动态合并版本化 +`translation-handoff.json` 与 SQLite 状态，返回 job、unit、provider run 的完整交接 +视图； `resource-index` 返回的资源 JSON 包含 release、平台、bundle path、TextAsset 和 TextUnit metadata， 并可按 release、平台、destination、bundle path、archive entry、parse status 和 TextUnit format 做资源级过滤； `localized-status` 只有在 `localized-version-state.json`、`current` symlink 和 diff --git a/docs/guides/official-resource-test-pull.md b/docs/guides/official-resource-test-pull.md index c39c0d0..35ab2df 100644 --- a/docs/guides/official-resource-test-pull.md +++ b/docs/guides/official-resource-test-pull.md @@ -322,7 +322,7 @@ cargo run -p bat-infrastructure --bin bat -- \ 默认平台是 `Windows,Android`，无需显式传 `--platforms`；只有要覆盖默认平台时才传。`--interval` 是正常检查周期，默认 `1h`；watch/daemon 模式还会在每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 强制执行一次自动刷新，该轮会注入 `force=true`，并且会中断普通 interval 的 sleep。`--error-retry` 是下载、发现或校验失败后的重试周期，默认 `60s`，也可以用 `--error-retry-seconds 60`。CLI 默认启动时向 stderr 打印 `BlueArchiveToolkit` ASCII banner，并把阶段进度日志写到 stderr，包括自动发现、proxy、server-info、marker、catalog、audit、download、snapshot 和 publish 阶段；download 阶段会输出已完成计数和单文件开始/完成状态，下载执行保持顺序处理，已完成计数保持单调不倒退，audit 阶段会输出官方 `.hash`、本地 BLAKE3、需修复项和 ZIP 结构校验结果摘要。daemon 还会写 `bat-events.jsonl` 结构化日志并按大小轮转。命令结果默认以人类可读摘要写到 stdout。需要纯机器输出时加 `--json --no-progress`，需要显式开启进度日志则用 `--progress`；只想关闭横幅但保留日志时可加 `--no-banner`。错误时 stderr 输出 JSON error，watch 模式下错误 JSON 的 `next_retry_seconds` 使用失败重试周期；如果未关闭 progress，错误 JSON 前可能已有 banner 和进度日志。普通错误 exit `1`，资源目录锁冲突 exit `75`，`verify` 或 `doctor` 发现问题也返回非 0。 -生产可以直接运行 `--watch`，也可以用 `--daemon` 后台运行，或者用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产官方资源目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录；上层读取原版资源时应读取 `--output/current`，不要读取 `.staging` 或 `versions` 中未切换的目录。汉化 Patch/导出应写入 `--localized-output`，并保留官方相对目录结构，不能写回 `--output/current`。发布状态分两档：`not_localized` 只发布原版资源、不发布汉化资源；`localized` 发布原版和汉化两套资源。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一官方资源目录；live daemon 还会阻止前台写命令直接修改它正在管理的同一目录。 +生产可以直接运行 `--watch`，也可以用 `--daemon` 后台运行，或者用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。下载默认并发 8，可用 `--download-concurrency` / `BAT_DOWNLOAD_CONCURRENCY` 配置为 `1..=256`；worker 动态领取共享 plan，finished 进度即时按完成数统计，发布 report 仍按 plan 顺序。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产官方资源目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录；上层读取原版资源时应读取 `--output/current`，不要读取 `.staging` 或 `versions` 中未切换的目录。汉化 Patch/导出应写入 `--localized-output`，并保留官方相对目录结构，不能写回 `--output/current`。发布状态分两档：`not_localized` 只发布原版资源、不发布汉化资源；`localized` 发布原版和汉化两套资源。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一官方资源目录；live daemon 还会阻止前台写命令直接修改它正在管理的同一目录。 需要只做探测时可以加 `--dry-run`。需要关闭本地 audit 或 repair 时可以显式使用 `--no-audit-local` 或 `--no-repair`，但生产同步默认应保持开启。 diff --git a/docs/reference/rpc-backend-api.md b/docs/reference/rpc-backend-api.md index 13ed295..1486570 100644 --- a/docs/reference/rpc-backend-api.md +++ b/docs/reference/rpc-backend-api.md @@ -151,6 +151,11 @@ SQLite `ResourceRepository`，索引不存在时返回 `ok=true` 且 TextUnit 明细索引派生，记录 TextUnit 任务、跳过原因和解析诊断。 - `crowdin-textunit-queue.json`：只包含已产生 TextUnit 的离线任务，预留给后续 Crowdin worker；当前不会发出网络请求。 +- `translation-tasks.sqlite`：当前 release 的可变 worker 状态库，记录 + queued / running / failed / completed / skipped、attempt count、provider run + ID 和 failure reason；schema 由 `schema_migrations` 版本表管理。 +- `translation-handoff.json`：当前 release 的版本化 job/unit/provider run 交接 + 快照；worker 更新后的实时状态仍以 `translation-tasks.sqlite` 为准。 删除资源只进入 `official-resource-changes.json`，不进入 Crowdin handoff。 @@ -162,6 +167,8 @@ SQLite `ResourceRepository`，索引不存在时返回 `ok=true` 且 | `parse.text_units` | 已实现 | `{ "offset": 0, "limit": 100, "destination": "*Table*", "archive_entry": "*.bytes", "path_id": 1, "class_id": 114, "field_path": "*Text*", "format": "json" }` | 当前官方 release 的 TextUnit 明细分页。 | | `parse.errors` | 已实现 | `{ "offset": 0, "limit": 100, "destination": "*Table*", "archive_entry": "*.bytes", "path_id": 1, "class_id": 114, "field_path": "*Text*", "format": "json" }` | 当前官方 release 的解析错误分页。 | | `translation.tasks` | 已实现 | `{ "offset": 0, "limit": 100, "task_id": "...", "release_id": "...", "destination": "...", "archive_entry": "...", "status": "skipped_parse_failed", "parse_status": "failed", "format": "json", "has_reason": true }` | 当前官方 release 的离线 TextUnit 翻译任务状态分页。 | +| `translation.handoff` | 已实现 | `null` | 当前官方 release 的 job、unit、provider run 交接视图；动态合并队列和 SQLite worker 状态。 | +| `translation.task.update` | 已实现 | `{ "task_id": "...", "status": "failed", "failure_reason": "...", "provider_run_id": "..." }` | 写入当前 release 的 provider worker 状态，返回可回查任务记录。 | `parse.status` 是只读查询；没有当前 release 或没有解析缓存时返回 `ok=true` 且 `data.available=false`。解析缓存来自官方原版资源目录，不读取 @@ -195,15 +202,25 @@ offset 和 error。TypeTree-covered managed reference 字段会进入结构化 完整 managed reference registry 等暂不支持结构会进入解析错误，而不是静默降级为 低保真文本。 -`translation.tasks` 查询 `official-textunit-tasks.json`，用于查看离线 -TextUnit 翻译任务候选、跳过状态和失败/跳过原因；没有当前 release 或没有任务队列时 -返回 `ok=true` 且 `data.available=false`。过滤参数包括 `task_id`、 +`translation.tasks` 优先查询当前 release 的 `translation-tasks.sqlite`，旧 release +没有该文件时回退到 `official-textunit-tasks.json`；用于查看离线 TextUnit +翻译任务候选和 provider worker 状态。没有当前 release 或没有任务队列时返回 +`ok=true` 且 `data.available=false`。过滤参数包括 `task_id`、 `official_release_id`/`release_id`、`destination`、`path_pattern`、 -`archive_entry`、`status`/`task_status`、`parse_status`、`text_unit_format`/`format` -和 `has_reason`。`entries[]` 会包含 `official_release_id`、`destination`、 -`archive_entry`、`parse_status`、`status`、`reason`、TextAsset/TextUnit 摘要和 -校验指纹。当前它只反映 Rust 侧离线 handoff 状态；真实 Crowdin worker 消费、 -重试和完成状态仍需后续 worker 模型写回后才能暴露。 +`archive_entry`、`status`/`task_status`、`worker_status`、`parse_status`、 +`text_unit_format`/`format`、`has_reason` 和 `has_failure_reason`。 +`entries[]` 会包含 `official_release_id`、`destination`、`archive_entry`、 +`parse_status`、队列 `status`、`task_status`、`failure_reason`、`attempt_count`、 +`provider_run_id`、TextAsset/TextUnit 摘要和校验指纹。 + +`translation.task.update` 只更新当前 release 的 SQLite 状态库，不改写 immutable +队列文件，也不主动访问 Crowdin。`status` 支持 `queued`、`running`、`failed`、 +`completed` 和 `skipped`；进入 `running` 会增加 attempt count，`completed` 会 +记录完成时间，`failed` 可写入 `failure_reason`。因此 worker 消费 handoff 后， +bat-api 可通过 `translation.tasks` 查询单项任务，也可通过 +`translation.handoff` 获取完整 job/unit/provider run 状态。`translation.handoff` +不会触发下载或 provider 网络请求；没有当前 release 或任务队列时返回 +`data.available=false`。 ### localized diff --git a/docs/reports/CURRENT_GAPS.md b/docs/reports/CURRENT_GAPS.md index 3a74d79..cb26ba4 100644 --- a/docs/reports/CURRENT_GAPS.md +++ b/docs/reports/CURRENT_GAPS.md @@ -286,9 +286,12 @@ - 官方 release 发布后会写出 `official-resource-changes.json` 和 `crowdin-translation-handoff.json`，新增+变更资源进入解析/翻译 handoff，删除资源只进入差异记录。 - `Resource` metadata 已通过 SQLite `metadata_json` 兼容迁移保存 release、平台、bundle path、parse status、TextAsset 名称和 TextUnit 数量/格式；`resource.index` 会返回这些 metadata。 - 官方 release 发布后会持久化 `official-textunit-index.json`，记录单条 TextUnit 和解析错误；`parse.text_units` / `parse.errors` RPC 和 `parse-text-units` / `parse-errors` CLI 可按 destination、archive entry、path id、class id、field path 和 format 分页过滤。 -- 官方 release 发布后会从 Added/Modified 资源、parse cache 和 TextUnit 明细索引派生 `official-textunit-tasks.json` 和 `crowdin-textunit-queue.json`；删除资源不会进入队列。 -- `translation.tasks` RPC/CLI 已能分页查询 `official-textunit-tasks.json` 中的离线 TextUnit 任务状态、parse 状态、TextUnit format 和跳过/失败 reason。 -- 真实 Crowdin worker 消费后的任务状态、重试次数、远端失败原因和完成结果查询仍需补齐。 +- 官方 release 发布后会从 Added/Modified 资源、parse cache 和 TextUnit 明细索引派生 `official-textunit-tasks.json`、`crowdin-textunit-queue.json` 和 `translation-tasks.sqlite`；删除资源不会进入队列。 +- 官方 release 发布后还会写入版本化 `translation-handoff.json`；`translation.handoff` RPC/CLI 动态合并该快照与 SQLite worker 状态，暴露 job、unit、provider run、attempt 和 failure reason。 +- `translation-tasks.sqlite` 由 `schema_migrations` 管理 durable task state、attempt count、provider run ID 和 failure reason；重复同步会保留已有 worker 状态。 +- `translation.tasks` RPC/CLI 优先查询 `translation-tasks.sqlite`，旧 release 没有状态库时回退到 `official-textunit-tasks.json`；返回队列 `status`、worker `task_status`、failure reason 和时间/尝试次数。 +- `translation.task.update` RPC 已提供 queued/running/failed/completed/skipped 状态回写契约，provider worker 可在消费 handoff 后按 task_id 更新并由同一查询接口反查。 +- 真实 Crowdin 网络 worker、翻译记忆和完整 localized repack 仍属于后续翻译系统工作，不在当前 Rust 离线状态仓储范围内。 验收： @@ -554,7 +557,9 @@ 1. issue #24：失败 staging 复用回归已补；核对残余场景。 2. issue #1：RPC 主体、文件级 `patch.apply` / `unityfs.patch_text_asset` / `unityfs.patch_string_field` / `unityfs.patch_field` 已落地；剩余发布级 patch build/rollback、复杂 UnityFS 语义编辑与设计边界确认。 -3. issue #17 及子 issue：已按 wontfix 关闭多线程下载（顺序下载 + 指数退避）。 +3. issue #17 的历史顺序/重试契约仍保留；当前 issue #33/#35 已补有界 downloader + scheduler 和默认并发 8，范围 `1..=256`，worker 完成后立即领取下一个任务， + 进度即时按完成数上报，最终 report 保持 plan 顺序。 4. **G-008：已决策关闭**（同步 CLI = Rust `bat`；见 `GO_STATUS.md`）。 5. **G-009 / issue #19**：资源 bootstrap/分发 MVP 已编码；优先服务器联调与索引实勘，非「从零实现」。 6. issue #2 / G-007（P1）：Addressables 可校验字段。 diff --git a/infrastructure/src/bin/bat_official_sync.rs b/infrastructure/src/bin/bat_official_sync.rs index 57aa34a..01ee442 100644 --- a/infrastructure/src/bin/bat_official_sync.rs +++ b/infrastructure/src/bin/bat_official_sync.rs @@ -3,6 +3,8 @@ use bat_assetbundle::UnitySerializedReplacementValue; use bat_core::domain::{Resource, ResourceType}; use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository}; use bat_core::{ApiError, ErrorCode}; +#[cfg(test)] +use bat_infrastructure::DEFAULT_DOWNLOAD_CONCURRENCY; use bat_infrastructure::{ apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file, apply_unityfs_text_asset_patch_file, changed_endpoint_urls, diff_extended_snapshot, @@ -16,11 +18,12 @@ use bat_infrastructure::{ OfficialUpdateConfig, OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService, OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState, PatchApplyKind, PatchApplyParams, - PatchApplyReport, ReleaseFlowStatusCode, SqliteResourceRepository, UnityFsFieldPatchParams, + PatchApplyReport, ReleaseFlowStatusCode, SqliteResourceRepository, + SqliteTranslationTaskRepository, TranslationTaskStatus, UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams, UnityFsTextAssetPatchParams, LOCALIZED_CURRENT_LINK, LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_VERSIONS_DIR, - LOCALIZED_VERSION_STATE_FILE, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE, - PRIVATE_FILE_MODE, + LOCALIZED_VERSION_STATE_FILE, MAX_DOWNLOAD_CONCURRENCY, MIN_DOWNLOAD_CONCURRENCY, + OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE, PRIVATE_FILE_MODE, }; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -175,6 +178,7 @@ fn run() -> anyhow::Result { | CliCommand::ParseTextUnits | CliCommand::ParseErrors | CliCommand::TranslationTasks + | CliCommand::TranslationHandoff | CliCommand::LocalizedStatus | CliCommand::ResourceIndex => { run_readonly_query_command(&options)?; @@ -243,12 +247,14 @@ struct CliOptions { query_bundle_path: Option, query_archive_entry: Option, query_task_status: Option, + query_worker_status: Option, query_parse_status: Option, query_path_id: Option, query_class_id: Option, query_field_path: Option, query_format: Option, query_has_reason: Option, + query_has_failure_reason: Option, query_option_explicit: bool, patch_kind: Option, patch_source_path: Option, @@ -302,12 +308,14 @@ impl Default for CliOptions { query_bundle_path: None, query_archive_entry: None, query_task_status: None, + query_worker_status: None, query_parse_status: None, query_path_id: None, query_class_id: None, query_field_path: None, query_format: None, query_has_reason: None, + query_has_failure_reason: None, query_option_explicit: false, patch_kind: None, patch_source_path: None, @@ -349,6 +357,7 @@ enum CliCommand { ParseTextUnits, ParseErrors, TranslationTasks, + TranslationHandoff, LocalizedStatus, ResourceIndex, PatchApply, @@ -800,6 +809,8 @@ const RPC_METHOD_PARSE_STATUS: &str = "parse.status"; const RPC_METHOD_PARSE_TEXT_UNITS: &str = "parse.text_units"; const RPC_METHOD_PARSE_ERRORS: &str = "parse.errors"; const RPC_METHOD_TRANSLATION_TASKS: &str = "translation.tasks"; +const RPC_METHOD_TRANSLATION_HANDOFF: &str = "translation.handoff"; +const RPC_METHOD_TRANSLATION_TASK_UPDATE: &str = "translation.task.update"; const RPC_METHOD_LOCALIZED_STATUS: &str = "localized.status"; const RPC_METHOD_CATALOG_STATUS: &str = "catalog.status"; const RPC_METHOD_CATALOG_VERSIONS: &str = "catalog.versions"; @@ -2278,6 +2289,16 @@ fn dispatch_rpc_method( build_translation_tasks_report(state_dir, query, offset, limit), ) } + RPC_METHOD_TRANSLATION_HANDOFF => rpc_envelope_from_result( + request_id, + "translation.handoff", + build_translation_handoff_report(state_dir), + ), + RPC_METHOD_TRANSLATION_TASK_UPDATE => rpc_envelope_from_result( + request_id, + "translation.task.update", + update_translation_task_status_report(state_dir, request.params.as_ref()), + ), RPC_METHOD_LOCALIZED_STATUS => rpc_envelope_from_result( request_id, "localized.status", @@ -2873,17 +2894,46 @@ fn query_resource_repository( }) } +fn query_translation_task_repository( + repository_path: &Path, + query: &OfficialTextUnitTaskQuery, + offset: usize, + limit: usize, +) -> anyhow::Result<(u64, Vec)> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(async { + let repository = SqliteTranslationTaskRepository::open(repository_path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let total_entries = repository + .count(query) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let entries = repository + .list(query) + .await + .map_err(|error| anyhow::anyhow!("{error}"))? + .into_iter() + .skip(offset) + .take(limit) + .collect(); + Ok((total_entries, entries)) + }) +} + fn repository_file_exists_no_symlink(path: &Path) -> anyhow::Result { + sqlite_file_exists_no_symlink(path, "资源索引数据库") +} + +fn sqlite_file_exists_no_symlink(path: &Path, label: &str) -> anyhow::Result { match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() => Err(anyhow::anyhow!( - "资源索引数据库不能是 symlink：{}", - path.display() - )), + Ok(metadata) if metadata.file_type().is_symlink() => { + Err(anyhow::anyhow!("{label}不能是 symlink：{}", path.display())) + } Ok(metadata) if metadata.is_file() => Ok(true), - Ok(_) => Err(anyhow::anyhow!( - "资源索引数据库不是普通文件：{}", - path.display() - )), + Ok(_) => Err(anyhow::anyhow!("{label}不是普通文件：{}", path.display())), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), Err(error) => Err(error.into()), } @@ -3089,6 +3139,8 @@ fn build_translation_tasks_report( let task_queue_path = record .resource_root .join(bat_infrastructure::OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE); + let task_repository_path = + SqliteTranslationTaskRepository::repository_path(&record.resource_root); let Some(queue) = bat_infrastructure::read_textunit_task_queue_at(&record.resource_root) .map_err(anyhow::Error::msg)? else { @@ -3097,21 +3149,56 @@ fn build_translation_tasks_report( "current_version_id": record.id, "resource_root": record.resource_root, "textunit_task_queue_path": task_queue_path, + "task_repository_path": task_repository_path, + "task_repository_available": false, })); }; - let matches = bat_infrastructure::query_textunit_tasks(&queue, &query); - let total_entries = matches.len(); - let entries = matches - .into_iter() - .skip(offset) - .take(limit) - .cloned() - .collect::>(); + let task_repository_available = + sqlite_file_exists_no_symlink(&task_repository_path, "翻译任务状态数据库")?; + let (total_entries, entries) = if task_repository_available { + query_translation_task_repository(&task_repository_path, &query, offset, limit)? + } else { + let mut queue_query = query.clone(); + queue_query.task_status = None; + queue_query.has_failure_reason = None; + let matches = bat_infrastructure::query_textunit_tasks(&queue, &queue_query); + let persisted = matches + .into_iter() + .cloned() + .map(|task| { + bat_infrastructure::PersistedTranslationTask::from_queued_task( + task, + queue.generated_unix_seconds, + ) + }) + .filter(|task| { + query + .task_status + .as_ref() + .is_none_or(|status| task.task_status.as_str() == status) + }) + .filter(|task| { + query + .has_failure_reason + .is_none_or(|has_reason| task.failure_reason.is_some() == has_reason) + }) + .collect::>(); + let total_entries = persisted.len(); + let entries = persisted + .into_iter() + .skip(offset) + .take(limit) + .collect::>(); + (total_entries as u64, entries) + }; Ok(serde_json::json!({ "available": true, "current_version_id": record.id, "resource_root": record.resource_root, "textunit_task_queue_path": task_queue_path, + "task_repository_path": task_repository_path, + "task_repository_available": task_repository_available, + "task_repository_schema_version": bat_infrastructure::TRANSLATION_TASK_SCHEMA_VERSION, "summary": queue.summary, "total_entries": total_entries, "offset": offset, @@ -3121,6 +3208,124 @@ fn build_translation_tasks_report( })) } +/// `translation.handoff`：当前已发布版本的动态翻译交接视图。 +fn build_translation_handoff_report(state_dir: &Path) -> anyhow::Result { + let (_, version_state) = read_daemon_resource_state(state_dir)?; + let current = version_state + .as_ref() + .and_then(|state| state.current_completed_version.as_ref()); + let Some(record) = current else { + return Ok(serde_json::json!({ "available": false })); + }; + let resource_root = &record.resource_root; + let task_queue_path = resource_root.join(bat_infrastructure::OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE); + let handoff_path = resource_root.join(bat_infrastructure::TRANSLATION_HANDOFF_FILE); + let repository_path = SqliteTranslationTaskRepository::repository_path(resource_root); + let Some(queue) = bat_infrastructure::read_textunit_task_queue_at(resource_root) + .map_err(anyhow::Error::msg)? + else { + return Ok(serde_json::json!({ + "available": false, + "current_version_id": record.id, + "resource_root": resource_root, + "textunit_task_queue_path": task_queue_path, + "translation_handoff_path": handoff_path, + "task_repository_path": repository_path, + })); + }; + let task_repository_available = + sqlite_file_exists_no_symlink(&repository_path, "翻译任务状态数据库")?; + let tasks = if task_repository_available { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(async { + let repository = SqliteTranslationTaskRepository::open(&repository_path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .list(&OfficialTextUnitTaskQuery::default()) + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })? + } else { + queue + .tasks + .iter() + .cloned() + .map(|task| { + bat_infrastructure::PersistedTranslationTask::from_queued_task( + task, + queue.generated_unix_seconds, + ) + }) + .collect::>() + }; + let handoff = bat_infrastructure::build_translation_handoff(&queue, &tasks); + let handoff_file_available = sqlite_file_exists_no_symlink(&handoff_path, "翻译 handoff")?; + Ok(serde_json::json!({ + "available": true, + "current_version_id": record.id, + "resource_root": resource_root, + "textunit_task_queue_path": task_queue_path, + "translation_handoff_path": handoff_path, + "translation_handoff_file_available": handoff_file_available, + "task_repository_path": repository_path, + "task_repository_available": task_repository_available, + "task_repository_schema_version": bat_infrastructure::TRANSLATION_TASK_SCHEMA_VERSION, + "handoff_schema_version": bat_infrastructure::TRANSLATION_HANDOFF_SCHEMA_VERSION, + "handoff": handoff, + })) +} + +/// `translation.task.update`：写入 provider worker 的可回查状态。 +fn update_translation_task_status_report( + state_dir: &Path, + params: Option<&serde_json::Value>, +) -> anyhow::Result { + let task_id = rpc_string_param(params, "task_id") + .ok_or_else(|| anyhow::anyhow!("translation.task.update 缺少 task_id"))?; + let status_label = rpc_string_param(params, "status") + .ok_or_else(|| anyhow::anyhow!("translation.task.update 缺少 status"))?; + let status = TranslationTaskStatus::parse(status_label) + .ok_or_else(|| anyhow::anyhow!("不支持的翻译任务 worker 状态：{status_label}"))?; + let failure_reason = rpc_string_param(params, "failure_reason") + .or_else(|| rpc_string_param(params, "reason")) + .map(str::to_string); + let provider_run_id = rpc_string_param(params, "provider_run_id").map(str::to_string); + let (_, version_state) = read_daemon_resource_state(state_dir)?; + let current = version_state + .as_ref() + .and_then(|state| state.current_completed_version.as_ref()) + .ok_or_else(|| anyhow::anyhow!("没有可更新翻译任务的当前官方 release"))?; + let repository_path = SqliteTranslationTaskRepository::repository_path(¤t.resource_root); + if !sqlite_file_exists_no_symlink(&repository_path, "翻译任务状态数据库")? { + return Err(anyhow::anyhow!( + "翻译任务状态数据库不存在：{}", + repository_path.display() + )); + } + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let task = runtime.block_on(async { + let repository = SqliteTranslationTaskRepository::open(&repository_path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .update_status(task_id, status, failure_reason, provider_run_id) + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })?; + Ok(serde_json::json!({ + "available": true, + "current_version_id": current.id, + "task_repository_path": repository_path, + "entry": task, + })) +} + fn textunit_query_json(query: &OfficialTextUnitQuery) -> serde_json::Value { serde_json::json!({ "destination": query.destination.clone(), @@ -3141,9 +3346,11 @@ fn translation_task_query_json(query: &OfficialTextUnitTaskQuery) -> serde_json: "path_pattern": query.path_pattern.clone(), "archive_entry": query.archive_entry.clone(), "status": query.status.clone(), + "task_status": query.task_status.clone(), "parse_status": query.parse_status.clone(), "text_unit_format": query.text_unit_format.clone(), "has_reason": query.has_reason, + "has_failure_reason": query.has_failure_reason, }) } @@ -3398,11 +3605,13 @@ fn rpc_translation_task_query_params( status: rpc_string_param(params, "status") .or_else(|| rpc_string_param(params, "task_status")) .map(str::to_string), + task_status: rpc_string_param(params, "worker_status").map(str::to_string), parse_status: rpc_string_param(params, "parse_status").map(str::to_string), text_unit_format: rpc_string_param(params, "text_unit_format") .or_else(|| rpc_string_param(params, "format")) .map(str::to_string), has_reason: rpc_bool_param(params, "has_reason"), + has_failure_reason: rpc_bool_param(params, "has_failure_reason"), }; Ok((query, offset, limit)) } @@ -4357,6 +4566,7 @@ fn readonly_query_rpc_method(command: CliCommand) -> Option<&'static str> { CliCommand::ParseTextUnits => Some(RPC_METHOD_PARSE_TEXT_UNITS), CliCommand::ParseErrors => Some(RPC_METHOD_PARSE_ERRORS), CliCommand::TranslationTasks => Some(RPC_METHOD_TRANSLATION_TASKS), + CliCommand::TranslationHandoff => Some(RPC_METHOD_TRANSLATION_HANDOFF), CliCommand::LocalizedStatus => Some(RPC_METHOD_LOCALIZED_STATUS), CliCommand::ResourceIndex => Some(RPC_METHOD_RESOURCE_INDEX), _ => None, @@ -4471,6 +4681,9 @@ fn readonly_query_rpc_params(options: &CliOptions) -> Option if let Some(status) = options.query_task_status.as_ref() { params.insert("status".to_string(), serde_json::json!(status)); } + if let Some(status) = options.query_worker_status.as_ref() { + params.insert("worker_status".to_string(), serde_json::json!(status)); + } if let Some(parse_status) = options.query_parse_status.as_ref() { params.insert("parse_status".to_string(), serde_json::json!(parse_status)); } @@ -4480,6 +4693,12 @@ fn readonly_query_rpc_params(options: &CliOptions) -> Option if let Some(has_reason) = options.query_has_reason { params.insert("has_reason".to_string(), serde_json::json!(has_reason)); } + if let Some(has_failure_reason) = options.query_has_failure_reason { + params.insert( + "has_failure_reason".to_string(), + serde_json::json!(has_failure_reason), + ); + } } _ => {} } @@ -4515,6 +4734,7 @@ fn build_readonly_query_report( options.query_offset, options.query_limit, ), + RPC_METHOD_TRANSLATION_HANDOFF => build_translation_handoff_report(&options.state_dir), RPC_METHOD_LOCALIZED_STATUS => { build_localized_status_report(&options.state_dir, &options.config) } @@ -4539,13 +4759,15 @@ fn validate_readonly_query_options(options: &CliOptions) -> anyhow::Result<()> { || options.query_field_path.is_some(); let has_translation_task_filter = options.query_task_id.is_some() || options.query_task_status.is_some() - || options.query_has_reason.is_some(); + || options.query_worker_status.is_some() + || options.query_has_reason.is_some() + || options.query_has_failure_reason.is_some(); match options.command { CliCommand::ResourceIndex => { if has_parse_object_filter || has_translation_task_filter { return Err(anyhow::anyhow!( - "--path-id/--class-id/--field-path 只适用于 parse-text-units 或 parse-errors；--task-id/--task-status/--has-reason 只适用于 translation-tasks" + "--path-id/--class-id/--field-path 只适用于 parse-text-units 或 parse-errors；--task-id/--task-status/--worker-status/--has-reason/--has-failure-reason 只适用于 translation-tasks" )); } } @@ -4556,7 +4778,7 @@ fn validate_readonly_query_options(options: &CliOptions) -> anyhow::Result<()> { || options.query_parse_status.is_some() { return Err(anyhow::anyhow!( - "--resource-type/--hash/--release-id/--platform/--bundle-path/--parse-status 只适用于 resource-index 或 translation-tasks；--task-id/--task-status/--has-reason 只适用于 translation-tasks" + "--resource-type/--hash/--release-id/--platform/--bundle-path/--parse-status 只适用于 resource-index 或 translation-tasks；--task-id/--task-status/--worker-status/--has-reason/--has-failure-reason 只适用于 translation-tasks" )); } } @@ -4567,6 +4789,11 @@ fn validate_readonly_query_options(options: &CliOptions) -> anyhow::Result<()> { )); } } + CliCommand::TranslationHandoff if options.query_option_explicit => { + return Err(anyhow::anyhow!( + "translation-handoff 不接受查询过滤参数；请使用 translation-tasks 查询单项任务" + )); + } CliCommand::ParseStatus | CliCommand::LocalizedStatus if options.query_option_explicit => { return Err(anyhow::anyhow!( "查询过滤参数只适用于 resource-index、parse-text-units、parse-errors 或 translation-tasks" @@ -4788,6 +5015,8 @@ fn translation_task_query_from_options(options: &CliOptions) -> OfficialTextUnit parse_status: options.query_parse_status.clone(), text_unit_format: options.query_format.clone(), has_reason: options.query_has_reason, + has_failure_reason: options.query_has_failure_reason, + task_status: options.query_worker_status.clone(), } } @@ -6446,6 +6675,8 @@ fn daemon_child_args(options: &CliOptions) -> Vec { } args.push("--curl".to_string()); args.push(config.curl_command.to_string_lossy().to_string()); + args.push("--download-concurrency".to_string()); + args.push(config.download_concurrency.to_string()); match config.curl_proxy.mode() { CurlProxyMode::Auto if options.proxy_option_explicit => { args.push("--proxy".to_string()); @@ -6960,6 +7191,8 @@ BAT_AUTO_DISCOVER=1 # 逗号分隔：windows,android #BAT_PLATFORMS=windows,android #BAT_CURL=curl +# 最大并发下载数（默认 8；范围 1..=256） +#BAT_DOWNLOAD_CONCURRENCY=8 #BAT_UNZIP=unzip # ---- 输出 ---- @@ -7138,6 +7371,10 @@ fn apply_bat_env_overrides( if let Some(v) = value("BAT_CURL") { options.config.curl_command = PathBuf::from(v); } + if let Some(v) = value("BAT_DOWNLOAD_CONCURRENCY") { + options.config.download_concurrency = + parse_download_concurrency(&v, "环境变量 BAT_DOWNLOAD_CONCURRENCY")?; + } if let Some(v) = value("BAT_UNZIP") { options.config.unzip_command = PathBuf::from(v); } @@ -7245,6 +7482,10 @@ fn parse_args_with_env( ensure_command_not_set(options.command, "translation-tasks")?; options.command = CliCommand::TranslationTasks; } + "translation-handoff" => { + ensure_command_not_set(options.command, "translation-handoff")?; + options.command = CliCommand::TranslationHandoff; + } "localized-status" => { ensure_command_not_set(options.command, "localized-status")?; options.command = CliCommand::LocalizedStatus; @@ -7359,6 +7600,11 @@ fn parse_args_with_env( "--curl" => { options.config.curl_command = PathBuf::from(next_option_value(&mut args, &flag)?); } + "--download-concurrency" => { + options.config.download_concurrency = + parse_download_concurrency(&next_option_value(&mut args, &flag)?, &flag)?; + options.sync_option_explicit = true; + } "--proxy" => { options.config.curl_proxy = parse_proxy_config(&next_option_value(&mut args, &flag)?)?; @@ -7543,6 +7789,10 @@ fn parse_args_with_env( options.query_task_status = Some(next_option_value(&mut args, &flag)?); options.query_option_explicit = true; } + "--worker-status" => { + options.query_worker_status = Some(next_option_value(&mut args, &flag)?); + options.query_option_explicit = true; + } "--parse-status" => { options.query_parse_status = Some(next_option_value(&mut args, &flag)?); options.query_option_explicit = true; @@ -7587,6 +7837,14 @@ fn parse_args_with_env( options.query_has_reason = Some(false); options.query_option_explicit = true; } + "--has-failure-reason" => { + options.query_has_failure_reason = Some(true); + options.query_option_explicit = true; + } + "--no-failure-reason" => { + options.query_has_failure_reason = Some(false); + options.query_option_explicit = true; + } "--patch-kind" => { options.patch_kind = Some(parse_patch_apply_kind(&next_option_value( &mut args, &flag, @@ -7704,6 +7962,7 @@ fn parse_args_with_env( | CliCommand::ParseTextUnits | CliCommand::ParseErrors | CliCommand::TranslationTasks + | CliCommand::TranslationHandoff | CliCommand::LocalizedStatus | CliCommand::ResourceIndex => { validate_readonly_query_options(&options)?; @@ -7833,6 +8092,18 @@ fn tools_are_non_default(config: &OfficialUpdateConfig, baseline: &OfficialUpdat || config.unzip_command != baseline.unzip_command } +fn parse_download_concurrency(value: &str, source: &str) -> anyhow::Result { + let parsed = value + .parse::() + .map_err(|error| anyhow::anyhow!("{source} 无效：{error}"))?; + if !(MIN_DOWNLOAD_CONCURRENCY..=MAX_DOWNLOAD_CONCURRENCY).contains(&parsed) { + return Err(anyhow::anyhow!( + "{source} 必须在 {MIN_DOWNLOAD_CONCURRENCY}..={MAX_DOWNLOAD_CONCURRENCY} 范围内" + )); + } + Ok(parsed) +} + fn next_option_value( args: &mut impl Iterator, flag: &str, @@ -7896,6 +8167,7 @@ fn print_usage(binary: &str) { eprintln!(" parse-text-units Query current official TextUnit detail index"); eprintln!(" parse-errors Query current official parse/extraction diagnostics"); eprintln!(" translation-tasks Query current offline TextUnit translation task status"); + eprintln!(" translation-handoff Query current translation job/unit/provider handoff"); eprintln!(" localized-status Show localized release status for current official release"); eprintln!(" resource-index Query CAS + ResourceRepository index"); eprintln!(" patch-apply Apply a Binary/JSON/Text patch file"); @@ -7946,6 +8218,9 @@ fn print_usage(binary: &str) { eprintln!(" --import-resource-db SQLite ResourceRepository path"); eprintln!(" --snapshot Override snapshot path (default: /current/official-sync-snapshot.json)"); eprintln!(" --curl curl executable (default: curl)"); + eprintln!( + " --download-concurrency Bounded parallel downloads (default: 8, range 1..=256)" + ); eprintln!(" --proxy curl proxy override (default: auto from env)"); eprintln!(" --no-proxy Force direct curl connections"); eprintln!(" --unzip unzip executable (default: unzip)"); @@ -7970,6 +8245,9 @@ fn print_usage(binary: &str) { eprintln!(" --bundle-path Filter resource-index by metadata bundle path"); eprintln!(" --archive-entry Filter resource-index, parse detail, or translation-tasks by ZIP/archive entry"); eprintln!(" --task-status Filter translation-tasks by task status"); + eprintln!( + " --worker-status Filter translation-tasks by provider worker status" + ); eprintln!(" --parse-status Filter resource-index or translation-tasks by parse status"); eprintln!(" --path-id Filter parse detail by Unity object path ID"); eprintln!(" --class-id Filter parse detail by Unity class ID"); @@ -7978,6 +8256,9 @@ fn print_usage(binary: &str) { eprintln!( " --has-reason | --no-reason Filter translation-tasks by diagnostic reason presence" ); + eprintln!( + " --has-failure-reason | --no-failure-reason Filter translation-tasks by provider failure reason" + ); eprintln!(); eprintln!("Write patch:"); eprintln!(" --patch-kind Patch type for patch-apply"); @@ -8098,6 +8379,10 @@ mod tests { #[test] fn env_defaults_apply_and_cli_overrides() { + assert_eq!( + CliOptions::default().config.download_concurrency, + DEFAULT_DOWNLOAD_CONCURRENCY + ); let options = parse_with_env( &["bat"], &[ @@ -8109,6 +8394,7 @@ mod tests { ("BAT_AUTO_DISCOVER", "1"), ("BAT_STATE_DIR", "/srv/state"), ("BAT_INTERVAL_SECONDS", "120"), + ("BAT_DOWNLOAD_CONCURRENCY", "3"), ], ) .unwrap(); @@ -8127,6 +8413,7 @@ mod tests { Some(PathBuf::from("/srv/bat/resources.sqlite")) ); assert!(options.config.auto_discover); + assert_eq!(options.config.download_concurrency, 3); assert_eq!(options.state_dir, PathBuf::from("/srv/state")); assert_eq!(options.interval, Duration::from_secs(120)); @@ -8206,6 +8493,20 @@ mod tests { assert!(parse_with_env(&["bat"], &[("BAT_WATCH", "maybe")]).is_err()); assert!(parse_with_env(&["bat"], &[("BAT_INTERVAL_SECONDS", "abc")]).is_err()); assert!(parse_with_env(&["bat"], &[("BAT_PROXY", "ftp://x")]).is_err()); + assert!(parse_with_env(&["bat"], &[("BAT_DOWNLOAD_CONCURRENCY", "0")]).is_err()); + assert!(parse_with_env(&["bat"], &[("BAT_DOWNLOAD_CONCURRENCY", "257")]).is_err()); + assert!(parse(&["bat", "--download-concurrency", "0"]).is_err()); + assert!(parse(&["bat", "--download-concurrency", "257"]).is_err()); + } + + #[test] + fn cli_download_concurrency_is_preserved_for_daemon_child() { + let options = parse(&["bat", "--download-concurrency", "4"]).unwrap(); + assert_eq!(options.config.download_concurrency, 4); + let args = daemon_child_args(&options); + assert!(args + .windows(2) + .any(|pair| pair == ["--download-concurrency", "4"])); } #[test] @@ -11242,6 +11543,34 @@ mod tests { bat_infrastructure::write_textunit_task_queue_at(current_dir, &queue).unwrap(); } + fn write_translation_task_repository_fixture(current_dir: &Path) { + let queue = bat_infrastructure::read_textunit_task_queue_at(current_dir) + .unwrap() + .unwrap(); + let repository_path = + bat_infrastructure::SqliteTranslationTaskRepository::repository_path(current_dir); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let repository = + bat_infrastructure::SqliteTranslationTaskRepository::new(&repository_path) + .await + .unwrap(); + repository.sync_queue(&queue).await.unwrap(); + repository + .update_status( + "textunit/v-current/Bundle/a.bundle", + bat_infrastructure::TranslationTaskStatus::Failed, + Some("Crowdin provider rejected the payload".to_string()), + Some("crowdin-run-1".to_string()), + ) + .await + .unwrap(); + }); + } + fn write_textunit_index_fixture(current_dir: &Path) { let index = bat_infrastructure::OfficialTextUnitIndex { version: bat_infrastructure::OFFICIAL_TEXTUNIT_INDEX_VERSION, @@ -11326,6 +11655,7 @@ mod tests { let output_root = temp.path().join("output"); let current_dir = write_catalog_fixture(&state_dir, &output_root, "bundle-b2", None); write_textunit_task_queue_fixture(¤t_dir); + write_translation_task_repository_fixture(¤t_dir); let envelope = dispatch_rpc_method( &rpc_request( @@ -11392,6 +11722,80 @@ mod tests { value["data"]["entries"][0]["reason"], serde_json::Value::Null ); + + let envelope = dispatch_rpc_method( + &rpc_request( + "translation.tasks", + Some(serde_json::json!({ + "worker_status": "failed", + "has_failure_reason": true, + "offset": 0, + "limit": 10 + })), + ), + &state_dir, + &new_daemon_control(), + &test_task_context(), + "req-translation-3".to_string(), + ); + let value = serde_json::to_value(&envelope).unwrap(); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["task_repository_available"], true); + assert_eq!(value["data"]["total_entries"], 1); + assert_eq!(value["data"]["entries"][0]["task_status"], "failed"); + assert_eq!( + value["data"]["entries"][0]["failure_reason"], + "Crowdin provider rejected the payload" + ); + assert_eq!(value["data"]["entries"][0]["attempt_count"], 0); + + let envelope = dispatch_rpc_method( + &rpc_request( + "translation.task.update", + Some(serde_json::json!({ + "task_id": "textunit/v-current/Bundle/a.bundle", + "status": "completed", + "provider_run_id": "crowdin-run-2" + })), + ), + &state_dir, + &new_daemon_control(), + &test_task_context(), + "req-translation-4".to_string(), + ); + let value = serde_json::to_value(&envelope).unwrap(); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["entry"]["task_status"], "completed"); + assert_eq!( + value["data"]["entry"]["failure_reason"], + serde_json::Value::Null + ); + assert_eq!(value["data"]["entry"]["provider_run_id"], "crowdin-run-2"); + assert!(value["data"]["entry"]["completed_unix_seconds"].is_number()); + + let envelope = dispatch_rpc_method( + &rpc_request("translation.handoff", None), + &state_dir, + &new_daemon_control(), + &test_task_context(), + "req-translation-handoff".to_string(), + ); + let value = serde_json::to_value(&envelope).unwrap(); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["available"], true); + assert_eq!( + value["data"]["handoff_schema_version"], + bat_infrastructure::TRANSLATION_HANDOFF_SCHEMA_VERSION + ); + assert_eq!( + value["data"]["handoff"]["units"].as_array().unwrap().len(), + 2 + ); + assert_eq!(value["data"]["handoff"]["units"][0]["status"], "translated"); + assert_eq!( + value["data"]["handoff"]["provider_runs"][0]["provider_run_id"], + "crowdin-run-2" + ); } #[test] diff --git a/infrastructure/src/downloader.rs b/infrastructure/src/downloader.rs new file mode 100644 index 0000000..09eeb9c --- /dev/null +++ b/infrastructure/src/downloader.rs @@ -0,0 +1,260 @@ +//! Downloader backend and bounded scheduling contracts. +//! +//! The scheduler is deliberately independent from curl, manifests, and +//! official URL rules. Those concerns belong to a backend and the caller, +//! which keeps retry, proxy, and verification policy composable. + +use std::sync::{mpsc, Arc, Mutex}; + +/// Lowest supported download concurrency. +pub const MIN_DOWNLOAD_CONCURRENCY: usize = 1; +/// Highest supported download concurrency. +pub const MAX_DOWNLOAD_CONCURRENCY: usize = 256; +/// Default official download concurrency. +pub const DEFAULT_DOWNLOAD_CONCURRENCY: usize = 8; + +/// A backend that executes one already-planned download task. +pub trait DownloaderBackend: Send + Sync { + /// Successful result returned for one task. + type Output: Send; + /// Failure returned for one task. + type Error: Send; + + /// Executes one task. The scheduler owns ordering and concurrency only. + fn download(&self, task: T) -> Result; +} + +/// A bounded worker scheduler. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DownloadScheduler { + max_concurrency: usize, +} + +impl DownloadScheduler { + /// Creates a scheduler with the supported bounded range. + /// + /// The official CLI validates input and reports out-of-range values. + /// This lower-level constructor remains total for library callers and + /// clamps values to the same safety bounds. + pub fn new(max_concurrency: usize) -> Self { + Self { + max_concurrency: max_concurrency + .clamp(MIN_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY), + } + } + + /// Returns the configured upper bound. + pub fn max_concurrency(self) -> usize { + self.max_concurrency + } + + /// Executes tasks with a bounded number of workers. + /// + /// Results are returned in input order even when workers finish out of + /// order. A failed task does not cause additional tasks to be scheduled + /// after it, because already-started bounded work must be joined cleanly; + /// callers decide whether a failed result invalidates the whole release. + pub fn execute(self, backend: &B, tasks: Vec) -> Vec> + where + T: Send + 'static, + B: DownloaderBackend, + { + self.execute_with_observer( + backend, + tasks, + |_, _| Ok::<(), std::convert::Infallible>(()), + ) + .expect("infallible download observer cannot fail") + } + + /// Executes tasks and observes each result as soon as a worker returns it. + /// + /// The observer runs on the coordinator thread, while worker threads + /// immediately take another pending task after sending their result. An + /// observer error stops further observation but still drains and joins all + /// workers before returning, so no background transfer is left detached. + pub fn execute_with_observer( + self, + backend: &B, + tasks: Vec, + mut observer: F, + ) -> Result>, E> + where + T: Send + 'static, + B: DownloaderBackend, + F: FnMut(usize, &Result) -> Result<(), E>, + { + if tasks.is_empty() { + return Ok(Vec::new()); + } + + if self.max_concurrency == 1 { + let mut results = Vec::with_capacity(tasks.len()); + for (index, task) in tasks.into_iter().enumerate() { + let result = backend.download(task); + observer(index, &result)?; + results.push(result); + } + return Ok(results); + } + + let total = tasks.len(); + let worker_count = self.max_concurrency.min(total); + let pending = Arc::new(Mutex::new(tasks.into_iter().enumerate())); + let (result_sender, result_receiver) = mpsc::channel(); + + std::thread::scope(|scope| { + for _ in 0..worker_count { + let pending = Arc::clone(&pending); + let result_sender = result_sender.clone(); + scope.spawn(move || loop { + let task = pending + .lock() + .expect("download scheduler task queue poisoned") + .next(); + let Some((index, task)) = task else { + break; + }; + let result = backend.download(task); + if result_sender.send((index, result)).is_err() { + break; + } + }); + } + drop(result_sender); + + let mut results = std::iter::repeat_with(|| None) + .take(total) + .collect::>(); + let mut observer_error = None; + for (index, result) in result_receiver { + if observer_error.is_none() { + if let Err(error) = observer(index, &result) { + observer_error = Some(error); + } + } + results[index] = Some(result); + } + let results = results + .into_iter() + .map(|result| result.expect("download scheduler lost a task result")) + .collect(); + match observer_error { + Some(error) => Err(error), + None => Ok(results), + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::thread; + use std::time::Duration; + + struct TestBackend { + active: AtomicUsize, + max_active: AtomicUsize, + } + + impl DownloaderBackend for TestBackend { + type Output = usize; + type Error = String; + + fn download(&self, task: usize) -> Result { + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.max_active.fetch_max(active, Ordering::SeqCst); + thread::sleep(Duration::from_millis(2)); + self.active.fetch_sub(1, Ordering::SeqCst); + Ok(task * 2) + } + } + + #[test] + fn scheduler_preserves_result_order_and_respects_bound() { + let backend = TestBackend { + active: AtomicUsize::new(0), + max_active: AtomicUsize::new(0), + }; + let results = DownloadScheduler::new(2).execute(&backend, (0..8).collect()); + + assert_eq!( + results.into_iter().map(Result::unwrap).collect::>(), + (0..8).map(|value| value * 2).collect::>() + ); + assert!(backend.max_active.load(Ordering::SeqCst) <= 2); + assert!(backend.max_active.load(Ordering::SeqCst) >= 2); + } + + #[test] + fn zero_concurrency_is_conservative() { + assert_eq!( + DownloadScheduler::new(0).max_concurrency(), + MIN_DOWNLOAD_CONCURRENCY + ); + } + + #[test] + fn scheduler_caps_untrusted_upper_bound() { + assert_eq!( + DownloadScheduler::new(usize::MAX).max_concurrency(), + MAX_DOWNLOAD_CONCURRENCY + ); + } + + #[test] + fn observer_receives_completion_without_a_global_barrier() { + struct UnevenBackend { + active: AtomicUsize, + task_two_started_while_task_zero_active: AtomicUsize, + } + + impl DownloaderBackend for UnevenBackend { + type Output = usize; + type Error = String; + + fn download(&self, task: usize) -> Result { + if task == 0 { + self.active.fetch_add(1, Ordering::SeqCst); + thread::sleep(Duration::from_millis(20)); + self.active.fetch_sub(1, Ordering::SeqCst); + } else { + if task == 2 && self.active.load(Ordering::SeqCst) > 0 { + self.task_two_started_while_task_zero_active + .fetch_add(1, Ordering::SeqCst); + } + thread::sleep(Duration::from_millis(if task == 1 { 1 } else { 5 })); + } + Ok(task) + } + } + + let backend = UnevenBackend { + active: AtomicUsize::new(0), + task_two_started_while_task_zero_active: AtomicUsize::new(0), + }; + let mut completed = Vec::new(); + let results = DownloadScheduler::new(2) + .execute_with_observer(&backend, vec![0, 1, 2], |index, _| { + completed.push(index); + Ok::<(), ()>(()) + }) + .unwrap(); + + assert_eq!( + results.into_iter().map(Result::unwrap).collect::>(), + vec![0, 1, 2] + ); + assert_eq!(completed.len(), 3); + assert!(completed[0] == 1, "短任务应在长任务之前回传：{completed:?}"); + assert_eq!( + backend + .task_two_started_while_task_zero_active + .load(Ordering::SeqCst), + 1, + "worker 完成 task 1 后应立即领取 task 2" + ); + } +} diff --git a/infrastructure/src/lib.rs b/infrastructure/src/lib.rs index cf2dbdd..666160f 100644 --- a/infrastructure/src/lib.rs +++ b/infrastructure/src/lib.rs @@ -12,6 +12,7 @@ pub mod cas; mod curl_transfer; +pub mod downloader; pub mod import; pub mod localized_patch; pub mod official_changes; @@ -28,12 +29,17 @@ pub mod patch_ops; pub mod path_security; pub mod release_flow; pub mod resources; +pub mod translation_tasks; mod zip_validation; pub use cas::FileSystemCasRepository; pub use curl_transfer::{ redact_proxy_url, resolve_curl_proxy, CurlProxyConfig, CurlProxyMode, ResolvedCurlProxy, }; +pub use downloader::{ + DownloadScheduler, DownloaderBackend, DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY, + MIN_DOWNLOAD_CONCURRENCY, +}; pub use import::{ BundleSource, ImportedResource, ResourceImportCategory, ResourceImportReport, ResourceImportService, @@ -124,6 +130,14 @@ pub use path_security::{ }; pub use release_flow::ReleaseFlowStatusCode; pub use resources::{InMemoryResourceRepository, SqliteResourceRepository}; +pub use translation_tasks::{ + build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at, + write_translation_handoff_at, PersistedTranslationTask, ProviderRun, ProviderRunStatus, + SqliteTranslationTaskRepository, TranslationHandoff, TranslationJob, TranslationJobStatus, + TranslationTaskStatus, TranslationTaskSyncReport, TranslationUnit, TranslationUnitStatus, + TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE, + TRANSLATION_TASK_SCHEMA_VERSION, +}; /// Infrastructure 版本号 pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/infrastructure/src/official_download.rs b/infrastructure/src/official_download.rs index d3a3c5f..f4ebfdc 100644 --- a/infrastructure/src/official_download.rs +++ b/infrastructure/src/official_download.rs @@ -1,6 +1,10 @@ //! Official JP resource download execution. use crate::curl_transfer::{run_curl_with_retry_with_proxy, CurlProxyConfig, CurlRetryError}; +use crate::downloader::{ + DownloadScheduler, DownloaderBackend, DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY, + MIN_DOWNLOAD_CONCURRENCY, +}; use crate::official_pull::OfficialResourcePullPlan; use crate::path_security::{ ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, @@ -9,7 +13,10 @@ use crate::path_security::{ use crate::zip_validation::{ path_has_zip_extension, url_or_path_has_zip_extension, validate_zip_structure, }; -use bat_adapters::official::yostar_jp::{is_official_yostar_jp_url, YostarJpResourceEndpointKind}; +use bat_adapters::official::yostar_jp::YostarJpResourceEndpointKind; +use bat_adapters::official::{ + destination_under_root, DownloadUrlMapper, OfficialResourceBackend, YostarJpBackend, +}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashSet}; use std::fs::{self, File}; @@ -539,9 +546,11 @@ impl OfficialLocalResourceState { #[derive(Debug, Clone)] pub struct OfficialResourcePullService { output_root: PathBuf, + backend: YostarJpBackend, curl_command: PathBuf, curl_proxy: CurlProxyConfig, retry_attempts: usize, + max_concurrency: usize, } impl OfficialResourcePullService { @@ -557,9 +566,11 @@ impl OfficialResourcePullService { ) -> Self { Self { output_root: output_root.into(), + backend: YostarJpBackend, curl_command: curl_command.into(), curl_proxy: CurlProxyConfig::default(), retry_attempts: DEFAULT_RETRY_ATTEMPTS, + max_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY, } } @@ -575,6 +586,22 @@ impl OfficialResourcePullService { self } + /// Sets the bounded number of concurrent downloads. + /// + /// The default is [`DEFAULT_DOWNLOAD_CONCURRENCY`]. Values are kept + /// within the supported `1..=256` range; the public update configuration + /// validates input before constructing this service. + pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self { + self.max_concurrency = + max_concurrency.clamp(MIN_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY); + self + } + + /// Returns the configured download concurrency. + pub fn max_concurrency(&self) -> usize { + self.max_concurrency + } + /// Returns the output root used for downloaded files. pub fn output_root(&self) -> &Path { &self.output_root @@ -666,7 +693,7 @@ impl OfficialResourcePullService { } let mut planned: Vec = Vec::with_capacity(total); for url in urls { - if !is_official_yostar_jp_url(&url) { + if !self.backend.is_official_url(&url) { return Err(DownloadError::new( bat_core::ErrorCode::NON_OFFICIAL_URL, format!("拒绝下载非官方 URL：{url}"), @@ -695,6 +722,17 @@ impl OfficialResourcePullService { }); } + if self.max_concurrency > 1 { + return self.pull_planned_concurrently( + planned, + manifest, + official_hash_pairs, + total, + &mut progress, + &mut should_cancel, + ); + } + // Phase B：按 plan 顺序处理每个 URL。下载或复用完成并写入 manifest 后， // 立即尝试校验已经到齐的官方 `.bytes/.hash` pair，避免把文件级问题延后到整轮末尾。 let mut completed_count = 0usize; @@ -824,6 +862,183 @@ impl OfficialResourcePullService { }) } + fn pull_planned_concurrently( + &self, + planned: Vec, + mut manifest: OfficialDownloadManifest, + official_hash_pairs: Vec, + total: usize, + progress: &mut impl FnMut(OfficialResourcePullProgress), + should_cancel: &mut impl FnMut() -> bool, + ) -> Result { + if should_cancel() { + return Err("官方资源拉取已被停止请求中断".to_string().into()); + } + for item in &planned { + progress(OfficialResourcePullProgress::started( + 0, + total, + item.url.clone(), + )); + } + + let backend = CurlDownloadBackend { service: self }; + let mut completed_count = 0usize; + let mut items_by_plan_index: Vec> = + (0..total).map(|_| None).collect(); + let mut verified_hashes = Vec::new(); + let mut verified_hash_urls = HashSet::::new(); + let mut processed_urls = HashSet::::new(); + + DownloadScheduler::new(self.max_concurrency).execute_with_observer( + &backend, + planned.clone(), + |plan_index, result| { + let item = &planned[plan_index]; + if should_cancel() { + return Err("官方资源拉取已被停止请求中断".to_string().into()); + } + + let needs_manifest = item.existing.is_none(); + let verification = match result { + Ok(pull_result) => { + let verification_result = self + .clear_quarantine_entry(&item.url) + .and_then(|_| { + if needs_manifest { + self.record_download_manifest_entry( + &mut manifest, + &item.url, + &item.destination, + ) + } else { + Ok(pull_result.verification.clone()) + } + }) + .and_then(|verification| { + if needs_manifest { + self.write_download_manifest(&manifest) + .map(|_| verification) + } else { + Ok(verification) + } + }); + match verification_result { + Ok(verification) => verification, + Err(error) => { + let error = PullOneError::plain(format!( + "记录下载 manifest 失败：{error}" + )); + self.record_quarantine_entry( + &item.url, + &item.destination, + &error, + )?; + progress(OfficialResourcePullProgress::failed( + completed_count, + total, + item.url.clone(), + &error, + )); + return Err(DownloadError::new( + error.error_code(), + format!( + "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", + item.url, + self.download_quarantine_path().display(), + error.message + ), + )); + } + } + } + Err(error) => { + self.record_quarantine_entry(&item.url, &item.destination, error)?; + progress(OfficialResourcePullProgress::failed( + completed_count, + total, + item.url.clone(), + error, + )); + return Err(DownloadError::new( + error.error_code(), + format!( + "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", + item.url, + self.download_quarantine_path().display(), + error.message + ), + )); + } + }; + + completed_count += 1; + progress(OfficialResourcePullProgress::finished( + completed_count, + total, + item.url.clone(), + result + .as_ref() + .expect("successful result handled above") + .status, + result + .as_ref() + .expect("successful result handled above") + .bytes, + result + .as_ref() + .expect("successful result handled above") + .transferred_bytes, + verification, + )); + processed_urls.insert(item.url.clone()); + let newly_verified_hashes = self.verify_ready_official_hashes( + &official_hash_pairs, + &processed_urls, + &mut verified_hash_urls, + &mut verified_hashes, + &mut manifest, + )?; + for verification in newly_verified_hashes { + progress(OfficialResourcePullProgress::verification( + completed_count, + total, + verification.data_url.clone(), + verification, + )); + } + let pull_result = result + .as_ref() + .expect("successful result handled above"); + items_by_plan_index[plan_index] = Some(OfficialResourcePullItem { + url: item.url.clone(), + destination: item.destination.clone(), + bytes: pull_result.bytes, + transferred_bytes: pull_result.transferred_bytes, + status: pull_result.status, + }); + Ok(()) + }, + )?; + + self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?; + let items = items_by_plan_index + .into_iter() + .enumerate() + .map(|(index, item)| { + item.ok_or_else(|| { + DownloadError::from(format!( + "并发下载结果缺少 plan index={index}，拒绝发布不完整资源" + )) + }) + }) + .collect::, _>>()?; + Ok(OfficialResourcePullReport { + items, + verified_hashes, + }) + } + /// Audits every URL in a pull plan against the local download manifest. /// /// This performs no network I/O. It checks that each URL has a manifest @@ -914,7 +1129,7 @@ impl OfficialResourcePullService { /// `TableCatalog.bytes`, `BundlePackingInfo.bytes`, and /// `MediaCatalog.bytes`. pub fn fetch_bytes(&self, url: &str) -> Result, DownloadError> { - if !is_official_yostar_jp_url(url) { + if !self.backend.is_official_url(url) { return Err(DownloadError::new( bat_core::ErrorCode::NON_OFFICIAL_URL, format!("拒绝拉取非官方 URL：{url}"), @@ -1581,36 +1796,12 @@ impl OfficialResourcePullService { fn destination_for_url(&self, url: &str) -> Result { self.ensure_output_root_safe()?; - if !is_official_yostar_jp_url(url) { + if !self.backend.is_official_url(url) { return Err(format!("URL 不是官方 JP host：{url}")); } - let rest = url - .strip_prefix("https://") - .ok_or_else(|| format!("官方 URL 必须使用 https：{url}"))?; - let (host, path) = rest - .split_once('/') - .ok_or_else(|| format!("官方 URL 缺少路径：{url}"))?; - - let mut relative_destination = PathBuf::from(sanitize_segment(host)); - for segment in path.split('/') { - if segment.is_empty() { - continue; - } - if segment == "." || segment == ".." { - return Err(format!("官方 URL 包含不安全路径片段：{url}")); - } - - // 官方资源 URL 不携带 query/fragment。若出现则直接拒绝，而非静默剥除—— - // 否则仅 query 不同的两个 URL 会映射到同一目标文件而相互覆盖， - // 并导致每轮 hash 复用校验失配、反复重下。 - if segment.contains('?') || segment.contains('#') { - return Err(format!("官方资源 URL 不允许包含 query 或 fragment：{url}")); - } - relative_destination.push(sanitize_segment(segment)); - } - - let destination = self.output_root.join(relative_destination); + let relative_destination = self.backend.relative_destination(url)?; + let destination = destination_under_root(&self.output_root, &relative_destination)?; ensure_path_within_root(&self.output_root, &destination)?; Ok(destination) } @@ -1834,6 +2025,7 @@ fn default_download_quarantine_version() -> u32 { /// Phase A 产出的单个下载计划项：URL、目标路径，以及若命中本地 manifest /// 校验则带上「已验证可跳过」的结果（`existing`）。 +#[derive(Debug, Clone)] struct PlannedDownload { url: String, destination: PathBuf, @@ -1854,6 +2046,21 @@ struct PullOneError { retry_error: Option, } +struct CurlDownloadBackend<'a> { + service: &'a OfficialResourcePullService, +} + +impl DownloaderBackend for CurlDownloadBackend<'_> { + type Output = PullOneResult; + type Error = PullOneError; + + fn download(&self, task: PlannedDownload) -> Result { + task.existing + .map(Ok) + .unwrap_or_else(|| self.service.pull_one(&task.url, &task.destination)) + } +} + impl PullOneError { fn plain(message: String) -> Self { Self { @@ -2088,20 +2295,6 @@ fn read_u32_le(bytes: &[u8], offset: usize) -> u32 { ]) } -fn sanitize_segment(segment: &str) -> String { - segment - .chars() - .map(|ch| { - if ch.is_control() || matches!(ch, '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|') - { - '_' - } else { - ch - } - }) - .collect() -} - #[cfg(test)] mod tests { use super::*; @@ -3263,21 +3456,26 @@ exit 22 .official_hash .as_ref() .expect("verification event must carry official hash detail"); - let hash_finished_index = events - .iter() - .position(|event| { - event.kind == OfficialResourcePullProgressKind::Finished - && event.url == hash.hash_url + let pair_finished_indices = [hash.data_url.as_str(), hash.hash_url.as_str()] + .into_iter() + .map(|url| { + events + .iter() + .position(|event| { + event.kind == OfficialResourcePullProgressKind::Finished + && event.url == url + }) + .expect("hash pair member must finish before verification") }) - .expect("hash sidecar must finish before verification"); + .collect::>(); let verification_index = events .iter() .position(|event| std::ptr::eq(event, *verification_event)) .expect("verification event must be present in event stream"); assert_eq!( verification_index, - hash_finished_index + 1, - "official hash verification must run immediately after sidecar is complete" + pair_finished_indices.into_iter().max().unwrap() + 1, + "official hash verification must run immediately after the hash pair is complete" ); } @@ -3320,7 +3518,8 @@ exit 22 write_fake_curl(&curl_path); // 顺序下载：每个 URL 恰好一次 started + 一次 finished，全部文件落盘。 - let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) + .with_max_concurrency(1); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), @@ -3354,6 +3553,71 @@ exit 22 assert_eq!(manifest.entries.len(), all_urls.len()); } + #[test] + fn downloads_run_with_bounded_concurrency_and_keep_report_order() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) + .with_max_concurrency(3); + let plan = build_official_pull_plan_for_platforms( + discovery_plan(), + inventory(), + &[PatchPlatform::Windows], + ); + let all_urls = plan.all_urls().unwrap(); + let mut events = Vec::new(); + let report = service + .pull_with_progress(&plan, |event| events.push(event)) + .unwrap(); + + assert_eq!(service.max_concurrency(), 3); + assert_eq!(report.items.len(), all_urls.len()); + assert_eq!( + report + .items + .iter() + .map(|item| &item.url) + .collect::>(), + all_urls.iter().collect::>() + ); + for url in &all_urls { + assert_eq!( + events + .iter() + .filter(|event| { + event.url == *url && event.kind == OfficialResourcePullProgressKind::Started + }) + .count(), + 1, + "url {url} 的 started 次数" + ); + assert_eq!( + events + .iter() + .filter(|event| { + event.url == *url + && event.kind == OfficialResourcePullProgressKind::Finished + }) + .count(), + 1, + "url {url} 的 finished 次数" + ); + } + let finished_indices = events + .iter() + .filter(|event| event.kind == OfficialResourcePullProgressKind::Finished) + .map(|event| event.index) + .collect::>(); + assert_eq!(finished_indices, (1..=all_urls.len()).collect::>()); + assert_eq!( + service.read_download_manifest().unwrap().entries.len(), + all_urls.len() + ); + } + #[test] fn retries_transient_download_failures() { let out_dir = TempDir::new().unwrap(); diff --git a/infrastructure/src/official_textunit_queue.rs b/infrastructure/src/official_textunit_queue.rs index 9bd7df6..f3c2e15 100644 --- a/infrastructure/src/official_textunit_queue.rs +++ b/infrastructure/src/official_textunit_queue.rs @@ -16,6 +16,7 @@ use crate::path_security::{ ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, STATE_FILE_MODE, }; +use crate::sync_translation_task_repository_at; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -121,6 +122,10 @@ pub struct OfficialTextUnitTaskQuery { pub text_unit_format: Option, /// Filter tasks by whether a diagnostic reason is present. pub has_reason: Option, + /// Filter tasks by whether a provider failure reason is present. + pub has_failure_reason: Option, + /// Filter by mutable provider-worker status. + pub task_status: Option, } /// Aggregate counters for an incremental TextUnit task queue. @@ -355,6 +360,16 @@ pub fn write_official_textunit_queues( write_textunit_task_queue_at(resource_root, &task_queue)?; let task_queue_path = resource_root.join(OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("构建官方 TextUnit 任务状态同步运行时失败：{error}"))?; + runtime + .block_on(sync_translation_task_repository_at( + resource_root, + &task_queue, + )) + .map_err(|error| format!("同步官方 TextUnit 任务状态到 SQLite 失败：{error}"))?; let crowdin_queue = CrowdinTextUnitQueue::from_textunit_task_queue(task_queue_path.clone(), &task_queue); write_crowdin_textunit_queue_at(resource_root, &crowdin_queue)?; @@ -492,7 +507,10 @@ fn parse_entries_by_destination( by_destination } -fn textunit_task_matches(task: &OfficialTextUnitTask, query: &OfficialTextUnitTaskQuery) -> bool { +pub(crate) fn textunit_task_matches( + task: &OfficialTextUnitTask, + query: &OfficialTextUnitTaskQuery, +) -> bool { if query .task_id .as_ref() diff --git a/infrastructure/src/official_update.rs b/infrastructure/src/official_update.rs index 1d66479..95b1885 100644 --- a/infrastructure/src/official_update.rs +++ b/infrastructure/src/official_update.rs @@ -29,6 +29,9 @@ use crate::path_security::{ read_file_no_symlink, validate_output_root, write_file_atomic, STATE_FILE_MODE, }; use crate::release_flow::ReleaseFlowStatusCode; +use crate::translation_tasks::{ + build_translation_handoff, sync_translation_task_repository_at, write_translation_handoff_at, +}; use crate::{ build_official_pull_plan_for_platform_inventory, build_official_sync_plan, changed_endpoint_urls, default_official_platforms, DownloadError, @@ -42,15 +45,17 @@ use crate::{ read_parse_cache_at, OfficialParseCacheService, OfficialParseConfig, OfficialParseSummary, }; use crate::{FileSystemCasRepository, SqliteResourceRepository}; +use crate::{DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY, MIN_DOWNLOAD_CONCURRENCY}; use bat_adapters::official::game_main_config::YostarJpGameMainConfig; -use bat_adapters::official::inventory::{ - YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, -}; use bat_adapters::official::launcher::YostarJpLauncherManifestFile; use bat_adapters::official::yostar_jp::{ - server_info_url, PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, + PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, YostarJpResourceEndpointKind, YostarJpServerInfo, YostarJpSyncSnapshot, }; +use bat_adapters::official::{ + OfficialResourceBackend, PlatformCatalogInput, YostarJpBackend, + YostarJpPlatformDownloadInventory, +}; use bat_core::ErrorCode; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -115,6 +120,10 @@ pub struct OfficialUpdateConfig { pub curl_command: PathBuf, /// Proxy selection used by all official `curl` transfers. pub curl_proxy: CurlProxyConfig, + /// Maximum number of resource downloads executed concurrently. + /// + /// `8` is the default; values are accepted only in `1..=256`. + pub download_concurrency: usize, /// Unzip command used when a metadata change requires GameMainConfig parsing. pub unzip_command: PathBuf, /// Dry run reports decisions and optional plan URLs without writing sync state. @@ -151,6 +160,7 @@ impl Default for OfficialUpdateConfig { snapshot_path: None, curl_command: PathBuf::from("curl"), curl_proxy: CurlProxyConfig::default(), + download_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY, unzip_command: PathBuf::from("unzip"), dry_run: false, plan: false, @@ -844,6 +854,8 @@ pub struct OfficialUpdateReport { pub crowdin_textunit_queue_path: Option, /// Incremental TextUnit task queue summary. pub textunit_task_summary: Option, + /// Versioned translation worker handoff view. + pub translation_handoff_path: Option, /// Snapshot path written after success. pub snapshot_written: Option, /// Launcher bootstrap artifact written for this run. @@ -1221,7 +1233,8 @@ impl OfficialUpdateService { &active_resource_root, &config.curl_command, ) - .with_proxy_config(config.curl_proxy.clone()); + .with_proxy_config(config.curl_proxy.clone()) + .with_max_concurrency(config.download_concurrency); let snapshot_path = snapshot_path_for(config, &active_resource_root); let bootstrap_cache_path = config.bootstrap_cache_path(); @@ -1668,6 +1681,7 @@ impl OfficialUpdateService { textunit_task_queue_path: None, crowdin_textunit_queue_path: None, textunit_task_summary: None, + translation_handoff_path: None, snapshot_written: None, launcher_bootstrap_artifact_path: None, repository_import_enabled: config.import_repository, @@ -1846,7 +1860,8 @@ impl OfficialUpdateService { &publish_plan.staging_path, &config.curl_command, ) - .with_proxy_config(config.curl_proxy.clone()); + .with_proxy_config(config.curl_proxy.clone()) + .with_max_concurrency(config.download_concurrency); let pruned_stale_resource_count = staging_fetcher .prune_stale_manifest_entries(&pull_plan) .map_err(anyhow::Error::msg)?; @@ -2201,6 +2216,7 @@ fn run_post_sync_textunit_queue_if_needed( && is_crowdin_textunit_queue_current(resource_root, &queue) .map_err(anyhow::Error::msg)? { + sync_translation_task_repository(resource_root, &queue)?; progress(OfficialUpdateProgress::new( "textunit", format!( @@ -2212,7 +2228,8 @@ fn run_post_sync_textunit_queue_if_needed( )); report.textunit_task_queue_path = Some(task_queue_path); report.crowdin_textunit_queue_path = Some(crowdin_queue_path); - report.textunit_task_summary = Some(queue.summary); + report.textunit_task_summary = Some(queue.summary.clone()); + write_translation_handoff_for_release(resource_root, &queue, report, progress)?; return Ok(()); } progress(OfficialUpdateProgress::new( @@ -2227,6 +2244,19 @@ fn run_post_sync_textunit_queue_if_needed( run_post_sync_textunit_queue(resource_root, report, progress, should_cancel) } +fn sync_translation_task_repository( + resource_root: &Path, + queue: &crate::official_textunit_queue::OfficialTextUnitTaskQueue, +) -> anyhow::Result<()> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime + .block_on(sync_translation_task_repository_at(resource_root, queue)) + .map_err(|error| anyhow::anyhow!("{error}"))?; + Ok(()) +} + fn run_post_sync_textunit_queue( resource_root: &Path, report: &mut OfficialUpdateReport, @@ -2257,6 +2287,11 @@ fn run_post_sync_textunit_queue( ), )); apply_textunit_queue_report(report, queue_report); + if let Some(queue) = + read_textunit_task_queue_at(resource_root).map_err(anyhow::Error::msg)? + { + write_translation_handoff_for_release(resource_root, &queue, report, progress)?; + } } Err(error) => { progress(OfficialUpdateProgress::new( @@ -2277,6 +2312,48 @@ fn apply_textunit_queue_report( report.textunit_task_summary = Some(queue_report.summary); } +fn write_translation_handoff_for_release( + resource_root: &Path, + queue: &crate::official_textunit_queue::OfficialTextUnitTaskQueue, + report: &mut OfficialUpdateReport, + progress: &mut dyn FnMut(OfficialUpdateProgress), +) -> anyhow::Result<()> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let tasks = runtime.block_on(async { + let repository = crate::translation_tasks::SqliteTranslationTaskRepository::new( + crate::translation_tasks::SqliteTranslationTaskRepository::repository_path( + resource_root, + ), + ) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .sync_queue(queue) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .list(&crate::official_textunit_queue::OfficialTextUnitTaskQuery::default()) + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })?; + let handoff = build_translation_handoff(queue, &tasks); + write_translation_handoff_at(resource_root, &handoff).map_err(anyhow::Error::msg)?; + let path = resource_root.join(crate::translation_tasks::TRANSLATION_HANDOFF_FILE); + report.translation_handoff_path = Some(path.clone()); + progress(OfficialUpdateProgress::new( + "textunit", + format!( + "翻译 handoff 已发布：任务={} provider_run={} {}", + handoff.units.len(), + handoff.provider_runs.len(), + path.display() + ), + )); + Ok(()) +} + fn run_post_sync_repository_import( config: &OfficialUpdateConfig, resource_root: &Path, @@ -2370,8 +2447,9 @@ fn build_pull_plan( should_cancel: &mut dyn FnMut() -> bool, ) -> anyhow::Result { check_shutdown_requested(should_cancel)?; - let discovery = server_info - .discovery_plan(connection_group, app_version, platforms) + let backend = YostarJpBackend; + let discovery = backend + .discovery_plan(server_info, connection_group, app_version, platforms) .map_err(anyhow::Error::msg)?; progress(OfficialUpdateProgress::new( "plan", @@ -2402,7 +2480,9 @@ fn load_server_info( match source { OfficialServerInfoSource::LocalPath(path) => Ok(fs::read(path)?), OfficialServerInfoSource::OfficialFile(file_name) => { - let url = server_info_url(file_name).map_err(anyhow::Error::msg)?; + let url = YostarJpBackend + .server_info_url(file_name) + .map_err(anyhow::Error::msg)?; fetcher.fetch_bytes(&url).map_err(anyhow::Error::new) } OfficialServerInfoSource::OfficialUrl(url) => { @@ -2489,6 +2569,7 @@ fn waiting_for_official_resources_report( textunit_task_queue_path: None, crowdin_textunit_queue_path: None, textunit_task_summary: None, + translation_handoff_path: None, snapshot_written: None, launcher_bootstrap_artifact_path: None, repository_import_enabled: config.import_repository, @@ -2894,6 +2975,12 @@ pub fn diff_extended_snapshot( } fn validate_update_paths(config: &OfficialUpdateConfig) -> Result<(), String> { + if !(MIN_DOWNLOAD_CONCURRENCY..=MAX_DOWNLOAD_CONCURRENCY).contains(&config.download_concurrency) + { + return Err(format!( + "下载并发数必须在 {MIN_DOWNLOAD_CONCURRENCY}..={MAX_DOWNLOAD_CONCURRENCY} 范围内" + )); + } validate_output_root(&config.output_root)?; validate_output_root(&config.localized_output_root)?; validate_separate_output_roots(&config.output_root, &config.localized_output_root)?; @@ -4047,17 +4134,14 @@ fn build_inventory_from_seed_catalogs( anyhow::anyhow!("官方发现结果缺少 {} MediaCatalog.bytes", platform.as_str()) })?; - platform_catalogs.push(YostarJpPlatformCatalogInventory::from_catalog_bytes( - *platform, + platform_catalogs.push(PlatformCatalogInput { + platform: *platform, bundle_packing_info, media_catalog, - )); + }); } - Ok(YostarJpPlatformDownloadInventory::from_catalog_bytes( - table_catalog, - platform_catalogs, - )) + Ok(YostarJpBackend.inventory(table_catalog, &platform_catalogs)) } #[derive(Debug, Default)] diff --git a/infrastructure/src/translation_tasks.rs b/infrastructure/src/translation_tasks.rs new file mode 100644 index 0000000..026ab55 --- /dev/null +++ b/infrastructure/src/translation_tasks.rs @@ -0,0 +1,1288 @@ +//! Durable translation-task state for official TextUnit handoff. +//! +//! The official TextUnit queue remains an immutable release artifact. This +//! module stores the mutable worker state separately so a provider worker can +//! retry or complete a task without rewriting the published release. + +use crate::official_textunit_queue::{ + textunit_task_matches, OfficialTextUnitTask, OfficialTextUnitTaskQuery, + OfficialTextUnitTaskQueue, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE, +}; +use crate::path_security::{ + ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, + STATE_FILE_MODE, +}; +use bat_core::Result; +use serde::{Deserialize, Serialize}; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::str::FromStr; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// SQLite schema version for durable translation task state. +pub const TRANSLATION_TASK_SCHEMA_VERSION: u32 = 1; +const TRANSLATION_TASK_SCHEMA_COMPONENT: &str = "translation_tasks"; +/// SQLite file name stored under a published official release root. +pub const TRANSLATION_TASK_REPOSITORY_FILE: &str = "translation-tasks.sqlite"; +/// Translation handoff schema version. +pub const TRANSLATION_HANDOFF_SCHEMA_VERSION: u32 = 1; +/// Versioned handoff file stored under a published official release root. +pub const TRANSLATION_HANDOFF_FILE: &str = "translation-handoff.json"; + +/// Lifecycle status for one translation job. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationJobStatus { + /// No provider work has started. + Queued, + /// At least one provider run is active. + Translating, + /// Provider output is waiting for human review. + Review, + /// All source units have translated output ready for patching. + Ready, + /// A localized release was published. + Published, + /// One or more provider runs failed. + Failed, +} + +impl TranslationJobStatus { + /// Returns the stable handoff label. + pub fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Translating => "translating", + Self::Review => "review", + Self::Ready => "ready", + Self::Published => "published", + Self::Failed => "failed", + } + } +} + +/// Lifecycle status for one resource/TextUnit task in a translation job. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationUnitStatus { + /// Waiting for a provider. + Queued, + /// A provider is processing the unit. + Translating, + /// Provider output exists but is not yet reviewed. + Translated, + /// Human review accepted the output. + Reviewed, + /// A localized patch was generated. + Patched, + /// The localized release contains this unit. + Published, + /// Provider processing failed. + Failed, + /// The source task was skipped by parsing or policy. + Skipped, +} + +/// Status of one provider execution recorded in a translation handoff. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProviderRunStatus { + /// Waiting to be scheduled. + Queued, + /// Provider work is active. + Running, + /// Provider work completed. + Succeeded, + /// Provider work failed. + Failed, + /// Provider work was cancelled. + Cancelled, +} + +/// Durable job-level translation state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TranslationJob { + /// Stable job ID derived from the official release. + pub job_id: String, + /// Official release consumed by this job. + pub official_release_id: String, + /// Previous official release, when known. + pub previous_release_id: Option, + /// Localized release produced by a later patch/publish stage. + pub localized_release_id: Option, + /// Current job lifecycle status. + pub status: TranslationJobStatus, + /// Number of source units represented by the handoff. + pub unit_count: usize, + /// Queue generation time. + pub created_unix_seconds: u64, + /// Last handoff state generation time. + pub updated_unix_seconds: u64, + /// Job-level diagnostic, if failed. + pub failure_reason: Option, +} + +/// One translation unit linked back to an immutable official task. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TranslationUnit { + /// Stable unit ID. + pub unit_id: String, + /// Owning translation job. + pub job_id: String, + /// Immutable official TextUnit task ID. + pub task_id: String, + /// Official release containing the source. + pub official_release_id: String, + /// Resource destination under the official release. + pub destination: String, + /// ZIP/archive entry, when the source was nested. + pub archive_entry: Option, + /// Number of extracted TextUnits represented by this task. + pub text_unit_count: usize, + /// Extracted TextUnit format labels. + pub text_unit_formats: Vec, + /// Current translation lifecycle state. + pub status: TranslationUnitStatus, + /// Provider failure diagnostic, when present. + pub failure_reason: Option, +} + +/// One provider execution associated with one or more translation units. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProviderRun { + /// Stable provider run ID. + pub provider_run_id: String, + /// Owning translation job. + pub job_id: String, + /// Provider identifier, reserved for future plugin implementations. + pub provider: String, + /// Current provider run status. + pub status: ProviderRunStatus, + /// Unit IDs submitted to this run. + pub unit_ids: Vec, + /// Number of attempts represented by this run. + pub attempt_count: u32, + /// Provider failure diagnostic, when present. + pub failure_reason: Option, +} + +/// Read-only handoff view consumed by translation workers and query clients. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TranslationHandoff { + /// Handoff schema version. + pub handoff_version: u32, + /// Official queue generation time. + pub generated_unix_seconds: u64, + /// Stable source queue file name. + pub source_queue_file: String, + /// Job-level state. + pub job: TranslationJob, + /// Resource/TextUnit task states. + pub units: Vec, + /// Provider run states. + pub provider_runs: Vec, +} + +/// Builds a translation handoff from immutable queue data and mutable worker +/// state. No provider or network call is made. +pub fn build_translation_handoff( + queue: &OfficialTextUnitTaskQueue, + tasks: &[PersistedTranslationTask], +) -> TranslationHandoff { + let job_id = format!("official-release:{}", queue.official_release_id); + let persisted = tasks + .iter() + .map(|task| (task.task.task_id.as_str(), task)) + .collect::>(); + let mut units = Vec::with_capacity(queue.tasks.len()); + let mut provider_runs = BTreeMap::::new(); + + for task in &queue.tasks { + let state = persisted + .get(task.task_id.as_str()) + .copied() + .map(|task| (task.task_status, task.failure_reason.clone(), task.attempt_count, task.provider_run_id.clone())) + .unwrap_or_else(|| { + ( + initial_task_status(task), + if task.status == crate::official_textunit_queue::OfficialTextUnitTaskStatus::SkippedParseFailed { + task.reason.clone() + } else { + None + }, + 0, + None, + ) + }); + let unit_status = match state.0 { + TranslationTaskStatus::Queued => TranslationUnitStatus::Queued, + TranslationTaskStatus::Running => TranslationUnitStatus::Translating, + TranslationTaskStatus::Failed => TranslationUnitStatus::Failed, + TranslationTaskStatus::Completed => TranslationUnitStatus::Translated, + TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped, + }; + let unit = TranslationUnit { + unit_id: task.task_id.clone(), + job_id: job_id.clone(), + task_id: task.task_id.clone(), + official_release_id: task.official_release_id.clone(), + destination: task.destination.clone(), + archive_entry: task.archive_entry.clone(), + text_unit_count: task.text_unit_count, + text_unit_formats: task.text_unit_formats.clone(), + status: unit_status, + failure_reason: state.1.clone(), + }; + if let Some(provider_run_id) = state.3 { + let run = provider_runs + .entry(provider_run_id.clone()) + .or_insert_with(|| ProviderRun { + provider_run_id, + job_id: job_id.clone(), + provider: "worker".to_string(), + status: ProviderRunStatus::Queued, + unit_ids: Vec::new(), + attempt_count: 0, + failure_reason: None, + }); + run.unit_ids.push(unit.unit_id.clone()); + run.attempt_count = run.attempt_count.max(state.2); + let candidate_status = match state.0 { + TranslationTaskStatus::Running => ProviderRunStatus::Running, + TranslationTaskStatus::Failed => ProviderRunStatus::Failed, + TranslationTaskStatus::Completed => ProviderRunStatus::Succeeded, + TranslationTaskStatus::Queued | TranslationTaskStatus::Skipped => run.status, + }; + if provider_run_status_rank(candidate_status) > provider_run_status_rank(run.status) { + run.status = candidate_status; + } + if state.1.is_some() { + run.failure_reason = state.1.clone(); + } + } + units.push(unit); + } + + let job_failure_reason = units.iter().find_map(|unit| unit.failure_reason.clone()); + let has_running = units + .iter() + .any(|unit| unit.status == TranslationUnitStatus::Translating); + let candidate_units = units + .iter() + .filter(|unit| unit.status != TranslationUnitStatus::Skipped) + .collect::>(); + let all_translated = !candidate_units.is_empty() + && candidate_units + .iter() + .all(|unit| unit.status == TranslationUnitStatus::Translated); + let job_status = if job_failure_reason.is_some() { + TranslationJobStatus::Failed + } else if has_running { + TranslationJobStatus::Translating + } else if all_translated { + TranslationJobStatus::Ready + } else { + TranslationJobStatus::Queued + }; + + TranslationHandoff { + handoff_version: TRANSLATION_HANDOFF_SCHEMA_VERSION, + generated_unix_seconds: queue.generated_unix_seconds, + source_queue_file: OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE.to_string(), + job: TranslationJob { + job_id, + official_release_id: queue.official_release_id.clone(), + previous_release_id: queue.previous_release_id.clone(), + localized_release_id: None, + status: job_status, + unit_count: units.len(), + created_unix_seconds: queue.generated_unix_seconds, + updated_unix_seconds: unix_seconds_now(), + failure_reason: job_failure_reason, + }, + units, + provider_runs: provider_runs.into_values().collect(), + } +} + +/// Writes a reviewed handoff view under one release root. +pub fn write_translation_handoff_at( + resource_root: &Path, + handoff: &TranslationHandoff, +) -> std::result::Result<(), String> { + let path = resource_root.join(TRANSLATION_HANDOFF_FILE); + ensure_path_within_root(resource_root, &path)?; + ensure_safe_file_target(resource_root, &path, "翻译 handoff")?; + let bytes = serde_json::to_vec_pretty(handoff) + .map_err(|error| format!("序列化翻译 handoff 失败 {}：{error}", path.display()))?; + write_file_atomic(&path, &bytes, STATE_FILE_MODE, "翻译 handoff") +} + +/// Reads the current translation handoff view from one release root. +pub fn read_translation_handoff_at( + resource_root: &Path, +) -> std::result::Result, String> { + let path = resource_root.join(TRANSLATION_HANDOFF_FILE); + let Some(bytes) = read_file_no_symlink(&path, "翻译 handoff")? else { + return Ok(None); + }; + let handoff: TranslationHandoff = serde_json::from_slice(&bytes) + .map_err(|error| format!("解析翻译 handoff 失败 {}：{error}", path.display()))?; + if handoff.handoff_version != TRANSLATION_HANDOFF_SCHEMA_VERSION { + return Err(format!( + "不支持的翻译 handoff 版本 {}，文件 {}", + handoff.handoff_version, + path.display() + )); + } + Ok(Some(handoff)) +} + +/// Mutable state owned by a translation worker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationTaskStatus { + /// The task is waiting for a provider worker. + Queued, + /// A provider worker is processing the task. + Running, + /// The provider worker failed the task. + Failed, + /// The provider worker completed the task. + Completed, + /// The task was intentionally excluded from provider processing. + Skipped, +} + +impl TranslationTaskStatus { + /// Returns the stable RPC and database label. + pub fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Running => "running", + Self::Failed => "failed", + Self::Completed => "completed", + Self::Skipped => "skipped", + } + } + + /// Parses a stable RPC or database label. + pub fn parse(value: &str) -> Option { + match value { + "queued" => Some(Self::Queued), + "running" => Some(Self::Running), + "failed" => Some(Self::Failed), + "completed" => Some(Self::Completed), + "skipped" => Some(Self::Skipped), + _ => None, + } + } +} + +/// One official TextUnit task with mutable provider state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersistedTranslationTask { + /// Immutable task data derived from the published release. + #[serde(flatten)] + pub task: OfficialTextUnitTask, + /// Current worker state. + pub task_status: TranslationTaskStatus, + /// Provider failure reason, if the worker recorded one. + pub failure_reason: Option, + /// Number of provider attempts. + pub attempt_count: u32, + /// First persistence time as Unix seconds. + pub created_unix_seconds: u64, + /// Last state or metadata update time as Unix seconds. + pub updated_unix_seconds: u64, + /// Completion time as Unix seconds, if completed. + #[serde(skip_serializing_if = "Option::is_none")] + pub completed_unix_seconds: Option, + /// Provider-side run identifier, if known. + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_run_id: Option, +} + +/// Result of synchronizing an immutable release queue into SQLite. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TranslationTaskSyncReport { + /// Number of newly inserted task rows. + pub inserted_count: usize, + /// Number of existing rows whose immutable task data was refreshed. + pub refreshed_count: usize, + /// Number of existing worker states preserved. + pub preserved_state_count: usize, + /// Number of stale rows removed because they are absent from the new queue. + pub removed_count: usize, +} + +/// SQLite-backed translation task state repository. +#[derive(Debug, Clone)] +pub struct SqliteTranslationTaskRepository { + pool: SqlitePool, +} + +impl SqliteTranslationTaskRepository { + /// Opens or creates the database and applies the translation-task schema. + pub async fn new(path: impl AsRef) -> Result { + Self::open_with(path.as_ref(), true).await + } + + /// Opens an existing database and applies compatible migrations. + pub async fn open(path: impl AsRef) -> Result { + Self::open_with(path.as_ref(), false).await + } + + async fn open_with(path: &Path, create_if_missing: bool) -> Result { + if create_if_missing { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + } + + let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", path.display())) + .map_err(|error| bat_core::Error::Other(error.into()))? + .create_if_missing(create_if_missing); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .map_err(|error| bat_core::Error::Other(error.into()))?; + let repository = Self { pool }; + repository.init_schema().await?; + Ok(repository) + } + + /// Returns the durable translation-task database path under one release root. + pub fn repository_path(resource_root: &Path) -> std::path::PathBuf { + resource_root.join(TRANSLATION_TASK_REPOSITORY_FILE) + } + + async fn init_schema(&self) -> Result<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS schema_migrations ( + component TEXT PRIMARY KEY NOT NULL, + version INTEGER NOT NULL CHECK(version >= 1) + ) + "#, + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS translation_tasks ( + task_id TEXT PRIMARY KEY NOT NULL, + official_release_id TEXT NOT NULL, + destination TEXT NOT NULL, + archive_entry TEXT, + queue_status TEXT NOT NULL, + queue_reason TEXT, + parse_status TEXT, + text_unit_formats_json TEXT NOT NULL DEFAULT '[]', + task_json TEXT NOT NULL DEFAULT '{}', + worker_status TEXT NOT NULL DEFAULT 'queued', + failure_reason TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK(attempt_count >= 0), + created_unix_seconds INTEGER NOT NULL, + updated_unix_seconds INTEGER NOT NULL, + completed_unix_seconds INTEGER, + provider_run_id TEXT + ) + "#, + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + + // These defaults keep old experimental databases readable while the + // schema version table records the migration boundary explicitly. + ensure_column(&self.pool, "translation_tasks", "queue_reason", "TEXT").await?; + ensure_column(&self.pool, "translation_tasks", "parse_status", "TEXT").await?; + ensure_column( + &self.pool, + "translation_tasks", + "text_unit_formats_json", + "TEXT NOT NULL DEFAULT '[]'", + ) + .await?; + ensure_column( + &self.pool, + "translation_tasks", + "task_json", + "TEXT NOT NULL DEFAULT '{}'", + ) + .await?; + ensure_column( + &self.pool, + "translation_tasks", + "worker_status", + "TEXT NOT NULL DEFAULT 'queued'", + ) + .await?; + ensure_column(&self.pool, "translation_tasks", "failure_reason", "TEXT").await?; + ensure_column( + &self.pool, + "translation_tasks", + "attempt_count", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &self.pool, + "translation_tasks", + "created_unix_seconds", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &self.pool, + "translation_tasks", + "updated_unix_seconds", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &self.pool, + "translation_tasks", + "completed_unix_seconds", + "INTEGER", + ) + .await?; + ensure_column(&self.pool, "translation_tasks", "provider_run_id", "TEXT").await?; + + let current: Option = + sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1") + .bind(TRANSLATION_TASK_SCHEMA_COMPONENT) + .fetch_optional(&self.pool) + .await + .map_err(db_error)?; + if current.is_some_and(|version| version > i64::from(TRANSLATION_TASK_SCHEMA_VERSION)) { + return Err(bat_core::Error::InvalidArgument(format!( + "不支持的翻译任务 schema 版本：{}", + current.unwrap_or_default() + ))); + } + sqlx::query( + r#" + INSERT INTO schema_migrations(component, version) + VALUES (?1, ?2) + ON CONFLICT(component) DO UPDATE SET version = excluded.version + "#, + ) + .bind(TRANSLATION_TASK_SCHEMA_COMPONENT) + .bind(i64::from(TRANSLATION_TASK_SCHEMA_VERSION)) + .execute(&self.pool) + .await + .map_err(db_error)?; + + Ok(()) + } + + /// Synchronizes one immutable release queue without resetting worker state. + pub async fn sync_queue( + &self, + queue: &OfficialTextUnitTaskQueue, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(db_error)?; + let mut report = TranslationTaskSyncReport::default(); + for task in &queue.tasks { + let task_json = serde_json::to_string(task) + .map_err(|error| bat_core::Error::Serialization(error.to_string()))?; + let existing: Option = sqlx::query_as( + r#" + SELECT task_json, worker_status, failure_reason, attempt_count, + created_unix_seconds, completed_unix_seconds, provider_run_id + FROM translation_tasks + WHERE task_id = ?1 + "#, + ) + .bind(&task.task_id) + .fetch_optional(&mut *transaction) + .await + .map_err(db_error)?; + + let now = unix_seconds_now_i64(); + let initial_status = initial_task_status(task); + let initial_failure_reason = (initial_status == TranslationTaskStatus::Skipped) + .then(|| task.reason.clone()) + .flatten(); + let formats = serde_json::to_string(&task.text_unit_formats) + .map_err(|error| bat_core::Error::Serialization(error.to_string()))?; + let parse_status = task.parse_status.map(parse_status_label); + + if let Some(( + previous_task_json, + worker_status, + failure_reason, + attempt_count, + created, + completed, + provider_run_id, + )) = existing + { + let immutable_unchanged = previous_task_json == task_json; + let (status, failure_reason, attempt_count, created, completed, provider_run_id) = + if immutable_unchanged { + ( + worker_status, + failure_reason, + attempt_count, + created, + completed, + provider_run_id, + ) + } else { + report.refreshed_count += 1; + ( + initial_status.as_str().to_string(), + initial_failure_reason.clone(), + 0_i64, + now, + None, + None, + ) + }; + if immutable_unchanged { + report.preserved_state_count += 1; + } + sqlx::query( + r#" + UPDATE translation_tasks + SET official_release_id = ?2, destination = ?3, archive_entry = ?4, + queue_status = ?5, queue_reason = ?6, parse_status = ?7, + text_unit_formats_json = ?8, task_json = ?9, + worker_status = ?10, failure_reason = ?11, attempt_count = ?12, + created_unix_seconds = ?13, updated_unix_seconds = ?14, + completed_unix_seconds = ?15, provider_run_id = ?16 + WHERE task_id = ?1 + "#, + ) + .bind(&task.task_id) + .bind(&task.official_release_id) + .bind(&task.destination) + .bind(&task.archive_entry) + .bind(task.status.as_str()) + .bind(&task.reason) + .bind(parse_status) + .bind(formats) + .bind(task_json) + .bind(status) + .bind(failure_reason) + .bind(attempt_count) + .bind(created) + .bind(now) + .bind(completed) + .bind(provider_run_id) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + } else { + report.inserted_count += 1; + sqlx::query( + r#" + INSERT INTO translation_tasks ( + task_id, official_release_id, destination, archive_entry, + queue_status, queue_reason, parse_status, text_unit_formats_json, + task_json, worker_status, failure_reason, attempt_count, + created_unix_seconds, updated_unix_seconds + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 0, ?12, ?12) + "#, + ) + .bind(&task.task_id) + .bind(&task.official_release_id) + .bind(&task.destination) + .bind(&task.archive_entry) + .bind(task.status.as_str()) + .bind(&task.reason) + .bind(parse_status) + .bind(formats) + .bind(task_json) + .bind(initial_status.as_str()) + .bind(initial_failure_reason) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + } + } + let task_ids = queue + .tasks + .iter() + .map(|task| task.task_id.as_str()) + .collect::>(); + let removed_count = if task_ids.is_empty() { + sqlx::query("DELETE FROM translation_tasks") + .execute(&mut *transaction) + .await + .map_err(db_error)? + .rows_affected() as usize + } else { + let mut query = + QueryBuilder::::new("DELETE FROM translation_tasks WHERE task_id NOT IN ("); + let mut separated = query.separated(", "); + for task_id in task_ids { + separated.push_bind(task_id); + } + separated.push_unseparated(")"); + query + .build() + .execute(&mut *transaction) + .await + .map_err(db_error)? + .rows_affected() as usize + }; + report.removed_count = removed_count; + transaction.commit().await.map_err(db_error)?; + Ok(report) + } + + /// Returns all persisted tasks matching the queue and worker filters. + pub async fn list( + &self, + query: &OfficialTextUnitTaskQuery, + ) -> Result> { + let rows: Vec = sqlx::query_as( + r#" + SELECT task_json, worker_status, failure_reason, attempt_count, + created_unix_seconds, updated_unix_seconds, + completed_unix_seconds, provider_run_id + FROM translation_tasks + ORDER BY task_id + "#, + ) + .fetch_all(&self.pool) + .await + .map_err(db_error)?; + rows.into_iter() + .map(PersistedTranslationTask::from_row) + .collect::>>() + .map(|tasks| { + tasks + .into_iter() + .filter(|task| matches_query(task, query)) + .collect() + }) + } + + /// Returns the number of persisted tasks matching a query. + pub async fn count(&self, query: &OfficialTextUnitTaskQuery) -> Result { + Ok(self.list(query).await?.len() as u64) + } + + /// Updates provider state and returns the durable task record. + pub async fn update_status( + &self, + task_id: &str, + status: TranslationTaskStatus, + failure_reason: Option, + provider_run_id: Option, + ) -> Result { + let current = self.find(task_id).await?; + let now = unix_seconds_now_i64(); + let attempt_count = if status == TranslationTaskStatus::Running + && current.task_status != TranslationTaskStatus::Running + { + current.attempt_count.saturating_add(1) + } else { + current.attempt_count + }; + let normalized_reason = failure_reason.filter(|reason| !reason.trim().is_empty()); + let completed = (status == TranslationTaskStatus::Completed).then_some(now); + sqlx::query( + r#" + UPDATE translation_tasks + SET worker_status = ?2, failure_reason = ?3, attempt_count = ?4, + updated_unix_seconds = ?5, completed_unix_seconds = ?6, + provider_run_id = COALESCE(?7, provider_run_id) + WHERE task_id = ?1 + "#, + ) + .bind(task_id) + .bind(status.as_str()) + .bind(normalized_reason) + .bind(i64::from(attempt_count)) + .bind(now) + .bind(completed) + .bind(provider_run_id) + .execute(&self.pool) + .await + .map_err(db_error)?; + self.find(task_id).await + } + + /// Finds one task by its stable ID. + pub async fn find(&self, task_id: &str) -> Result { + let row: Option = sqlx::query_as( + r#" + SELECT task_json, worker_status, failure_reason, attempt_count, + created_unix_seconds, updated_unix_seconds, + completed_unix_seconds, provider_run_id + FROM translation_tasks + WHERE task_id = ?1 + "#, + ) + .bind(task_id) + .fetch_optional(&self.pool) + .await + .map_err(db_error)?; + row.map(PersistedTranslationTask::from_row) + .transpose()? + .ok_or_else(|| bat_core::Error::NotFound(task_id.to_string())) + } +} + +type ExistingTaskRow = ( + String, + String, + Option, + i64, + i64, + Option, + Option, +); + +type TranslationTaskRow = ( + String, + String, + Option, + i64, + i64, + i64, + Option, + Option, +); + +impl PersistedTranslationTask { + /// Builds a persisted translation task from an immutable queue task. + pub fn from_task( + task: OfficialTextUnitTask, + task_status: TranslationTaskStatus, + failure_reason: Option, + attempt_count: u32, + created_unix_seconds: u64, + updated_unix_seconds: u64, + completed_unix_seconds: Option, + provider_run_id: Option, + ) -> Self { + Self { + task, + task_status, + failure_reason, + attempt_count, + created_unix_seconds, + updated_unix_seconds, + completed_unix_seconds, + provider_run_id, + } + } + + /// Builds a synthetic persisted task for older release roots without SQLite state. + pub fn from_queued_task(task: OfficialTextUnitTask, generated_unix_seconds: u64) -> Self { + let task_status = initial_task_status(&task); + let failure_reason = if task_status == TranslationTaskStatus::Skipped { + task.reason.clone() + } else { + None + }; + Self::from_task( + task, + task_status, + failure_reason, + 0, + generated_unix_seconds, + generated_unix_seconds, + None, + None, + ) + } + + fn from_row(row: TranslationTaskRow) -> Result { + let ( + task_json, + worker_status, + failure_reason, + attempt_count, + created_unix_seconds, + updated_unix_seconds, + completed_unix_seconds, + provider_run_id, + ) = row; + let task = serde_json::from_str(&task_json) + .map_err(|error| bat_core::Error::Serialization(error.to_string()))?; + let task_status = TranslationTaskStatus::parse(&worker_status).ok_or_else(|| { + bat_core::Error::Serialization(format!("未知翻译任务 worker 状态：{worker_status}")) + })?; + Ok(Self { + task, + task_status, + failure_reason, + attempt_count: u32::try_from(attempt_count).map_err(|_| { + bat_core::Error::Serialization("翻译任务 attempt_count 超出范围".to_string()) + })?, + created_unix_seconds: u64::try_from(created_unix_seconds).map_err(|_| { + bat_core::Error::Serialization("翻译任务 created 时间无效".to_string()) + })?, + updated_unix_seconds: u64::try_from(updated_unix_seconds).map_err(|_| { + bat_core::Error::Serialization("翻译任务 updated 时间无效".to_string()) + })?, + completed_unix_seconds: completed_unix_seconds + .map(|value| { + u64::try_from(value).map_err(|_| { + bat_core::Error::Serialization("翻译任务 completed 时间无效".to_string()) + }) + }) + .transpose()?, + provider_run_id, + }) + } +} + +/// Synchronizes a release queue into the durable SQLite repository for that release. +pub async fn sync_translation_task_repository_at( + resource_root: &Path, + queue: &OfficialTextUnitTaskQueue, +) -> Result { + let path = SqliteTranslationTaskRepository::repository_path(resource_root); + let repository = SqliteTranslationTaskRepository::new(&path).await?; + repository.sync_queue(queue).await +} + +fn matches_query(task: &PersistedTranslationTask, query: &OfficialTextUnitTaskQuery) -> bool { + let mut queue_query = query.clone(); + queue_query.task_status = None; + if !textunit_task_matches(&task.task, &queue_query) { + return false; + } + if query + .task_status + .as_ref() + .is_some_and(|status| task.task_status.as_str() != status) + { + return false; + } + if let Some(has_failure_reason) = query.has_failure_reason { + if task.failure_reason.is_some() != has_failure_reason { + return false; + } + } + true +} + +fn initial_task_status(task: &OfficialTextUnitTask) -> TranslationTaskStatus { + match task.status { + crate::official_textunit_queue::OfficialTextUnitTaskStatus::QueuedOffline => { + TranslationTaskStatus::Queued + } + crate::official_textunit_queue::OfficialTextUnitTaskStatus::SkippedNoParseEntry + | crate::official_textunit_queue::OfficialTextUnitTaskStatus::SkippedNoTextUnit + | crate::official_textunit_queue::OfficialTextUnitTaskStatus::SkippedParseFailed + | crate::official_textunit_queue::OfficialTextUnitTaskStatus::SkippedUnsupported => { + TranslationTaskStatus::Skipped + } + } +} + +fn parse_status_label(status: crate::official_parse::OfficialParseStatus) -> &'static str { + match status { + crate::official_parse::OfficialParseStatus::Parsed => "parsed", + crate::official_parse::OfficialParseStatus::SkippedUnsupported => "skipped_unsupported", + crate::official_parse::OfficialParseStatus::Failed => "failed", + } +} + +async fn ensure_column( + pool: &SqlitePool, + table: &str, + column: &str, + column_type: &str, +) -> Result<()> { + let exists: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM pragma_table_info(?1) WHERE name = ?2") + .bind(table) + .bind(column) + .fetch_one(pool) + .await + .map_err(db_error)?; + if exists == 0 { + let mut query = QueryBuilder::::new("ALTER TABLE "); + query + .push(table) + .push(" ADD COLUMN ") + .push(column) + .push(" "); + query.push(column_type); + query.build().execute(pool).await.map_err(db_error)?; + } + Ok(()) +} + +fn db_error(error: sqlx::Error) -> bat_core::Error { + bat_core::Error::Other(error.into()) +} + +fn unix_seconds_now_i64() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .try_into() + .unwrap_or(i64::MAX) +} + +fn unix_seconds_now() -> u64 { + unix_seconds_now_i64().max(0) as u64 +} + +fn provider_run_status_rank(status: ProviderRunStatus) -> u8 { + match status { + ProviderRunStatus::Queued => 0, + ProviderRunStatus::Succeeded => 1, + ProviderRunStatus::Running => 2, + ProviderRunStatus::Cancelled => 3, + ProviderRunStatus::Failed => 4, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::official_changes::OfficialResourceChangeKind; + use crate::official_parse::{OfficialParseSourceKind, OfficialParseStatus}; + use crate::official_textunit_queue::{ + OfficialTextUnitTaskStatus, OfficialTextUnitTaskSummary, + OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION, + }; + + fn task( + task_id: &str, + destination: &str, + status: OfficialTextUnitTaskStatus, + parse_status: Option, + reason: Option<&str>, + ) -> OfficialTextUnitTask { + OfficialTextUnitTask { + task_id: task_id.to_string(), + official_release_id: "release-current".to_string(), + destination: destination.to_string(), + change_kind: OfficialResourceChangeKind::Added, + url: format!("https://example.invalid/{destination}"), + bytes: 10, + blake3: format!("{task_id}-hash"), + parse_entry_key: Some(format!("direct:{destination}")), + archive_entry: None, + source_kind: Some(OfficialParseSourceKind::DirectBundle), + parse_status, + text_asset_count: usize::from(status == OfficialTextUnitTaskStatus::QueuedOffline), + text_assets: if status == OfficialTextUnitTaskStatus::QueuedOffline { + vec!["Scenario".to_string()] + } else { + Vec::new() + }, + text_unit_count: if status == OfficialTextUnitTaskStatus::QueuedOffline { + 3 + } else { + 0 + }, + text_unit_formats: if status == OfficialTextUnitTaskStatus::QueuedOffline { + vec!["plain".to_string()] + } else { + Vec::new() + }, + text_unit_error_count: 0, + status, + reason: reason.map(str::to_string), + } + } + + fn queue(tasks: Vec) -> OfficialTextUnitTaskQueue { + OfficialTextUnitTaskQueue { + queue_version: OFFICIAL_TEXTUNIT_TASK_QUEUE_VERSION, + official_release_id: "release-current".to_string(), + previous_release_id: Some("release-previous".to_string()), + generated_unix_seconds: 123, + current_resource_root: std::path::PathBuf::from("/tmp/release-current"), + summary: OfficialTextUnitTaskSummary { + resource_candidate_count: tasks.len(), + parse_entry_count: tasks.len(), + queued_task_count: tasks + .iter() + .filter(|task| task.status == OfficialTextUnitTaskStatus::QueuedOffline) + .count(), + skipped_parse_failed_count: tasks + .iter() + .filter(|task| task.status == OfficialTextUnitTaskStatus::SkippedParseFailed) + .count(), + text_unit_count: tasks.iter().map(|task| task.text_unit_count).sum(), + ..OfficialTextUnitTaskSummary::default() + }, + tasks, + } + } + + #[tokio::test] + async fn sqlite_translation_tasks_sync_and_preserve_worker_state() { + let temp = tempfile::TempDir::new().unwrap(); + let repository = + SqliteTranslationTaskRepository::new(temp.path().join("translation-tasks.sqlite")) + .await + .unwrap(); + let queue = queue(vec![ + task( + "task-a", + "Bundles/a.bundle", + OfficialTextUnitTaskStatus::QueuedOffline, + Some(OfficialParseStatus::Parsed), + None, + ), + task( + "task-b", + "Bundles/b.bundle", + OfficialTextUnitTaskStatus::SkippedParseFailed, + Some(OfficialParseStatus::Failed), + Some("parser failed"), + ), + ]); + + let first = repository.sync_queue(&queue).await.unwrap(); + assert_eq!(first.inserted_count, 2); + assert_eq!(first.preserved_state_count, 0); + let failed = repository + .list(&OfficialTextUnitTaskQuery { + task_status: Some("skipped".to_string()), + has_reason: Some(true), + ..OfficialTextUnitTaskQuery::default() + }) + .await + .unwrap(); + assert_eq!(failed.len(), 1); + assert_eq!(failed[0].task.task_id, "task-b"); + assert_eq!(failed[0].failure_reason.as_deref(), Some("parser failed")); + + let running = repository + .update_status( + "task-a", + TranslationTaskStatus::Running, + None, + Some("run-1".to_string()), + ) + .await + .unwrap(); + assert_eq!(running.task_status, TranslationTaskStatus::Running); + assert_eq!(running.attempt_count, 1); + let failed = repository + .update_status( + "task-a", + TranslationTaskStatus::Failed, + Some("remote provider rejected payload".to_string()), + None, + ) + .await + .unwrap(); + assert_eq!(failed.task_status, TranslationTaskStatus::Failed); + assert_eq!( + failed.failure_reason.as_deref(), + Some("remote provider rejected payload") + ); + assert_eq!(failed.provider_run_id.as_deref(), Some("run-1")); + + let second = repository.sync_queue(&queue).await.unwrap(); + assert_eq!(second.inserted_count, 0); + assert_eq!(second.preserved_state_count, 2); + let retrievable = repository + .list(&OfficialTextUnitTaskQuery { + task_status: Some("failed".to_string()), + has_failure_reason: Some(true), + ..OfficialTextUnitTaskQuery::default() + }) + .await + .unwrap(); + assert_eq!(retrievable.len(), 1); + assert_eq!(retrievable[0].task.task_id, "task-a"); + assert_eq!(retrievable[0].attempt_count, 1); + } + + #[test] + fn translation_handoff_maps_worker_state_and_preserves_provider_progress() { + let queue = queue(vec![ + task( + "task-a", + "Bundles/a.bundle", + OfficialTextUnitTaskStatus::QueuedOffline, + Some(OfficialParseStatus::Parsed), + None, + ), + task( + "task-b", + "Bundles/b.bundle", + OfficialTextUnitTaskStatus::QueuedOffline, + Some(OfficialParseStatus::Parsed), + None, + ), + task( + "task-c", + "Bundles/c.bundle", + OfficialTextUnitTaskStatus::SkippedParseFailed, + Some(OfficialParseStatus::Failed), + Some("parser failed"), + ), + ]); + let persisted = vec![ + PersistedTranslationTask { + task: queue.tasks[0].clone(), + task_status: TranslationTaskStatus::Running, + failure_reason: None, + attempt_count: 2, + created_unix_seconds: 123, + updated_unix_seconds: 124, + completed_unix_seconds: None, + provider_run_id: Some("provider-run-1".to_string()), + }, + PersistedTranslationTask { + task: queue.tasks[1].clone(), + task_status: TranslationTaskStatus::Completed, + failure_reason: None, + attempt_count: 1, + created_unix_seconds: 123, + updated_unix_seconds: 125, + completed_unix_seconds: Some(125), + provider_run_id: Some("provider-run-1".to_string()), + }, + ]; + + let handoff = build_translation_handoff(&queue, &persisted); + + assert_eq!(handoff.handoff_version, TRANSLATION_HANDOFF_SCHEMA_VERSION); + assert_eq!(handoff.source_queue_file, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE); + assert_eq!(handoff.job.status, TranslationJobStatus::Failed); + assert_eq!(handoff.job.unit_count, 3); + assert_eq!(handoff.units[0].status, TranslationUnitStatus::Translating); + assert_eq!(handoff.units[1].status, TranslationUnitStatus::Translated); + assert_eq!(handoff.units[2].status, TranslationUnitStatus::Skipped); + assert_eq!( + handoff.units[2].failure_reason.as_deref(), + Some("parser failed") + ); + assert_eq!(handoff.provider_runs.len(), 1); + assert_eq!(handoff.provider_runs[0].status, ProviderRunStatus::Running); + assert_eq!(handoff.provider_runs[0].unit_ids, ["task-a", "task-b"]); + assert_eq!(handoff.provider_runs[0].attempt_count, 2); + } + + #[test] + fn translation_handoff_file_round_trips_with_version_check() { + let temp = tempfile::TempDir::new().unwrap(); + let handoff = build_translation_handoff(&queue(Vec::new()), &[]); + + write_translation_handoff_at(temp.path(), &handoff).unwrap(); + let loaded = read_translation_handoff_at(temp.path()).unwrap().unwrap(); + + assert_eq!(loaded, handoff); + assert!(temp.path().join(TRANSLATION_HANDOFF_FILE).is_file()); + } +}