From 264e78c440a4273d32b26a191069f2df3a6c188c Mon Sep 17 00:00:00 2001 From: Yuyi-Oak <1722157266@qq.com> Date: Sat, 11 Jul 2026 01:06:02 +0800 Subject: [PATCH] feat: harden official sync scheduling and integrity --- CURRENT_STATUS.md | 9 +- DOCS_INDEX.md | 2 +- README.md | 2 +- docs/architecture/README.md | 2 +- .../architecture/official-resource-backend.md | 21 +- docs/guides/deployment.md | 2 +- docs/guides/official-resource-test-pull.md | 4 +- infrastructure/src/bin/bat_official_sync.rs | 150 +++++++++++- infrastructure/src/official_download.rs | 222 +++++++++++++++--- infrastructure/src/official_update.rs | 96 ++++---- .../official_game_main_config_bootstrap.rs | 63 +++++ 11 files changed, 474 insertions(+), 99 deletions(-) diff --git a/CURRENT_STATUS.md b/CURRENT_STATUS.md index 3550564..64b707e 100644 --- a/CURRENT_STATUS.md +++ b/CURRENT_STATUS.md @@ -1,9 +1,9 @@ # BlueArchiveToolkit 当前工作区状态 -- **更新时间**：2026-07-06 +- **更新时间**：2026-07-11 - **状态来源**：本地工作区盘点、代码验证和最新提交 - **状态分支**：`experiment` -- **最新功能提交**：`789402c feat: add official resource sync pipeline` +- **最新已推送功能提交**：`2bdc55a feat: support cancellable official sync` - **权威计划**：`PROJECT_PLAN.md` --- @@ -21,7 +21,8 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 5. 下载后使用本地 manifest 的 size + BLAKE3 校验复用文件；官方 seed `.hash` 使用 `xxHash32(seed=0)` 强校验。 6. 支持 `.part` 断点续传、失败后 clean retry、本地 manifest audit/repair。 7. `bat-official-sync --watch` 可常驻运行，正常检查默认每 1 小时一次；远端和本地一致时默认静默，失败后默认 60 秒快速重试；CLI 默认向 stderr 输出 ASCII banner 和人类可读 progress log。 -8. 非 dry-run 使用 `--output/.official-sync.lock` 防止并发写同一状态目录。 +8. 远端 snapshot 未变化但输出目录为空时，会按首次运行执行全量拉取；官方 seed `.hash` 校验失败时会清理对应 manifest 条目，避免失败产物被后续本地 audit 误判为可复用。 +9. 非 dry-run 使用 `--output/.official-sync.lock` 防止并发写同一状态目录。 仍需明确：这不是完整产品完成。Go CLI 最小入口、完整 AssetBundle 解析、Patch、翻译系统、API Server 和 Web 仍是后续工作；真实官方网络全量下载 smoke test 尚未记录在仓库文档中。 @@ -241,7 +242,7 @@ cargo run -p bat-infrastructure --bin bat-official-sync -- \ 2. 不要指向已有游戏客户端目录。 3. 不要指向 `/home/wanye/D/BlueArchive` 这类开发或人工维护资源目录。 4. `--auto-discover` 可以下载官方 metadata，并按官方 manifest 临时获取 `resources.assets` 以解析 `GameMainConfig`；旧 ZIP manifest 才会下载临时 game zip。该流程不会安装或启动官方 launcher。 -5. systemd service、容器或 Go 进程可以负责守护 `bat-official-sync --watch`，但定时检查逻辑已经在 Rust 内部；正常检查默认 1 小时，失败重试默认 60 秒。 +5. systemd service、容器或 Go 进程可以负责守护 `bat-official-sync --watch`，但定时检查逻辑已经在 Rust 内部；正常检查默认 1 小时，失败重试默认 60 秒，每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会强制执行一次自动刷新。 详细运行说明见 `docs/guides/official-resource-test-pull.md`。 diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index 1a12db7..bfba2ae 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -90,7 +90,7 @@ - Unity/Manifest/Client 适配器框架。 - CAS V1：原子写入、BLAKE3 校验、引用计数、GC、并发测试和损坏检测。 - 文档整理和路线图重制。 -- Rust 官方资源同步闭环：`bat-official-sync`、`--auto-discover`、`--watch`、snapshot、manifest audit/repair、官方 seed `.hash` 校验。 +- Rust 官方资源同步闭环：`bat-official-sync`、`--auto-discover`、`--watch`、北京时间固定强制刷新、snapshot、manifest audit/repair、官方 seed `.hash` 校验。 - Addressables 当前真实形态 fixture/golden 覆盖。 - SQLite Resource Repository 和粗粒度 FFI JSON 接口。 diff --git a/README.md b/README.md index e693aa7..b2a797d 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ cargo run -p bat-infrastructure --bin bat-official-sync -- \ --dry-run ``` -常驻自动更新，正常情况下默认每 1 小时检查一次；远端和本地一致时静默。下载或发现失败时默认 60 秒后重试，可用 `--error-retry 60s` 调整： +常驻自动更新，正常情况下默认每 1 小时检查一次；每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会强制执行一次自动刷新。远端和本地一致时静默。下载或发现失败时默认 60 秒后重试，可用 `--error-retry 60s` 调整： ```bash cargo run -p bat-infrastructure --bin bat-official-sync -- \ diff --git a/docs/architecture/README.md b/docs/architecture/README.md index fff1979..7448d43 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -116,7 +116,7 @@ official-sync-snapshot.json + official-download-manifest.json - 不安装、不执行官方 launcher。 - 默认平台 `Windows + Android`。 - `--auto-discover` 自动获取 `app-version`、`connection-group` 和 `server-info`。 -- `--watch` 常驻检查，正常检查默认 1 小时，失败重试默认 60 秒。 +- `--watch` 常驻检查，正常检查默认 1 小时，失败重试默认 60 秒，每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 强制刷新一次。 - 远端 marker 无变化且本地 manifest clean 时不下载。 - 本地文件损坏时 repair。 - 官方 seed `.hash` 强校验；Addressables `catalog_*.hash` 作为变更 marker。 diff --git a/docs/architecture/official-resource-backend.md b/docs/architecture/official-resource-backend.md index 55bfbdb..5d14d7c 100644 --- a/docs/architecture/official-resource-backend.md +++ b/docs/architecture/official-resource-backend.md @@ -126,11 +126,12 @@ 5. 缺少清单、清单不匹配或文件损坏时重新下载。 6. `TableCatalog.bytes`、`BundlePackingInfo.bytes`、`MediaCatalog.bytes` 总是刷新并用官方 `.hash` 强校验；该 `.hash` 是 `xxHash32(seed=0)` 的十进制文本。 7. `catalog_*.hash` 当前只作为 Addressables catalog 变更标记，不作为 zip/JSON 内容校验算法；Unity Addressables/SBP builder 对 JSON/bin catalog 使用 `HashingMethods.Calculate` 生成 `Hash128` 文本，运行时用它判断 remote catalog cache 是否过期，它不能套用 seed catalog 的 `xxHash32` 规则。 -8. 存在 `.part` 临时文件时通过 `curl --continue-at -` 尝试断点续传。 -9. 新下载写入 `.part`，成功后原子 rename 到最终路径。 -10. 成功下载后更新本地下载清单。 -11. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 -12. 非官方 URL 直接拒绝。 +8. 官方 seed `.hash` 校验失败会让当前下载失败，并移除对应 data/hash URL 的本地 manifest 条目，避免失败产物在下一轮被本地 BLAKE3 audit 误判为健康缓存。 +9. 存在 `.part` 临时文件时通过 `curl --continue-at -` 尝试断点续传。 +10. 新下载写入 `.part`，成功后原子 rename 到最终路径。 +11. 成功下载后更新本地下载清单。 +12. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 +13. 非官方 URL 直接拒绝。 路径映射时会做分段清理，避免把不安全路径写进输出目录。 @@ -186,11 +187,13 @@ 3. 生成当前 v2 snapshot，记录 `app_version`、`connection_group`、`bundle_version`、`addressables_root`、endpoint URL、seed `.hash` 内容、`catalog_*.hash` marker、launcher metadata 摘要和 `GameMainConfig` 摘要。 4. 读取上一次成功同步写出的 snapshot。 5. 使用 `OfficialSyncPlan` 和扩展 snapshot diff 判断是否需要下载；URL 未变但 `.hash` / marker 内容变化也会触发更新。 -6. 远端无变化时执行本地 download manifest audit，检查路径、size 和 BLAKE3。 -7. 远端变化或本地 audit 发现 repair_needed 时生成 pull plan，下载并校验官方 URL。 -8. 下载成功后写回新的 snapshot。 +6. 每轮都会基于最新 seed catalog 构建当前 pull plan，并检查输出目录是否已有当前 plan 的 manifest 条目或目标文件。 +7. 如果远端 snapshot 未变化但输出目录没有任何当前 plan 的本地资源，仍按首次运行处理并执行全量拉取。 +8. 远端无变化且本地已有资源时执行 download manifest audit，检查路径、size 和 BLAKE3。 +9. 远端变化、本地 audit 发现 repair_needed，或首次空目录运行时，下载并校验官方 URL。 +10. 下载成功后写回新的 snapshot。 -该入口不安装、不执行官方启动器，也不读取生产外的本地客户端目录。Rust 正式 binary 支持 `--watch` 常驻模式，正常情况下默认每 1 小时执行一次检查；远端和本地一致时静默等待下次检查，不一致时自动下载或 repair。下载、发现或校验失败时不等待完整正常周期，默认 60 秒后重试，可用 `--error-retry` 或 `--error-retry-seconds` 调整。单次运行仍保留为核心幂等路径，systemd service、容器或 Go 进程可以只负责守护该常驻进程；cron/systemd timer 调单次模式只是可选集成方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。 +该入口不安装、不执行官方启动器，也不读取生产外的本地客户端目录。Rust 正式 binary 支持 `--watch` 常驻模式，正常情况下默认每 1 小时执行一次检查；每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会中断普通 sleep 并强制执行一次自动刷新，该轮注入 `force=true`。远端和本地一致时静默等待下次检查，不一致时自动下载或 repair。下载、发现或校验失败时不等待完整正常周期，默认 60 秒后重试；如果固定时间强制刷新失败，会保留 pending force 并按失败重试周期继续重试，可用 `--error-retry` 或 `--error-retry-seconds` 调整。单次运行仍保留为核心幂等路径，systemd service、容器或 Go 进程可以只负责守护该常驻进程；cron/systemd timer 调单次模式只是可选集成方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。 对应实现主要在： diff --git a/docs/guides/deployment.md b/docs/guides/deployment.md index f010ea3..8e8f8f8 100644 --- a/docs/guides/deployment.md +++ b/docs/guides/deployment.md @@ -157,7 +157,7 @@ target/release/bat-official-sync --watch ``` -`--watch` 是 Rust 内部持久检查模式，正常情况下默认每 1 小时执行一次检查。远端和本地一致时默认静默；有远端变化或本地文件损坏时自动下载或 repair，并输出 JSON report。下载、发现或校验失败时默认 60 秒后重试，可显式加 `--error-retry 60s` 或 `--error-retry-seconds 60` 调整。默认 ASCII banner 和进度日志写到 stderr，成功 JSON report 写到 stdout；如果由上层服务严格解析 stderr/stdout，可加 `--no-progress`，只想关闭横幅可加 `--no-banner`。 +`--watch` 是 Rust 内部持久检查模式，正常情况下默认每 1 小时执行一次检查，并且每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会强制执行一次自动刷新。固定时间刷新会中断普通 interval 的 sleep，该轮注入 `force=true`；如果失败，会按失败重试周期继续重试。远端和本地一致时默认静默；有远端变化或本地文件损坏时自动下载或 repair，并输出 JSON report。下载、发现或校验失败时默认 60 秒后重试，可显式加 `--error-retry 60s` 或 `--error-retry-seconds 60` 调整。默认 ASCII banner 和进度日志写到 stderr，成功 JSON report 写到 stdout；如果由上层服务严格解析 stderr/stdout，可加 `--no-progress`，只想关闭横幅可加 `--no-banner`。 ### systemd service 示例 diff --git a/docs/guides/official-resource-test-pull.md b/docs/guides/official-resource-test-pull.md index dee2377..d2d1284 100644 --- a/docs/guides/official-resource-test-pull.md +++ b/docs/guides/official-resource-test-pull.md @@ -157,8 +157,10 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - 每轮都会先拉取轻量官方 metadata 和 `.hash` marker。 - URL 没变但 `.hash` / marker 内容变更时，也会判定为需要更新。 +- 即使远端 snapshot 未变化，只要输出目录没有任何当前 pull plan 的本地 manifest 条目或目标文件，也会按首次运行处理并执行全量拉取。 - 远端无变化时，默认执行本地 `official-download-manifest.json` audit。 - 本地文件缺失、路径不一致、size 不一致或 BLAKE3 不一致时，默认进入 repair 并重下必要文件。 +- 官方 seed `.hash` 校验失败会让本轮失败，并清理对应本地 manifest 条目；下一轮会继续把这类文件视为需要 repair，而不是把失败产物当作健康缓存复用。 - 远端和本地都一致：单次模式输出 `update_status=up_to_date`，watch 模式默认静默并等待下次检查。 - 有远端变化或本地 repair：生成 pull plan，下载完整官方资源，成功后更新 snapshot。 - `--dry-run`：只报告本次是否会下载，不写 snapshot；如果 cache miss，也不会写入新的 bootstrap cache。 @@ -229,7 +231,7 @@ cargo run -p bat-infrastructure --bin bat-official-sync -- \ --error-retry 60s ``` -`--interval` 是正常检查周期，默认 `1h`；`--error-retry` 是下载、发现或校验失败后的重试周期，默认 `60s`，也可以用 `--error-retry-seconds 60`。CLI 默认启动时向 stderr 打印 `BlueArchiveToolkit` ASCII banner，并把人类可读的阶段进度日志写到 stderr，包括自动发现、server-info、marker、catalog、audit、download 和 snapshot 阶段；成功时 stdout 输出稳定 JSON report。需要纯机器输出时加 `--no-progress`，需要显式开启则用 `--progress`；只想关闭横幅但保留日志时可加 `--no-banner`。错误时 stderr 输出 JSON error，watch 模式下错误 JSON 的 `next_retry_seconds` 使用失败重试周期；如果未关闭 progress，错误 JSON 前可能已有 banner 和进度日志。普通错误 exit `1`，状态目录锁冲突 exit `75`。 +`--interval` 是正常检查周期，默认 `1h`；watch 模式还会在每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 强制执行一次自动刷新，该轮会注入 `force=true`，并且会中断普通 interval 的 sleep。`--error-retry` 是下载、发现或校验失败后的重试周期，默认 `60s`，也可以用 `--error-retry-seconds 60`。CLI 默认启动时向 stderr 打印 `BlueArchiveToolkit` ASCII banner，并把人类可读的阶段进度日志写到 stderr，包括自动发现、server-info、marker、catalog、audit、download 和 snapshot 阶段；成功时 stdout 输出稳定 JSON report。需要纯机器输出时加 `--no-progress`，需要显式开启则用 `--progress`；只想关闭横幅但保留日志时可加 `--no-banner`。错误时 stderr 输出 JSON error，watch 模式下错误 JSON 的 `next_retry_seconds` 使用失败重试周期；如果未关闭 progress，错误 JSON 前可能已有 banner 和进度日志。普通错误 exit `1`，状态目录锁冲突 exit `75`。 生产可以直接运行 `--watch`，也可以用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一状态目录。 diff --git a/infrastructure/src/bin/bat_official_sync.rs b/infrastructure/src/bin/bat_official_sync.rs index 1fef037..4bb752b 100644 --- a/infrastructure/src/bin/bat_official_sync.rs +++ b/infrastructure/src/bin/bat_official_sync.rs @@ -7,12 +7,16 @@ use serde::Serialize; use std::env; use std::path::PathBuf; use std::thread; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const EXIT_ERROR: i32 = 1; const EXIT_LOCKED: i32 = 75; const DEFAULT_WATCH_INTERVAL_SECONDS: u64 = 60 * 60; const DEFAULT_ERROR_RETRY_SECONDS: u64 = 60; +const SECONDS_PER_DAY: u64 = 24 * 60 * 60; +const BEIJING_UTC_OFFSET_SECONDS: u64 = 8 * 60 * 60; +const DAILY_FORCED_REFRESH_LOCAL_SECONDS: [u64; 3] = [3 * 60 * 60, 16 * 60 * 60, 18 * 60 * 60]; +const DAILY_FORCED_REFRESH_LABEL: &str = "UTC+8 03:00, 16:00, 18:00"; const STARTUP_BANNER: &str = r#" ============================================================ ____ _ _ _ _ _____ _ _ _ _ @@ -120,25 +124,64 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> { } let service = OfficialUpdateService::new(); let mut logger = ProgressLogger::new(options.progress); + let mut next_forced_refresh_at = next_forced_refresh_at_or_after(SystemTime::now()); + let mut pending_scheduled_force = false; + logger.log_text( + "watch", + format!( + "daily forced refresh schedule: {DAILY_FORCED_REFRESH_LABEL}; next forced refresh in {}", + format_duration(duration_until(next_forced_refresh_at, SystemTime::now())) + ), + ); loop { + let now = SystemTime::now(); + if now >= next_forced_refresh_at { + pending_scheduled_force = true; + logger.log_text( + "watch", + format!("scheduled forced refresh triggered at {DAILY_FORCED_REFRESH_LABEL}"), + ); + next_forced_refresh_at = next_forced_refresh_after(now); + } + + let mut iteration_config = options.config.clone(); + if pending_scheduled_force { + iteration_config.force = true; + } + let mut sleep_for = options.interval; - logger.log_text("watch", "starting watch iteration"); - match service.run_with_progress(&options.config, |event| logger.log(event)) { + logger.log_text( + "watch", + if pending_scheduled_force { + "starting watch iteration with scheduled force refresh" + } else { + "starting watch iteration" + }, + ); + match service.run_with_progress(&iteration_config, |event| logger.log(event)) { Ok(report) => { + if pending_scheduled_force { + pending_scheduled_force = false; + } if should_print_status(report.update_status, options.quiet_up_to_date) { println!("{}", serde_json::to_string_pretty(&report)?); } + sleep_for = + sleep_for.min(duration_until(next_forced_refresh_at, SystemTime::now())); logger.log_text( "watch", format!( - "iteration finished with status={}; next check in {}", + "iteration finished with status={}; next check in {}; next scheduled force refresh in {}", report.update_status.as_str(), - format_duration(sleep_for) + format_duration(sleep_for), + format_duration(duration_until(next_forced_refresh_at, SystemTime::now())) ), ); } Err(error) => { sleep_for = options.error_retry_interval; + sleep_for = + sleep_for.min(duration_until(next_forced_refresh_at, SystemTime::now())); logger.log_text( "watch", format!( @@ -161,6 +204,62 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> { } } +fn next_forced_refresh_at_or_after(now: SystemTime) -> SystemTime { + forced_refresh_time(now, ForcedRefreshBoundary::AtOrAfter) +} + +fn next_forced_refresh_after(now: SystemTime) -> SystemTime { + forced_refresh_time(now, ForcedRefreshBoundary::After) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ForcedRefreshBoundary { + AtOrAfter, + After, +} + +fn forced_refresh_time(now: SystemTime, boundary: ForcedRefreshBoundary) -> SystemTime { + let local_seconds = beijing_local_seconds_since_epoch(now); + let local_day = local_seconds / SECONDS_PER_DAY; + let local_second_of_day = local_seconds % SECONDS_PER_DAY; + + for scheduled_second in DAILY_FORCED_REFRESH_LOCAL_SECONDS { + let matches_boundary = match boundary { + ForcedRefreshBoundary::AtOrAfter => scheduled_second >= local_second_of_day, + ForcedRefreshBoundary::After => scheduled_second > local_second_of_day, + }; + if matches_boundary { + return system_time_from_beijing_local_seconds( + local_day + .saturating_mul(SECONDS_PER_DAY) + .saturating_add(scheduled_second), + ); + } + } + + system_time_from_beijing_local_seconds( + local_day + .saturating_add(1) + .saturating_mul(SECONDS_PER_DAY) + .saturating_add(DAILY_FORCED_REFRESH_LOCAL_SECONDS[0]), + ) +} + +fn beijing_local_seconds_since_epoch(time: SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .saturating_add(BEIJING_UTC_OFFSET_SECONDS) +} + +fn system_time_from_beijing_local_seconds(local_seconds: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(local_seconds.saturating_sub(BEIJING_UTC_OFFSET_SECONDS)) +} + +fn duration_until(deadline: SystemTime, now: SystemTime) -> Duration { + deadline.duration_since(now).unwrap_or_default() +} + #[derive(Debug, Clone)] struct ProgressLogger { enabled: bool, @@ -365,6 +464,9 @@ fn print_usage(binary: &str) { [--curl curl] [--unzip unzip] [--dry-run] [--plan] [--force] [--audit-local|--no-audit-local] [--repair|--no-repair] \ [--watch] [--interval 1h|60m|3600s] [--error-retry 60s] [--quiet-up-to-date|--no-quiet-up-to-date] [--progress|--no-progress] [--banner|--no-banner]" ); + eprintln!( + "watch mode also runs scheduled force refreshes every day at {DAILY_FORCED_REFRESH_LABEL}" + ); } fn parse_duration(value: &str) -> anyhow::Result { @@ -616,6 +718,35 @@ mod tests { assert_eq!(parse_duration("3600").unwrap(), Duration::from_secs(3600)); } + #[test] + fn scheduled_forced_refresh_uses_beijing_local_times() { + let day = 20_000; + + assert_eq!( + next_forced_refresh_at_or_after(beijing_time(day, 2, 30, 0)), + beijing_time(day, 3, 0, 0) + ); + assert_eq!( + next_forced_refresh_at_or_after(beijing_time(day, 3, 0, 0)), + beijing_time(day, 3, 0, 0) + ); + assert_eq!( + next_forced_refresh_after(beijing_time(day, 3, 0, 0)), + beijing_time(day, 16, 0, 0) + ); + assert_eq!( + next_forced_refresh_after(beijing_time(day, 18, 0, 0)), + beijing_time(day + 1, 3, 0, 0) + ); + assert_eq!( + duration_until( + next_forced_refresh_at_or_after(beijing_time(day, 17, 45, 0)), + beijing_time(day, 17, 45, 0) + ), + Duration::from_secs(15 * 60) + ); + } + #[test] fn formats_progress_durations() { assert_eq!(format_duration(Duration::from_millis(7)), "00:00.007"); @@ -642,4 +773,13 @@ mod tests { ); assert_eq!(OfficialUpdateStatus::Downloaded.as_str(), "downloaded"); } + + fn beijing_time(day: u64, hour: u64, minute: u64, second: u64) -> SystemTime { + system_time_from_beijing_local_seconds( + day.saturating_mul(SECONDS_PER_DAY) + .saturating_add(hour.saturating_mul(60 * 60)) + .saturating_add(minute.saturating_mul(60)) + .saturating_add(second), + ) + } } diff --git a/infrastructure/src/official_download.rs b/infrastructure/src/official_download.rs index ed1a188..2a573e2 100644 --- a/infrastructure/src/official_download.rs +++ b/infrastructure/src/official_download.rs @@ -276,6 +276,22 @@ impl OfficialLocalManifestAuditReport { } } +/// Existing local state for an official pull plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OfficialLocalResourceState { + /// Number of expected URLs that already have download-manifest entries. + pub manifest_entry_count: usize, + /// Number of expected URL destinations that already exist on disk. + pub existing_file_count: usize, +} + +impl OfficialLocalResourceState { + /// Returns true when this plan has any reusable or auditable local state. + pub fn has_any_resources(self) -> bool { + self.manifest_entry_count > 0 || self.existing_file_count > 0 + } +} + /// Executes an official pull plan with the system `curl` command. #[derive(Debug, Clone)] pub struct OfficialResourcePullService { @@ -357,6 +373,9 @@ impl OfficialResourcePullService { let urls = plan.all_urls()?; let total = urls.len(); let mut items = Vec::new(); + let mut verified_hashes = Vec::new(); + let mut verified_hash_urls = HashSet::::new(); + let mut processed_urls = HashSet::::new(); for (offset, url) in urls.into_iter().enumerate() { if should_cancel() { return Err("official resource pull interrupted by shutdown request".to_string()); @@ -406,6 +425,14 @@ impl OfficialResourcePullService { result.bytes, result.transferred_bytes, )); + processed_urls.insert(url.clone()); + self.verify_ready_official_hashes( + &official_hash_pairs, + &processed_urls, + &mut verified_hash_urls, + &mut verified_hashes, + &mut manifest, + )?; items.push(OfficialResourcePullItem { url, destination, @@ -418,8 +445,7 @@ impl OfficialResourcePullService { if should_cancel() { return Err("official resource pull interrupted by shutdown request".to_string()); } - - let verified_hashes = self.verify_downloaded_official_hashes(&official_hash_pairs)?; + self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?; Ok(OfficialResourcePullReport { items, @@ -447,6 +473,34 @@ impl OfficialResourcePullService { Ok(OfficialLocalManifestAuditReport { items }) } + /// Returns whether the current output root already contains local state for + /// this plan. This is intentionally cheaper than a full audit and is used + /// to choose between audit-then-repair and first-time full pull flows. + pub fn local_resource_state( + &self, + plan: &OfficialResourcePullPlan, + ) -> Result { + let manifest = self.read_download_manifest()?; + let mut manifest_entry_count = 0; + let mut existing_file_count = 0; + + for url in plan.all_urls()? { + if manifest.entries.contains_key(&url) { + manifest_entry_count += 1; + } + + let destination = self.destination_for_url(&url)?; + if file_len_if_exists(&destination)?.is_some() { + existing_file_count += 1; + } + } + + Ok(OfficialLocalResourceState { + manifest_entry_count, + existing_file_count, + }) + } + /// Fetches an official URL into memory. /// /// This is intended for small discovery inputs such as `server-info`, @@ -604,37 +658,103 @@ impl OfficialResourcePullService { Ok(()) } - fn verify_downloaded_official_hashes( + fn verify_ready_official_hashes( &self, pairs: &[OfficialSeedHashPair], - ) -> Result, String> { - let mut verified = Vec::new(); - + processed_urls: &HashSet, + verified_hash_urls: &mut HashSet, + verified_hashes: &mut Vec, + manifest: &mut OfficialDownloadManifest, + ) -> Result<(), String> { for pair in pairs { - let data_path = self.destination_for_url(&pair.data_url)?; - let hash_path = self.destination_for_url(&pair.hash_url)?; - let data = fs::read(&data_path).map_err(|error| { - format!( - "Failed to read official hash data file {}: {error}", - data_path.display() - ) - })?; - let hash = fs::read(&hash_path).map_err(|error| { - format!( - "Failed to read official hash sidecar {}: {error}", - hash_path.display() - ) - })?; + if verified_hash_urls.contains(&pair.hash_url) { + continue; + } - verified.push(verify_official_seed_catalog_hash( - &pair.data_url, - &data, - &pair.hash_url, - &hash, - )?); + if !processed_urls.contains(&pair.data_url) || !processed_urls.contains(&pair.hash_url) + { + continue; + } + + if !self.hash_pair_files_exist(pair)? { + continue; + } + + let verification = match self.verify_official_seed_hash_pair_from_disk(pair) { + Ok(verification) => verification, + Err(error) => { + if let Err(cleanup_error) = + self.invalidate_download_manifest_hash_pair(manifest, pair) + { + return Err(format!( + "{error}; additionally failed to invalidate local manifest entries for official hash pair: {cleanup_error}" + )); + } + + return Err(error); + } + }; + + verified_hashes.push(verification); + verified_hash_urls.insert(pair.hash_url.clone()); } - Ok(verified) + Ok(()) + } + + fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result { + let data_path = self.destination_for_url(&pair.data_url)?; + let hash_path = self.destination_for_url(&pair.hash_url)?; + Ok(file_len_if_exists(&data_path)?.is_some() && file_len_if_exists(&hash_path)?.is_some()) + } + + fn verify_official_seed_hash_pair_from_disk( + &self, + pair: &OfficialSeedHashPair, + ) -> Result { + let data_path = self.destination_for_url(&pair.data_url)?; + let hash_path = self.destination_for_url(&pair.hash_url)?; + let data = fs::read(&data_path).map_err(|error| { + format!( + "Failed to read official hash data file {}: {error}", + data_path.display() + ) + })?; + let hash = fs::read(&hash_path).map_err(|error| { + format!( + "Failed to read official hash sidecar {}: {error}", + hash_path.display() + ) + })?; + + verify_official_seed_catalog_hash(&pair.data_url, &data, &pair.hash_url, &hash) + } + + fn invalidate_download_manifest_hash_pair( + &self, + manifest: &mut OfficialDownloadManifest, + pair: &OfficialSeedHashPair, + ) -> Result<(), String> { + manifest.entries.remove(&pair.data_url); + manifest.entries.remove(&pair.hash_url); + self.write_download_manifest(manifest) + } + + fn verify_all_official_hashes_are_complete( + &self, + pairs: &[OfficialSeedHashPair], + verified_hash_urls: &HashSet, + ) -> Result<(), String> { + for pair in pairs { + if !verified_hash_urls.contains(&pair.hash_url) { + return Err(format!( + "official hash pair was not verified for {} using {}", + pair.data_url, pair.hash_url + )); + } + } + + Ok(()) } fn read_download_manifest(&self) -> Result { @@ -1607,6 +1727,43 @@ fi .all(|item| item.status == OfficialResourcePullStatus::Downloaded)); } + #[test] + fn local_resource_state_reports_manifest_entries_and_existing_files() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_curl(&curl_path); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); + let plan = build_official_pull_plan(discovery_plan(), inventory()); + let urls = plan.all_urls().unwrap(); + + let empty_state = service.local_resource_state(&plan).unwrap(); + assert_eq!(empty_state.manifest_entry_count, 0); + assert_eq!(empty_state.existing_file_count, 0); + assert!(!empty_state.has_any_resources()); + + let first_destination = service.destination_for_url(&urls[0]).unwrap(); + fs::create_dir_all(first_destination.parent().unwrap()).unwrap(); + fs::write(&first_destination, b"local-only").unwrap(); + + let file_only_state = service.local_resource_state(&plan).unwrap(); + assert_eq!(file_only_state.manifest_entry_count, 0); + assert_eq!(file_only_state.existing_file_count, 1); + assert!(file_only_state.has_any_resources()); + + let mut manifest = OfficialDownloadManifest::default(); + service + .record_download_manifest_entry(&mut manifest, &urls[0], &first_destination) + .unwrap(); + service.write_download_manifest(&manifest).unwrap(); + + let manifest_state = service.local_resource_state(&plan).unwrap(); + assert_eq!(manifest_state.manifest_entry_count, 1); + assert_eq!(manifest_state.existing_file_count, 1); + assert!(manifest_state.has_any_resources()); + } + #[test] fn redownloads_existing_file_when_manifest_hash_mismatches() { let out_dir = TempDir::new().unwrap(); @@ -1677,6 +1834,17 @@ fi let error = service.pull(&plan).unwrap_err(); assert!(error.contains("official hash mismatch")); + + let manifest = service.read_download_manifest().unwrap(); + assert!(manifest.entries.is_empty()); + + let audit = service.audit_local_manifest(&plan).unwrap(); + assert!(!audit.is_clean()); + assert_eq!(audit.repair_needed_count(), 2); + assert!(audit + .items + .iter() + .all(|item| { item.status == OfficialLocalManifestAuditStatus::MissingManifestEntry })); } #[test] diff --git a/infrastructure/src/official_update.rs b/infrastructure/src/official_update.rs index afdfa11..67d7d7f 100644 --- a/infrastructure/src/official_update.rs +++ b/infrastructure/src/official_update.rs @@ -627,35 +627,39 @@ impl OfficialUpdateService { sync_plan.decision, config.force, remote_should_download ), )); - let pull_plan = if config.audit_local || remote_should_download || config.plan { - progress(OfficialUpdateProgress::new( - "plan", - "building official pull plan from seed catalogs", - )); - Some(build_pull_plan( - &server_info, - &connection_group, - &app_version, - platforms, - &fetcher, - &mut progress, - &mut should_cancel, - )?) - } else { - progress(OfficialUpdateProgress::new( - "plan", - "pull plan not needed for clean remote state and disabled plan output", - )); - None - }; - if let Some(pull_plan) = pull_plan.as_ref() { - let url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len(); - progress(OfficialUpdateProgress::new( - "plan", - format!("pull plan contains {url_count} official URLs"), - )); - } - let local_audit = if config.audit_local { + progress(OfficialUpdateProgress::new( + "plan", + "building official pull plan from latest seed catalogs", + )); + let pull_plan = build_pull_plan( + &server_info, + &connection_group, + &app_version, + platforms, + &fetcher, + &mut progress, + &mut should_cancel, + )?; + let url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len(); + progress(OfficialUpdateProgress::new( + "plan", + format!("pull plan contains {url_count} official URLs"), + )); + let local_state = fetcher + .local_resource_state(&pull_plan) + .map_err(anyhow::Error::msg)?; + let has_local_resources = local_state.has_any_resources(); + progress(OfficialUpdateProgress::new( + "local-state", + format!( + "manifest_entries={} existing_files={} has_local_resources={}", + local_state.manifest_entry_count, + local_state.existing_file_count, + has_local_resources + ), + )); + let should_audit_local = config.audit_local && has_local_resources; + let local_audit = if should_audit_local { progress(OfficialUpdateProgress::new( "audit", format!( @@ -665,13 +669,15 @@ impl OfficialUpdateService { )); Some( fetcher - .audit_local_manifest( - pull_plan - .as_ref() - .ok_or_else(|| anyhow::anyhow!("local audit requires a pull plan"))?, - ) + .audit_local_manifest(&pull_plan) .map_err(anyhow::Error::msg)?, ) + } else if config.audit_local { + progress(OfficialUpdateProgress::new( + "audit", + "no local resources found; skipping local audit before first full pull", + )); + None } else { progress(OfficialUpdateProgress::new( "audit", @@ -692,7 +698,8 @@ impl OfficialUpdateService { .as_ref() .map(|audit| !audit.is_clean()) .unwrap_or(false); - let should_download = remote_should_download || repair_needed; + let initial_pull_needed = !has_local_resources; + let should_download = remote_should_download || repair_needed || initial_pull_needed; progress(OfficialUpdateProgress::new( "audit", format!( @@ -703,8 +710,8 @@ impl OfficialUpdateService { progress(OfficialUpdateProgress::new( "decision", format!( - "final should_download={} repair_needed={}", - should_download, repair_needed + "final should_download={} repair_needed={} initial_pull_needed={}", + should_download, repair_needed, initial_pull_needed ), )); check_shutdown_requested(&mut should_cancel)?; @@ -760,13 +767,7 @@ impl OfficialUpdateService { if config.dry_run { if config.plan { - let urls = pull_plan - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!("dry-run plan requested but no pull plan exists") - })? - .all_urls() - .map_err(anyhow::Error::msg)?; + let urls = pull_plan.all_urls().map_err(anyhow::Error::msg)?; report.download_url_count = Some(urls.len()); report.download_urls = urls; progress(OfficialUpdateProgress::new( @@ -791,9 +792,6 @@ impl OfficialUpdateService { } check_shutdown_requested(&mut should_cancel)?; - let pull_plan = pull_plan - .as_ref() - .ok_or_else(|| anyhow::anyhow!("download requested but no pull plan exists"))?; let download_url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len(); progress(OfficialUpdateProgress::new( "download", @@ -801,7 +799,7 @@ impl OfficialUpdateService { )); let pull_report = fetcher .pull_with_progress_and_cancellation( - pull_plan, + &pull_plan, |event| { progress(progress_from_pull_event(event)); }, @@ -824,7 +822,7 @@ impl OfficialUpdateService { )); check_shutdown_requested(&mut should_cancel)?; let final_audit = fetcher - .audit_local_manifest(pull_plan) + .audit_local_manifest(&pull_plan) .map_err(anyhow::Error::msg)?; progress(OfficialUpdateProgress::new( "snapshot", diff --git a/infrastructure/tests/official_game_main_config_bootstrap.rs b/infrastructure/tests/official_game_main_config_bootstrap.rs index 676cd20..95acb41 100644 --- a/infrastructure/tests/official_game_main_config_bootstrap.rs +++ b/infrastructure/tests/official_game_main_config_bootstrap.rs @@ -185,6 +185,58 @@ fn official_update_service_emits_human_progress_events() { .contains("pull plan contains 19 official URLs"))); } +#[test] +fn official_update_first_run_pulls_without_pre_audit() { + let harness = TestHarness::new(); + let config = harness.sync_config("first-run-output"); + let mut events = Vec::::new(); + + let report = OfficialUpdateService::new() + .run_with_progress(&config, |event| events.push(event)) + .unwrap(); + + assert_eq!(report.update_status, OfficialUpdateStatus::Downloaded); + assert_eq!(report.resource_count, Some(19)); + assert_eq!(report.downloaded_count, 19); + assert_eq!(report.skipped_count, 0); + assert_eq!(report.local_manifest_repair_needed_count, 0); + assert!(events.iter().any(|event| { + event.stage == "local-state" && event.message.contains("has_local_resources=false") + })); + assert!(events + .iter() + .any(|event| { event.stage == "audit" && event.message.contains("skipping local audit") })); + assert!(events.iter().any(|event| { + event.stage == "decision" && event.message.contains("initial_pull_needed=true") + })); +} + +#[test] +fn official_update_second_run_audits_existing_resources_before_reuse() { + let harness = TestHarness::new(); + let config = harness.sync_config("second-run-output"); + OfficialUpdateService::new().run(&config).unwrap(); + + let mut events = Vec::::new(); + let report = OfficialUpdateService::new() + .run_with_progress(&config, |event| events.push(event)) + .unwrap(); + + assert_eq!(report.update_status, OfficialUpdateStatus::UpToDate); + assert_eq!(report.downloaded_count, 0); + assert_eq!(report.local_manifest_verified_count, 19); + assert_eq!(report.local_manifest_repair_needed_count, 0); + assert!(events.iter().any(|event| { + event.stage == "local-state" && event.message.contains("has_local_resources=true") + })); + assert!(events.iter().any(|event| { + event.stage == "audit" && event.message.contains("auditing local download manifest") + })); + assert!(events.iter().any(|event| { + event.stage == "decision" && event.message.contains("initial_pull_needed=false") + })); +} + struct TestHarness { temp: TempDir, curl_script: std::path::PathBuf, @@ -259,6 +311,17 @@ impl TestHarness { &self.curl_script, ) } + + fn sync_config(&self, output_name: &str) -> OfficialUpdateConfig { + OfficialUpdateConfig { + auto_discover: true, + launcher_version: TEST_LAUNCHER_VERSION.to_string(), + output_root: self.temp.path().join(output_name), + curl_command: self.curl_script.clone(), + unzip_command: self.unzip_script.clone(), + ..OfficialUpdateConfig::default() + } + } } fn write_executable(path: &Path, content: &str) {