From 4192d7ee4ce1ba2ac17c519305b2bbedd4aeb2be Mon Sep 17 00:00:00 2001 From: Yuyi-Oak <1722157266@qq.com> Date: Tue, 14 Jul 2026 01:30:12 +0800 Subject: [PATCH] fix: classify official download failures Distinguish terminal HTTP errors from retryable CDN/network failures, record quarantined resources, and report failed download progress. Fixes #7 --- CHANGELOG.md | 1 + CURRENT_STATUS.md | 8 +- README.md | 4 +- .../architecture/official-resource-backend.md | 10 +- docs/guides/official-resource-test-pull.md | 5 + docs/reports/CURRENT_GAPS.md | 26 + docs/reports/current-stage-prepush.md | 3 +- docs/reports/current-status-handoff.md | 2 + infrastructure/src/bin/bat_official_sync.rs | 45 ++ infrastructure/src/curl_transfer.rs | 308 ++++++++++ infrastructure/src/lib.rs | 1 + infrastructure/src/official_download.rs | 558 ++++++++++++++---- .../src/official_game_main_config.rs | 144 +++-- infrastructure/src/official_launcher.rs | 60 +- infrastructure/src/official_update.rs | 54 +- 15 files changed, 1034 insertions(+), 195 deletions(-) create mode 100644 infrastructure/src/curl_transfer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a25d0..ccdec98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - 新增官方资源同步生产部署模板：release binary symlink 路径、systemd unit、运行用户、日志位置、升级和回滚流程 - 新增真实官方网络全量拉取 smoke：`scripts/official-full-pull-smoke.sh`、`make official-smoke` 和 `docs/guides/official-full-pull-smoke.md` - 新增官方资源下载校验：官方 URL 拒绝、`.part` 续传、重试、本地 size+BLAKE3、官方 seed `.hash` 校验 +- 新增下载失败分类与 quarantine：403/404/普通 4xx 不重试，5xx/429/网络类错误重试，最终失败写入 `official-download-quarantine.json` 并阻止发布不完整资源；旧 launcher 包支持官方 primary/backup CDN 切换 - 新增 Addressables 当前真实形态 fixture/golden 测试 - 新增 SQLite Resource Repository 和粗粒度 FFI JSON 接口 - 补齐官方资源运行、架构、状态、缺口和交接文档 diff --git a/CURRENT_STATUS.md b/CURRENT_STATUS.md index 94da020..002396d 100644 --- a/CURRENT_STATUS.md +++ b/CURRENT_STATUS.md @@ -19,8 +19,8 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 3. 默认平台为 `Windows + Android`。 4. 能生成官方全量 pull plan，执行真实下载，维护 release 内的 `official-download-manifest.json`。 5. 下载后使用本地 manifest 的 size + BLAKE3 校验复用文件；所有 `.zip` 在下载验收、复用、本地 audit/verify 时做 ZIP 结构校验；官方 seed `.hash` 使用 `xxHash32(seed=0)` 强校验。 -6. 支持 `.part` 断点续传、失败后 clean retry、本地 manifest audit/repair。 -7. `bat --watch` 可常驻运行，`bat --daemon` 可后台运行并用 `bat status` / `bat stop` / `bat restart` / `bat reload` / `bat logs` 管理；daemon 使用 `bat.sock` Unix socket JSON-RPC 作为 live 控制通道，PID/状态/日志文件作为快照和 fallback，`bat-events.jsonl` 记录带轮转的结构化事件日志，`bat-control.lock` 串行化控制命令；正常检查默认每 1 小时一次；远端和本地一致时默认静默，失败后默认 60 秒快速重试；CLI 默认向 stdout 输出人类可读摘要，向 stderr 输出 ASCII banner 和 progress log，需要机器输出时使用 `--json --no-progress`。 +6. 支持 `.part` 断点续传、失败后 clean retry、本地 manifest audit/repair、403/404/5xx 分类重试、下载 quarantine 诊断，以及旧 launcher 包官方 primary/backup CDN 切换。 +7. `bat --watch` 可常驻运行，`bat --daemon` 可后台运行并用 `bat status` / `bat stop` / `bat restart` / `bat reload` / `bat logs` 管理；daemon 使用 `bat.sock` Unix socket JSON-RPC 作为 live 控制通道，PID/状态/日志文件作为快照和 fallback，`bat-events.jsonl` 记录带轮转的结构化事件日志，`bat-control.lock` 串行化控制命令；正常检查默认每 1 小时一次；远端和本地一致时默认静默，失败后默认 60 秒快速重试；CLI 默认向 stdout 输出人类可读摘要，向 stderr 输出 ASCII banner、progress log、失败分类和 quarantine 状态，需要机器输出时使用 `--json --no-progress`。 8. 远端 snapshot 未变化但输出目录为空时，会按首次运行执行全量拉取；官方 seed `.hash` 校验失败时会清理对应 manifest 条目，避免失败产物被后续本地 audit 误判为可复用。 9. 默认资源目录是 `./bat-resources`，默认后台状态目录是 `/tmp/bat-pid`；资源目录是发布根目录，包含 `current` symlink、`versions/` 和 `.staging/`，非 dry-run 会先写 staging，校验完成后发布 versioned 目录并原子切换 `current`；后台状态目录包含 `bat.sock`、`bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl` 和短生命周期 `bat-control.lock`；非 dry-run 使用 `--output/.official-sync.lock` 防止并发写同一资源目录，live daemon 会阻止前台写命令直接修改它正在管理的同一目录。 10. 官方同步会拒绝危险输出目录、路径逃逸和现有 symlink 路径组件；下载目标、`.part`、manifest、snapshot、PID、status、log 和控制锁文件不会跟随 symlink，daemon 状态类文件默认以 `0600` 权限创建。 @@ -128,7 +128,7 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： - `InMemoryResourceRepository`。 - `SqliteResourceRepository`。 - 官方 pull plan 构建。 -- `OfficialResourcePullService`：官方 URL 拒绝策略、目标路径映射、下载 manifest、`.part` 续传、重试、ZIP 结构校验、官方 seed `.hash` 校验、本地全量 verify。 +- `OfficialResourcePullService`：官方 URL 拒绝策略、目标路径映射、下载 manifest、下载 quarantine、`.part` 续传、403/404/5xx 分类重试、ZIP 结构校验、官方 seed `.hash` 校验、本地全量 verify。 - `OfficialUpdateService`：官方 metadata auto-discover、bootstrap cache、snapshot diff、marker diff、本地 audit/repair。 - `bat`：正式 CLI binary，支持 one-shot、`--watch`、`--daemon`、`status`、`stop`、`restart`、`reload`、`refresh`、`logs`、`verify`、`repair`、`doctor` 和 `clean-stable`。 @@ -136,7 +136,7 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： - 将官方同步下载结果作为用户级流程自动导入 CAS + ResourceRepository。 - 真实线上全量下载 smoke 已固化为 `scripts/official-full-pull-smoke.sh` 和 `make official-smoke`；实际运行报告由脚本写入隔离输出目录。 -- 增加更多失败恢复和权限场景测试。 +- 增加更多权限和极端文件系统场景测试。 ### `bat-assetbundle` diff --git a/README.md b/README.md index e46188f..4cd74e4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ - `bat-adapters` Unity、Manifest、Client 集成框架，以及当前真实形态 Addressables catalog 解析覆盖。 - `bat-cas-engine` CAS V1：原子写入、BLAKE3 校验、引用计数、GC、并发写入测试、损坏检测。 - `bat-infrastructure` CAS 适配层、SQLite Resource Repository、官方资源 pull/update 服务。 -- `bat`：官方资源自动发现、全量拉取、原子发布到 `current -> versions/`、本地 manifest audit/repair、`.part` 断点续传、ZIP 结构校验、官方 seed `.hash` 校验、snapshot/cache、`--watch` 常驻更新、`--daemon` 后台运行，以及 Unix socket JSON-RPC 后台控制命令 `status/stop/restart/reload/refresh/logs/verify/repair/doctor/clean-stable`。 +- `bat`：官方资源自动发现、全量拉取、原子发布到 `current -> versions/`、本地 manifest audit/repair、`.part` 断点续传、403/404/5xx 分类重试、下载 quarantine 诊断、ZIP 结构校验、官方 seed `.hash` 校验、snapshot/cache、`--watch` 常驻更新、`--daemon` 后台运行，以及 Unix socket JSON-RPC 后台控制命令 `status/stop/restart/reload/refresh/logs/verify/repair/doctor/clean-stable`。 - `bat-ffi` 粗粒度 JSON 接口：Manifest inspect 和官方 sync plan。 - 文档路线图、当前状态、缺口清单、官方资源运行指南。 @@ -111,6 +111,8 @@ cargo run -p bat-infrastructure --bin bat -- clean-stable `verify` 会以只读方式检查当前官方计划、`current` 指向的 active release 中 download manifest 的 size+BLAKE3、ZIP 结构，以及本地已有官方 seed `.bytes/.hash` 对的 xxHash32；发现缺失、远端变化或本地损坏会返回非 0。`repair` 会在异常资源存在时复用当前同步链路重新下载必要文件。`clean-stable` 只清理 `.part`、临时状态文件、失效或损坏的 PID/锁/socket，不删除正式资源。 +下载失败会按 curl exit 和 HTTP 状态分类：403/404/普通 4xx 视为不可重试，5xx、429、DNS、连接、超时、中断和网络类错误会按尝试次数重试。某个 URL 最终失败后会写入 `/current` 或 staging 下的 `official-download-quarantine.json`，stderr progress、daemon status 和 `bat-events.jsonl` 会记录失败类型、HTTP 状态、是否可重试、尝试次数和 quarantine 状态；同步会中断并阻止发布不完整资源。旧 launcher 包下载路径会在官方 primary CDN 失败后切换官方 backup CDN。 + CLI 默认启动时会向 stderr 打印 `BlueArchiveToolkit` ASCII banner，并继续把阶段进度日志写到 stderr，例如自动发现、拉取 catalog、audit、总体下载进度、单文件下载进度、校验结果摘要、snapshot 和 publish；命令结果默认以人类可读摘要写到 stdout。需要给上层程序保留稳定结构化输出时加 `--json --no-progress`，只想关闭横幅但保留日志时可加 `--no-banner`。 真实官方网络全量拉取 smoke 已固化为可重复命令，默认使用 `/tmp/bat-official-smoke-/` 隔离目录，不会写入已有客户端、生产目录或开发机人工维护资源目录： diff --git a/docs/architecture/official-resource-backend.md b/docs/architecture/official-resource-backend.md index 87f368c..f366870 100644 --- a/docs/architecture/official-resource-backend.md +++ b/docs/architecture/official-resource-backend.md @@ -130,8 +130,11 @@ 9. 存在 `.part` 临时文件时通过 `curl --continue-at -` 尝试断点续传。 10. 新下载写入 `.part`，成功并通过必要校验后原子 rename 到 staging 内最终路径；断点续传后的 `.zip` 如果结构无效，会删除 `.part` 并重新全量下载。 11. 成功下载后更新本地下载清单。 -12. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 -13. 非官方 URL 直接拒绝。 +12. curl 失败按 HTTP/网络类型分类：403/404/普通 4xx 不重试，5xx、429、DNS、连接、超时、中断和网络类错误按尝试次数重试。 +13. 单个 URL 最终失败时写入 `official-download-quarantine.json`，发出 Failed progress，并阻止发布不完整资源。 +14. 旧 launcher 包或 `resources.assets` 下载使用官方 launcher CDN 配置，primary CDN 失败后切换 official backup CDN；资源 patch host 不猜测非官方镜像。 +15. 记录最终文件大小、本次传输字节数、官方 hash 校验数和执行状态。 +16. 非官方 URL 直接拒绝。 路径映射时会做分段清理，并在写入前做输出目录安全校验、相对路径归属校验和现有路径组件 symlink 检查，避免把不安全路径写进输出目录或通过 symlink 跳出输出目录。 @@ -233,7 +236,8 @@ Linux 生产路径： - pull plan 会同时包含 discovery URLs 和 content URLs - 全量样本下是 `2` 个 discovery URL + `5` 个内容 URL = `7` 个 URL - `OfficialUpdateService` 能持久化 v2 snapshot，并在远端 marker 内容变化时触发下载决策 -- `bat` 默认向 stderr 输出 `BlueArchiveToolkit` ASCII banner 和 progress log，stdout 默认输出人类可读摘要；progress log 覆盖总体下载进度、单文件开始/完成状态和校验结果摘要；支持 `--json` 输出稳定 JSON，支持 `--no-progress` 关闭进度日志，支持 `--no-banner` 只关闭横幅，支持 `--watch --interval 1h --error-retry 60s` 常驻运行，支持 `--daemon` Unix socket JSON-RPC 控制、`status`、`stop`、`restart`、`reload`、`logs`、`refresh --force`、`verify`、`repair`、`doctor`、`clean-stable`，非 dry-run 使用 `.official-sync.lock` 防止并发写资源目录，控制命令使用 `bat-control.lock` 防止并发状态修改，资源发布使用 `.staging`、`versions` 和 `current` 原子切换，daemon 写 `bat-events.jsonl` 结构化日志并在 `status` 中暴露下载进度和调度状态 +- `bat` 默认向 stderr 输出 `BlueArchiveToolkit` ASCII banner 和 progress log，stdout 默认输出人类可读摘要；progress log 覆盖总体下载进度、单文件开始/完成状态、下载中断失败分类和校验结果摘要；支持 `--json` 输出稳定 JSON，支持 `--no-progress` 关闭进度日志，支持 `--no-banner` 只关闭横幅，支持 `--watch --interval 1h --error-retry 60s` 常驻运行，支持 `--daemon` Unix socket JSON-RPC 控制、`status`、`stop`、`restart`、`reload`、`logs`、`refresh --force`、`verify`、`repair`、`doctor`、`clean-stable`，非 dry-run 使用 `.official-sync.lock` 防止并发写资源目录，控制命令使用 `bat-control.lock` 防止并发状态修改，资源发布使用 `.staging`、`versions` 和 `current` 原子切换，daemon 写 `bat-events.jsonl` 结构化日志并在 `status` 中暴露下载进度、失败类型、HTTP 状态和调度状态 +- curl 失败分类和重试策略已覆盖 404 不重试、5xx 重试耗尽后 quarantine、launcher primary CDN 失败后切换 official backup CDN - `OfficialUpdateService` 能读写 `official-bootstrap-cache.json`，并支持默认开启的 `audit_local` / `repair` CLI 行为 - 下载层能在本地文件 size/BLAKE3/path、ZIP 结构或 manifest 不匹配时重新下载 - 官方 seed `.hash` mismatch 会导致下载失败，而不是降级为本地 BLAKE3 猜测 diff --git a/docs/guides/official-resource-test-pull.md b/docs/guides/official-resource-test-pull.md index 71739ac..a985055 100644 --- a/docs/guides/official-resource-test-pull.md +++ b/docs/guides/official-resource-test-pull.md @@ -155,6 +155,10 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - 远端无变化时，默认执行 active release 中的 `official-download-manifest.json` audit。 - 本地文件缺失、路径不一致、size 不一致、BLAKE3 不一致或 ZIP 结构无效时，默认进入 repair 并重下必要文件。 - 官方 seed `.hash` 校验失败会让本轮失败，并清理对应本地 manifest 条目；下一轮会继续把这类文件视为需要 repair，而不是把失败产物当作健康缓存复用。 +- curl 失败会按类型分类：403/404/普通 4xx 不重试，5xx、429、DNS、连接、超时、中断和网络类错误按尝试次数重试。 +- 单个 URL 最终失败后会写入 `official-download-quarantine.json`，progress log、daemon status 和 `bat-events.jsonl` 会记录失败类型、HTTP 状态、是否可重试、尝试次数和 quarantine 状态。 +- quarantine 项会跳过本轮发布并让同步失败，避免把不完整 staging 发布到 `current`；下一轮 repair/refresh 成功后会清理对应 quarantine 条目。 +- 旧 launcher 包或 `resources.assets` 下载路径使用官方 launcher CDN 配置，primary CDN 失败后会切换官方 backup CDN；资源 patch host 当前只使用 server-info 返回的官方 client-patch host，不猜测非官方镜像。 - 远端和本地都一致：单次模式输出 `update_status=up_to_date`，watch 模式默认静默并等待下次检查。 - 有远端变化或本地 repair：生成 pull plan，下载完整官方资源到 staging，成功后更新 snapshot 并原子发布到 `current`。 - `--dry-run`：只报告本次是否会下载，不写 snapshot；如果 cache miss，也不会写入新的 bootstrap cache。 @@ -168,6 +172,7 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - `/current/official-sync-snapshot.json`：上一次成功同步的 v2 snapshot，包含 app version、connection group、bundle version、addressables root、endpoint URL、官方 seed `.hash` 内容、Addressables `catalog_*.hash` marker、launcher metadata 摘要和 `GameMainConfig` 摘要。 - `/official-bootstrap-cache.json`：`--auto-discover` 的 `GameMainConfig` 解析缓存。launcher metadata 未变时复用缓存；metadata 变化时才通过官方 HTTP 按 manifest 下载必要 `resources.assets` 或旧版 game zip 到临时目录解析。 - `/current/official-download-manifest.json`：本地下载强校验清单，记录 URL、相对路径、size 和 BLAKE3。 +- `/current/official-download-quarantine.json` 或当前 staging 下同名文件：下载最终失败的 URL 诊断记录，包含失败类型、HTTP 状态、是否可重试、尝试次数和最后错误。 先 dry-run： diff --git a/docs/reports/CURRENT_GAPS.md b/docs/reports/CURRENT_GAPS.md index 9720f7f..0f17299 100644 --- a/docs/reports/CURRENT_GAPS.md +++ b/docs/reports/CURRENT_GAPS.md @@ -344,6 +344,32 @@ - 脚本退出码为 0 即表示 runbook 验收通过。 - 真实网络执行需要外部网络和足够磁盘空间；本仓库只保存 runbook、脚本和测试，不保存官方大文件。 +### G-019：下载失败重试策略不够精细 + +状态：**已关闭** + +历史现象： + +- curl 失败只按固定次数重试，错误信息主要保留最后一次 stderr。 +- 403/404 和 5xx 没有不同处理。 +- 单个 URL 长期失败时缺少可查询的 quarantine 诊断状态。 +- 旧 launcher 包下载虽然有 primary/backup CDN 路径，但失败信息没有统一分类。 + +处理结果： + +- 新增统一 curl 失败分类：`http_forbidden`、`http_not_found`、`http_client_error`、`http_too_many_requests`、`http_server_error`、`dns`、`connect`、`timeout`、`tls`、`interrupted`、`network` 等。 +- 403/404/普通 4xx 视为不可重试并提前停止；5xx、429、DNS、连接、超时、中断和网络类错误按尝试次数重试。 +- 单个资源 URL 最终失败会写入 `official-download-quarantine.json`，记录失败类型、HTTP 状态、是否可重试、尝试次数和最后错误。 +- 失败 URL 会发出 Failed progress，daemon status 和 `bat-events.jsonl` 暴露失败类型、HTTP 状态、重试属性和 quarantine 状态。 +- quarantine 会中断同步并阻止发布不完整 staging；下一轮成功下载或复用后清理对应 quarantine 条目。 +- 旧 launcher 包或 `resources.assets` 下载在 primary CDN 失败后会切换官方 backup CDN。 + +验收： + +- HTTP 404 不重试，写入 quarantine，manifest 不写失败项。 +- HTTP 5xx 重试到上限后写入 quarantine。 +- launcher primary CDN 失败后会尝试官方 backup CDN。 + --- ## 6. 当前关闭顺序建议 diff --git a/docs/reports/current-stage-prepush.md b/docs/reports/current-stage-prepush.md index afaab88..1d77e81 100644 --- a/docs/reports/current-stage-prepush.md +++ b/docs/reports/current-stage-prepush.md @@ -25,7 +25,8 @@ - `bat-cas-engine` CAS V1。 - `bat-adapters` 官方日服 URL 规则、server-info 解析、平台 discovery、inventory 枚举。 - `AddressablesCatalogDriver` 对当前真实形态 fixture/golden 的解析。 -- `OfficialResourcePullService` 的官方 URL 拒绝、`.part` 续传、重试、本地 manifest、官方 seed `.hash` 校验。 +- `OfficialResourcePullService` 的官方 URL 拒绝、`.part` 续传、403/404/5xx 分类重试、下载 quarantine、本地 manifest、官方 seed `.hash` 校验。 +- 旧 launcher 包或 `resources.assets` 下载的官方 primary/backup CDN 切换。 - `OfficialUpdateService` 的 auto-discover、bootstrap cache、v2 snapshot、remote marker diff、本地 audit/repair。 - `bat` one-shot 和 `--watch`。 - `bat` 运行时 progress log 覆盖总体下载进度、单文件进度和校验结果摘要。 diff --git a/docs/reports/current-status-handoff.md b/docs/reports/current-status-handoff.md index f18c77f..e2e1b11 100644 --- a/docs/reports/current-status-handoff.md +++ b/docs/reports/current-status-handoff.md @@ -18,6 +18,8 @@ - `--auto-discover` 通过官方 HTTP metadata 和临时目录解析 `GameMainConfig`。 - 默认平台是 `Windows + Android`。 - 真实下载会维护 `official-download-manifest.json`，并使用 size + BLAKE3、本地 audit/repair 和官方 seed `.hash` 校验。 +- 下载失败会按 403/404/5xx/网络类型分类；最终失败写入 `official-download-quarantine.json` 并阻止发布不完整资源。 +- 旧 launcher 包或 `resources.assets` 下载使用官方 primary/backup CDN 切换。 - `--watch` 是 Rust 内部常驻检查模式，正常检查默认 1 小时，失败重试默认 60 秒；外部 systemd/container 只负责守护进程。 - `bat` progress log 会输出总体下载进度、单文件进度和校验结果摘要。 - 真实官方网络 smoke 已固化为 `scripts/official-full-pull-smoke.sh`、`make official-smoke` 和 `docs/guides/official-full-pull-smoke.md`，默认写入 `/tmp` 隔离目录。 diff --git a/infrastructure/src/bin/bat_official_sync.rs b/infrastructure/src/bin/bat_official_sync.rs index e4aa12a..cee6763 100644 --- a/infrastructure/src/bin/bat_official_sync.rs +++ b/infrastructure/src/bin/bat_official_sync.rs @@ -503,6 +503,16 @@ struct DaemonDownloadProgress { status: Option, bytes: Option, transferred_bytes: Option, + #[serde(default)] + failure_kind: Option, + #[serde(default)] + failure_http_status: Option, + #[serde(default)] + failure_retryable: Option, + #[serde(default)] + failure_attempts: Option, + #[serde(default)] + quarantined: Option, } #[derive(Debug)] @@ -1980,6 +1990,31 @@ fn print_list(label: &str, values: &[String], limit: usize) { fn format_daemon_download_progress(progress: &DaemonDownloadProgress) -> String { let status = progress.status.as_deref().unwrap_or("running"); + if status == "failed" { + return format!( + "{}/{} failed kind={} http={} retryable={} attempts={} quarantined={} {}", + progress.index, + progress.total, + progress.failure_kind.as_deref().unwrap_or("unknown"), + progress + .failure_http_status + .map(|status| status.to_string()) + .unwrap_or_else(|| "none".to_string()), + progress + .failure_retryable + .map(|retryable| retryable.to_string()) + .unwrap_or_else(|| "unknown".to_string()), + progress + .failure_attempts + .map(|attempts| attempts.to_string()) + .unwrap_or_else(|| "0".to_string()), + progress + .quarantined + .map(|quarantined| quarantined.to_string()) + .unwrap_or_else(|| "false".to_string()), + progress.url + ); + } format!( "{}/{} {} {}", progress.index, progress.total, status, progress.url @@ -2900,6 +2935,11 @@ fn update_daemon_progress(state_dir: &Path, event: &OfficialUpdateProgress) -> a status: event.download_status.clone(), bytes: event.download_bytes, transferred_bytes: event.download_transferred_bytes, + failure_kind: event.download_failure_kind.clone(), + failure_http_status: event.download_failure_http_status, + failure_retryable: event.download_failure_retryable, + failure_attempts: event.download_failure_attempts, + quarantined: event.download_quarantined, }), _ if event.stage != "download" => None, _ => status.download_progress, @@ -3354,6 +3394,11 @@ impl RotatingStructuredLogger { "status": event.download_status.as_deref(), "bytes": event.download_bytes, "transferred_bytes": event.download_transferred_bytes, + "failure_kind": event.download_failure_kind.as_deref(), + "failure_http_status": event.download_failure_http_status, + "failure_retryable": event.download_failure_retryable, + "failure_attempts": event.download_failure_attempts, + "quarantined": event.download_quarantined, })), }); let mut line = serde_json::to_vec(&payload)?; diff --git a/infrastructure/src/curl_transfer.rs b/infrastructure/src/curl_transfer.rs new file mode 100644 index 0000000..56acd26 --- /dev/null +++ b/infrastructure/src/curl_transfer.rs @@ -0,0 +1,308 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CurlFailureKind { + HttpForbidden, + HttpNotFound, + HttpClientError, + HttpTooManyRequests, + HttpServerError, + HttpUnknown, + Dns, + Connect, + Timeout, + Tls, + Interrupted, + Network, + ProcessFailed, + Unknown, +} + +impl CurlFailureKind { + pub(crate) fn as_str(&self) -> &'static str { + match self { + Self::HttpForbidden => "http_forbidden", + Self::HttpNotFound => "http_not_found", + Self::HttpClientError => "http_client_error", + Self::HttpTooManyRequests => "http_too_many_requests", + Self::HttpServerError => "http_server_error", + Self::HttpUnknown => "http_unknown", + Self::Dns => "dns", + Self::Connect => "connect", + Self::Timeout => "timeout", + Self::Tls => "tls", + Self::Interrupted => "interrupted", + Self::Network => "network", + Self::ProcessFailed => "process_failed", + Self::Unknown => "unknown", + } + } + + pub(crate) fn is_retryable(&self) -> bool { + match self { + Self::HttpForbidden + | Self::HttpNotFound + | Self::HttpClientError + | Self::ProcessFailed => false, + Self::HttpTooManyRequests + | Self::HttpServerError + | Self::HttpUnknown + | Self::Dns + | Self::Connect + | Self::Timeout + | Self::Tls + | Self::Interrupted + | Self::Network + | Self::Unknown => true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CurlFailure { + pub(crate) url: String, + pub(crate) destination: Option, + pub(crate) exit_code: Option, + pub(crate) stderr: String, + pub(crate) http_status: Option, + pub(crate) kind: CurlFailureKind, +} + +impl CurlFailure { + fn from_output(url: &str, destination: Option<&Path>, output: &Output) -> Self { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let exit_code = output.status.code(); + let http_status = parse_http_status(&stderr); + let kind = classify_failure(exit_code, http_status); + Self { + url: url.to_string(), + destination: destination.map(Path::to_path_buf), + exit_code, + stderr, + http_status, + kind, + } + } + + fn process_failed( + url: &str, + destination: Option<&Path>, + curl_command: &Path, + error: std::io::Error, + ) -> Self { + Self { + url: url.to_string(), + destination: destination.map(Path::to_path_buf), + exit_code: None, + stderr: format!("启动 curl 命令失败 {}：{error}", curl_command.display()), + http_status: None, + kind: CurlFailureKind::ProcessFailed, + } + } + + pub(crate) fn retryable(&self) -> bool { + self.kind.is_retryable() + } + + fn format_compact(&self) -> String { + let mut parts = vec![ + format!("kind={}", self.kind.as_str()), + format!("retryable={}", self.retryable()), + format!("url={}", self.url), + ]; + if let Some(status) = self.http_status { + parts.push(format!("http_status={status}")); + } + if let Some(code) = self.exit_code { + parts.push(format!("curl_exit={code}")); + } + if let Some(destination) = self.destination.as_ref() { + parts.push(format!("destination={}", destination.display())); + } + if !self.stderr.is_empty() { + parts.push(format!("stderr={}", self.stderr)); + } + parts.join(" ") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CurlAttemptFailure { + pub(crate) attempt: usize, + pub(crate) attempts: usize, + pub(crate) failure: CurlFailure, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CurlRetryError { + pub(crate) attempts: Vec, +} + +impl CurlRetryError { + pub(crate) fn final_failure(&self) -> Option<&CurlFailure> { + self.attempts.last().map(|attempt| &attempt.failure) + } + + pub(crate) fn attempt_count(&self) -> usize { + self.attempts.len() + } + + pub(crate) fn final_kind_label(&self) -> Option<&'static str> { + self.final_failure().map(|failure| failure.kind.as_str()) + } + + pub(crate) fn final_http_status(&self) -> Option { + self.final_failure().and_then(|failure| failure.http_status) + } + + pub(crate) fn final_retryable(&self) -> Option { + self.final_failure().map(CurlFailure::retryable) + } + + pub(crate) fn summary(&self) -> String { + let Some(final_failure) = self.final_failure() else { + return "curl 未执行就已失败".to_string(); + }; + let policy = if final_failure.retryable() { + "重试次数已耗尽" + } else { + "不可重试，已提前停止" + }; + let attempts = self + .attempts + .iter() + .map(|attempt| { + format!( + "第 {}/{} 次尝试失败：{}", + attempt.attempt, + attempt.attempts, + attempt.failure.format_compact() + ) + }) + .collect::>() + .join("；"); + format!("{policy}；{attempts}") + } +} + +impl std::fmt::Display for CurlRetryError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.summary()) + } +} + +impl std::error::Error for CurlRetryError {} + +pub(crate) fn run_curl_with_retry( + curl_command: &Path, + url: &str, + destination: Option<&Path>, + attempts: usize, + mut build_command: impl FnMut() -> Command, +) -> Result { + let attempts = attempts.max(1); + let mut failures = Vec::new(); + + for attempt in 1..=attempts { + let failure = match build_command().output() { + Ok(output) if output.status.success() => return Ok(output), + Ok(output) => CurlFailure::from_output(url, destination, &output), + Err(error) => CurlFailure::process_failed(url, destination, curl_command, error), + }; + let retryable = failure.retryable(); + failures.push(CurlAttemptFailure { + attempt, + attempts, + failure, + }); + if !retryable { + break; + } + } + + Err(CurlRetryError { attempts: failures }) +} + +fn classify_failure(exit_code: Option, http_status: Option) -> CurlFailureKind { + if exit_code == Some(22) { + return match http_status { + Some(403) => CurlFailureKind::HttpForbidden, + Some(404) => CurlFailureKind::HttpNotFound, + Some(408 | 425 | 429) => CurlFailureKind::HttpTooManyRequests, + Some(400..=499) => CurlFailureKind::HttpClientError, + Some(500..=599) => CurlFailureKind::HttpServerError, + Some(_) => CurlFailureKind::HttpUnknown, + None => CurlFailureKind::HttpUnknown, + }; + } + + match exit_code { + Some(5 | 6) => CurlFailureKind::Dns, + Some(7) => CurlFailureKind::Connect, + Some(18) => CurlFailureKind::Interrupted, + Some(28) => CurlFailureKind::Timeout, + Some(35) => CurlFailureKind::Tls, + Some(52 | 56) => CurlFailureKind::Network, + Some(_) => CurlFailureKind::Unknown, + None => CurlFailureKind::ProcessFailed, + } +} + +fn parse_http_status(stderr: &str) -> Option { + let bytes = stderr.as_bytes(); + let mut index = 0usize; + while index < bytes.len() { + if !bytes[index].is_ascii_digit() { + index += 1; + continue; + } + let start = index; + while index < bytes.len() && bytes[index].is_ascii_digit() { + index += 1; + } + if index - start == 3 { + let status = stderr[start..index].parse::().ok()?; + if (100..=599).contains(&status) { + return Some(status); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::process::ExitStatusExt; + + #[test] + fn classifies_http_status_codes_from_curl_stderr() { + let status = std::process::ExitStatus::from_raw(22 << 8); + let output = Output { + status, + stdout: Vec::new(), + stderr: b"curl: (22) The requested URL returned error: 404".to_vec(), + }; + let failure = CurlFailure::from_output("https://example.test/a", None, &output); + + assert_eq!(failure.http_status, Some(404)); + assert_eq!(failure.kind, CurlFailureKind::HttpNotFound); + assert!(!failure.retryable()); + } + + #[test] + fn treats_5xx_as_retryable() { + let status = std::process::ExitStatus::from_raw(22 << 8); + let output = Output { + status, + stdout: Vec::new(), + stderr: b"curl: (22) The requested URL returned error: 503".to_vec(), + }; + let failure = CurlFailure::from_output("https://example.test/a", None, &output); + + assert_eq!(failure.http_status, Some(503)); + assert_eq!(failure.kind, CurlFailureKind::HttpServerError); + assert!(failure.retryable()); + } +} diff --git a/infrastructure/src/lib.rs b/infrastructure/src/lib.rs index 1e598ec..de28cb3 100644 --- a/infrastructure/src/lib.rs +++ b/infrastructure/src/lib.rs @@ -11,6 +11,7 @@ #![warn(clippy::all)] pub mod cas; +mod curl_transfer; pub mod import; pub mod official_download; pub mod official_game_main_config; diff --git a/infrastructure/src/official_download.rs b/infrastructure/src/official_download.rs index ee8da75..5a2fef7 100644 --- a/infrastructure/src/official_download.rs +++ b/infrastructure/src/official_download.rs @@ -1,5 +1,6 @@ //! Official JP resource download execution. +use crate::curl_transfer::{run_curl_with_retry, CurlRetryError}; use crate::official_pull::OfficialResourcePullPlan; use crate::path_security::{ ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, @@ -15,9 +16,12 @@ use std::fs::{self, File}; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json"; +const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json"; const DOWNLOAD_MANIFEST_VERSION: u32 = 1; +const DOWNLOAD_QUARANTINE_VERSION: u32 = 1; const DEFAULT_RETRY_ATTEMPTS: usize = 3; /// Outcome for one official resource pull item. @@ -49,6 +53,8 @@ pub enum OfficialResourcePullProgressKind { Started, /// A URL finished as skipped, resumed, or downloaded. Finished, + /// A URL failed after retry classification and was quarantined for this run. + Failed, } impl OfficialResourcePullProgressKind { @@ -57,6 +63,7 @@ impl OfficialResourcePullProgressKind { match self { Self::Started => "started", Self::Finished => "finished", + Self::Failed => "failed", } } } @@ -78,6 +85,16 @@ pub struct OfficialResourcePullProgress { pub bytes: Option, /// Bytes transferred during this run, when `kind` is `Finished`. pub transferred_bytes: Option, + /// Stable failure kind label, when `kind` is `Failed`. + pub failure_kind: Option, + /// HTTP status parsed from curl stderr, when available. + pub failure_http_status: Option, + /// Whether retry policy considered the final failure retryable. + pub failure_retryable: Option, + /// Number of curl attempts that were executed before this failure. + pub failure_attempts: Option, + /// Whether the URL was written to the local quarantine manifest. + pub quarantined: bool, } impl OfficialResourcePullProgress { @@ -90,6 +107,11 @@ impl OfficialResourcePullProgress { status: None, bytes: None, transferred_bytes: None, + failure_kind: None, + failure_http_status: None, + failure_retryable: None, + failure_attempts: None, + quarantined: false, } } @@ -109,6 +131,28 @@ impl OfficialResourcePullProgress { status: Some(status), bytes: Some(bytes), transferred_bytes: Some(transferred_bytes), + failure_kind: None, + failure_http_status: None, + failure_retryable: None, + failure_attempts: None, + quarantined: false, + } + } + + fn failed(index: usize, total: usize, url: String, error: &PullOneError) -> Self { + Self { + kind: OfficialResourcePullProgressKind::Failed, + index, + total, + url, + status: None, + bytes: None, + transferred_bytes: None, + failure_kind: error.failure_kind().map(ToOwned::to_owned), + failure_http_status: error.http_status(), + failure_retryable: error.retryable(), + failure_attempts: error.attempts(), + quarantined: true, } } } @@ -434,6 +478,11 @@ impl OfficialResourcePullService { self.output_root.join(DOWNLOAD_MANIFEST_FILE) } + /// Returns the local quarantine path used to diagnose failed downloads. + pub fn download_quarantine_path(&self) -> PathBuf { + self.output_root.join(DOWNLOAD_QUARANTINE_FILE) + } + /// Executes the plan and downloads every official URL to disk. pub fn pull( &self, @@ -504,9 +553,27 @@ impl OfficialResourcePullService { self.validated_existing_file(&url, &destination, &manifest)? }; let result = if let Some(result) = result { + self.clear_quarantine_entry(&url)?; result } else { - let result = self.pull_one(&url, &destination)?; + let result = match self.pull_one(&url, &destination) { + Ok(result) => result, + Err(error) => { + self.record_quarantine_entry(&url, &destination, &error)?; + progress(OfficialResourcePullProgress::failed( + index, + total, + url.clone(), + &error, + )); + return Err(format!( + "官方资源下载中断：URL 已进入 quarantine 并跳过本轮，不会发布不完整资源；url={url} quarantine={}；{}", + self.download_quarantine_path().display(), + error.message + )); + } + }; + self.clear_quarantine_entry(&url)?; self.record_download_manifest_entry(&mut manifest, &url, &destination)?; self.write_download_manifest(&manifest)?; result @@ -642,49 +709,32 @@ impl OfficialResourcePullService { return Err(format!("拒绝拉取非官方 URL：{url}")); } - let attempts = self.retry_attempts.max(1); - let mut last_error = None; - for attempt in 1..=attempts { - match self.fetch_bytes_once(url) { - Ok(bytes) => return Ok(bytes), - Err(error) => { - last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}")); - } - } - } - - Err(last_error.unwrap_or_else(|| format!("curl 未执行就已失败：{url}"))) - } - - fn fetch_bytes_once(&self, url: &str) -> Result, String> { - let output = Command::new(&self.curl_command) - .arg("--fail") - .arg("--location") - .arg("--silent") - .arg("--show-error") - .arg("--url") - .arg(url) - .output() - .map_err(|error| { - format!( - "启动 curl 命令失败 {}：{error}", - self.curl_command.display() - ) - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("curl 拉取失败 {url}：{}", stderr.trim())); - } + let output = + run_curl_with_retry(&self.curl_command, url, None, self.retry_attempts, || { + let mut command = Command::new(&self.curl_command); + command + .arg("--fail") + .arg("--location") + .arg("--silent") + .arg("--show-error") + .arg("--url") + .arg(url); + command + }) + .map_err(|error| format!("curl 拉取失败 {url}：{error}"))?; Ok(output.stdout) } - fn pull_one(&self, url: &str, destination: &Path) -> Result { - ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?; + fn pull_one(&self, url: &str, destination: &Path) -> Result { + ensure_safe_file_target(&self.output_root, destination, "下载目标文件") + .map_err(PullOneError::plain)?; let partial = partial_path_for(destination); - ensure_safe_file_target(&self.output_root, &partial, "临时下载文件")?; - let partial_len_before = file_len_if_exists(&partial)?.unwrap_or_default(); + ensure_safe_file_target(&self.output_root, &partial, "临时下载文件") + .map_err(PullOneError::plain)?; + let partial_len_before = file_len_if_exists(&partial) + .map_err(PullOneError::plain)? + .unwrap_or_default(); let resumed = partial_len_before > 0; let mut status = if resumed { OfficialResourcePullStatus::Resumed @@ -697,43 +747,60 @@ impl OfficialResourcePullService { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { - format!("续传失败 {url}：{error}；清理后重新下载也失败：{retry_error}") + PullOneError::from_retry( + format!("续传失败 {url}：{error}；清理后重新下载也失败：{retry_error}"), + retry_error, + ) })?; status = OfficialResourcePullStatus::Downloaded; } else if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { - format!("续传后的 ZIP 结构校验失败 {url}：{error}；清理后重新下载也失败：{retry_error}") + PullOneError::from_retry( + format!("续传后的 ZIP 结构校验失败 {url}：{error}；清理后重新下载也失败：{retry_error}"), + retry_error, + ) })?; status = OfficialResourcePullStatus::Downloaded; } } else { - self.download_one(url, &partial, false)?; + self.download_one(url, &partial, false).map_err(|error| { + PullOneError::from_retry(format!("下载失败 {url}：{error}"), error) + })?; } if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); - return Err(error); + return Err(PullOneError::plain(error)); } - ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?; - if file_len_if_exists(destination)?.is_some() { - fs::remove_file(destination).map_err(|error| { - format!("替换已有目标文件失败 {}：{error}", destination.display()) - })?; + ensure_safe_file_target(&self.output_root, destination, "下载目标文件") + .map_err(PullOneError::plain)?; + if file_len_if_exists(destination) + .map_err(PullOneError::plain)? + .is_some() + { + fs::remove_file(destination) + .map_err(|error| format!("替换已有目标文件失败 {}：{error}", destination.display())) + .map_err(PullOneError::plain)?; } - ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?; + ensure_safe_file_target(&self.output_root, destination, "下载目标文件") + .map_err(PullOneError::plain)?; - fs::rename(&partial, destination).map_err(|error| { - format!( - "移动临时下载文件失败 {} -> {}：{error}", - partial.display(), - destination.display() - ) - })?; + fs::rename(&partial, destination) + .map_err(|error| { + format!( + "移动临时下载文件失败 {} -> {}：{error}", + partial.display(), + destination.display() + ) + }) + .map_err(PullOneError::plain)?; - let bytes = file_len_if_exists(destination)? - .ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))?; + let bytes = file_len_if_exists(destination) + .map_err(PullOneError::plain)? + .ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display())) + .map_err(PullOneError::plain)?; Ok(PullOneResult { bytes, @@ -746,53 +813,34 @@ impl OfficialResourcePullService { }) } - fn download_one(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> { - let attempts = self.retry_attempts.max(1); - let mut last_error = None; - for attempt in 1..=attempts { - match self.download_one_once(url, destination, resume) { - Ok(()) => return Ok(()), - Err(error) => { - last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}")); + fn download_one( + &self, + url: &str, + destination: &Path, + resume: bool, + ) -> Result<(), CurlRetryError> { + run_curl_with_retry( + &self.curl_command, + url, + Some(destination), + self.retry_attempts, + || { + let mut command = Command::new(&self.curl_command); + command + .arg("--fail") + .arg("--location") + .arg("--silent") + .arg("--show-error") + .arg("--output") + .arg(destination); + if resume { + command.arg("--continue-at").arg("-"); } - } - } - - Err(last_error - .unwrap_or_else(|| format!("curl 未执行就已失败：{url} -> {}", destination.display()))) - } - - fn download_one_once(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> { - let mut command = Command::new(&self.curl_command); - command - .arg("--fail") - .arg("--location") - .arg("--silent") - .arg("--show-error") - .arg("--output") - .arg(destination); - if resume { - command.arg("--continue-at").arg("-"); - } - command.arg("--url").arg(url); - - let output = command.output().map_err(|error| { - format!( - "启动 curl 命令失败 {}：{error}", - self.curl_command.display() - ) - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!( - "curl 下载失败 {url} -> {}：{}", - destination.display(), - stderr.trim() - )); - } - - Ok(()) + command.arg("--url").arg(url); + command + }, + ) + .map(|_| ()) } fn verify_ready_official_hashes( @@ -935,6 +983,82 @@ impl OfficialResourcePullService { Ok(()) } + fn read_download_quarantine(&self) -> Result { + self.ensure_output_root_safe()?; + let path = self.download_quarantine_path(); + ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; + let Some(bytes) = read_file_no_symlink(&path, "下载 quarantine")? else { + return Ok(OfficialDownloadQuarantineManifest::default()); + }; + + let manifest: OfficialDownloadQuarantineManifest = serde_json::from_slice(&bytes) + .map_err(|error| format!("解析下载 quarantine 失败 {}：{error}", path.display()))?; + + if manifest.version != DOWNLOAD_QUARANTINE_VERSION { + return Err(format!( + "不支持的下载 quarantine 版本 {}，文件 {}", + manifest.version, + path.display() + )); + } + + Ok(manifest) + } + + fn write_download_quarantine( + &self, + manifest: &OfficialDownloadQuarantineManifest, + ) -> Result<(), String> { + self.ensure_output_root_ready()?; + let path = self.download_quarantine_path(); + if let Some(parent) = path.parent() { + ensure_safe_directory_path(parent, "下载 quarantine 目录")?; + fs::create_dir_all(parent).map_err(|error| { + format!("创建下载 quarantine 目录失败 {}：{error}", parent.display()) + })?; + ensure_safe_directory_path(parent, "下载 quarantine 目录")?; + } + ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; + + let bytes = serde_json::to_vec_pretty(manifest) + .map_err(|error| format!("序列化下载 quarantine 失败 {}：{error}", path.display()))?; + write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 quarantine")?; + + Ok(()) + } + + fn record_quarantine_entry( + &self, + url: &str, + destination: &Path, + error: &PullOneError, + ) -> Result<(), String> { + let mut manifest = self.read_download_quarantine()?; + manifest.entries.insert( + url.to_string(), + OfficialDownloadQuarantineEntry { + url: url.to_string(), + destination: self.relative_destination(destination)?, + failed_at_unix_seconds: unix_seconds_now(), + attempts: error.attempts().unwrap_or(0), + failure_kind: error.failure_kind().map(ToOwned::to_owned), + http_status: error.http_status(), + retryable: error.retryable(), + skipped_this_run: true, + last_error: error.message.clone(), + }, + ); + self.write_download_quarantine(&manifest) + } + + fn clear_quarantine_entry(&self, url: &str) -> Result<(), String> { + let mut manifest = self.read_download_quarantine()?; + if manifest.entries.remove(url).is_none() { + return Ok(()); + } + self.write_download_quarantine(&manifest) + } + fn validated_existing_file( &self, url: &str, @@ -1341,10 +1465,44 @@ struct OfficialDownloadManifestEntry { blake3: String, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialDownloadQuarantineManifest { + #[serde(default = "default_download_quarantine_version")] + version: u32, + #[serde(default)] + entries: BTreeMap, +} + +impl Default for OfficialDownloadQuarantineManifest { + fn default() -> Self { + Self { + version: DOWNLOAD_QUARANTINE_VERSION, + entries: BTreeMap::new(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct OfficialDownloadQuarantineEntry { + url: String, + destination: String, + failed_at_unix_seconds: u64, + attempts: usize, + failure_kind: Option, + http_status: Option, + retryable: Option, + skipped_this_run: bool, + last_error: String, +} + fn default_download_manifest_version() -> u32 { DOWNLOAD_MANIFEST_VERSION } +fn default_download_quarantine_version() -> u32 { + DOWNLOAD_QUARANTINE_VERSION +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PullOneResult { bytes: u64, @@ -1352,6 +1510,58 @@ struct PullOneResult { status: OfficialResourcePullStatus, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct PullOneError { + message: String, + retry_error: Option, +} + +impl PullOneError { + fn plain(message: String) -> Self { + Self { + message, + retry_error: None, + } + } + + fn from_retry(message: String, retry_error: CurlRetryError) -> Self { + Self { + message, + retry_error: Some(retry_error), + } + } + + fn failure_kind(&self) -> Option<&'static str> { + self.retry_error + .as_ref() + .and_then(CurlRetryError::final_kind_label) + } + + fn http_status(&self) -> Option { + self.retry_error + .as_ref() + .and_then(CurlRetryError::final_http_status) + } + + fn retryable(&self) -> Option { + self.retry_error + .as_ref() + .and_then(CurlRetryError::final_retryable) + } + + fn attempts(&self) -> Option { + self.retry_error.as_ref().map(CurlRetryError::attempt_count) + } +} + +impl std::fmt::Display for PullOneError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for PullOneError {} + fn file_len_if_exists(path: &Path) -> Result, String> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { @@ -1370,6 +1580,13 @@ fn partial_path_for(destination: &Path) -> PathBuf { PathBuf::from(partial) } +fn unix_seconds_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + fn blake3_file_hex(path: &Path) -> Result { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { @@ -1769,6 +1986,35 @@ fi "# } + fn fake_http_error_curl_script(status: u16) -> String { + format!( + r#"#!/bin/sh +set -eu +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --url) + url="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +state="$0.state" +count=0 +if [ -f "$state" ]; then + count="$(cat "$state")" +fi +count=$((count + 1)) +printf '%s' "$count" > "$state" +printf 'curl: (22) The requested URL returned error: {status}\n' >&2 +exit 22 +"# + ) + } + fn write_shell_script(path: &Path, script: &str) { fs::write(path, script).unwrap(); #[cfg(unix)] @@ -1800,6 +2046,32 @@ fi write_shell_script(path, fake_bad_zip_curl_script()); } + fn write_fake_http_error_curl(path: &Path, status: u16) { + write_shell_script(path, &fake_http_error_curl_script(status)); + } + + fn one_url_plan(url: &str) -> OfficialResourcePullPlan { + OfficialResourcePullPlan { + discovery: YostarJpResourceDiscoveryPlan { + connection_group_name: "Prod-Audit".to_string(), + app_version: "1.70.0".to_string(), + bundle_version: None, + addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" + .to_string(), + endpoints: vec![YostarJpResourceEndpoint { + kind: YostarJpResourceEndpointKind::TableCatalog, + platform: None, + url: url.to_string(), + }], + }, + inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( + YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), + &[], + ), + platforms: Vec::new(), + } + } + fn one_file_zip(name: &[u8], data: &[u8]) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); @@ -1965,7 +2237,7 @@ fi symlink(&escape_file, partial_path_for(&destination)).unwrap(); let error = service.pull_one(url, &destination).unwrap_err(); - assert!(error.contains("symlink")); + assert!(error.to_string().contains("symlink")); assert_eq!(fs::read(&escape_file).unwrap(), b"outside"); } @@ -2470,6 +2742,82 @@ fi ); } + #[test] + fn does_not_retry_terminal_http_404_and_records_quarantine() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_http_error_curl(&curl_path, 404); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) + .with_retry_attempts(3); + let url = + "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Missing.bytes"; + let plan = one_url_plan(url); + let mut events = Vec::new(); + + let error = service + .pull_with_progress(&plan, |event| events.push(event)) + .unwrap_err(); + + assert!(error.contains("quarantine")); + assert!(error.contains("http_not_found")); + assert!(error.contains("retryable=false")); + assert_eq!( + fs::read_to_string(curl_path.with_extension("state")).unwrap(), + "1" + ); + let failed = events + .iter() + .find(|event| event.kind == OfficialResourcePullProgressKind::Failed) + .unwrap(); + assert_eq!(failed.failure_kind.as_deref(), Some("http_not_found")); + assert_eq!(failed.failure_http_status, Some(404)); + assert_eq!(failed.failure_retryable, Some(false)); + assert_eq!(failed.failure_attempts, Some(1)); + assert!(failed.quarantined); + + let quarantine = service.read_download_quarantine().unwrap(); + let entry = quarantine.entries.get(url).unwrap(); + assert_eq!(entry.http_status, Some(404)); + assert_eq!(entry.failure_kind.as_deref(), Some("http_not_found")); + assert_eq!(entry.retryable, Some(false)); + assert_eq!(entry.attempts, 1); + assert!(entry.skipped_this_run); + assert!(service.read_download_manifest().unwrap().entries.is_empty()); + } + + #[test] + fn retries_http_5xx_before_quarantine() { + let out_dir = TempDir::new().unwrap(); + let bin_dir = TempDir::new().unwrap(); + let curl_path = bin_dir.path().join("curl"); + write_fake_http_error_curl(&curl_path, 503); + + let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) + .with_retry_attempts(3); + let url = + "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Busy.bytes"; + let plan = one_url_plan(url); + + let error = service.pull(&plan).unwrap_err(); + + assert!(error.contains("quarantine")); + assert!(error.contains("http_server_error")); + assert!(error.contains("retryable=true")); + assert_eq!( + fs::read_to_string(curl_path.with_extension("state")).unwrap(), + "3" + ); + let quarantine = service.read_download_quarantine().unwrap(); + let entry = quarantine.entries.get(url).unwrap(); + assert_eq!(entry.http_status, Some(503)); + assert_eq!(entry.failure_kind.as_deref(), Some("http_server_error")); + assert_eq!(entry.retryable, Some(true)); + assert_eq!(entry.attempts, 3); + assert!(entry.last_error.contains("重试次数已耗尽")); + } + #[test] fn custom_platform_selection_is_supported() { let out_dir = TempDir::new().unwrap(); diff --git a/infrastructure/src/official_game_main_config.rs b/infrastructure/src/official_game_main_config.rs index 3e49384..da8a56e 100644 --- a/infrastructure/src/official_game_main_config.rs +++ b/infrastructure/src/official_game_main_config.rs @@ -4,6 +4,7 @@ //! artifacts into a temporary directory, extracts only the data needed to read //! `GameMainConfig`, and does not install or execute the official launcher. +use crate::curl_transfer::run_curl_with_retry; use crate::official_launcher::launcher_package_url; use crate::official_launcher::OfficialLauncherBootstrapService; use crate::official_launcher::{ @@ -115,44 +116,27 @@ impl OfficialGameMainConfigBootstrapService { } fn download_file(&self, url: &str, destination: &Path) -> Result<(), String> { - let attempts = DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS; - let mut last_error = None; - for attempt in 1..=attempts { - match self.download_file_once(url, destination) { - Ok(()) => return Ok(()), - Err(error) => { - last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}")); - } - } - } - - Err(last_error.unwrap_or_else(|| format!("curl 未执行就已失败：{url}"))) - } - - fn download_file_once(&self, url: &str, destination: &Path) -> Result<(), String> { - let output = Command::new(&self.curl_command) - .arg("--fail") - .arg("--location") - .arg("--silent") - .arg("--show-error") - .arg("--output") - .arg(destination) - .arg("--url") - .arg(url) - .output() - .map_err(|error| { - format!( - "启动 curl 命令失败 {}：{error}", - self.curl_command.display() - ) - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("curl 拉取失败 {url}：{}", stderr.trim())); - } - - Ok(()) + run_curl_with_retry( + &self.curl_command, + url, + Some(destination), + DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS, + || { + let mut command = Command::new(&self.curl_command); + command + .arg("--fail") + .arg("--location") + .arg("--silent") + .arg("--show-error") + .arg("--output") + .arg(destination) + .arg("--url") + .arg(url); + command + }, + ) + .map(|_| ()) + .map_err(|error| format!("curl 拉取失败 {url}：{error}")) } fn download_file_with_fallback( @@ -168,7 +152,7 @@ impl OfficialGameMainConfigBootstrapService { let backup_url = launcher_package_url(backup_cdn_root, relative_path)?; self.download_file(&backup_url, destination).map_err(|backup_error| { format!( - "下载官方游戏资源失败：主地址失败（{primary_error}），备用地址也失败（{backup_error}）" + "下载官方游戏资源失败：主 CDN 失败后已切换官方备用 CDN；主地址失败（{primary_error}），备用地址也失败（{backup_error}）" ) }) } @@ -391,3 +375,85 @@ fn find_resources_assets(root: &Path) -> Option { } None } + +#[cfg(test)] +mod tests { + use super::*; + + fn write_shell_script(path: &Path, script: &str) { + fs::write(path, script).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = fs::metadata(path).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).unwrap(); + } + } + + fn fake_primary_404_backup_ok_curl_script() -> &'static str { + r#"#!/bin/sh +set -eu +out="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + --url) + url="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +printf '%s\n' "$url" >> "$0.urls" +case "$url" in + https://launcher-pkg-ba-jp.yo-star.com/*) + printf 'curl: (22) The requested URL returned error: 404\n' >&2 + exit 22 + ;; + https://launcher-pkg-ba-jp-bk.yo-star.com/*) + mkdir -p "$(dirname "$out")" + printf '%s' 'backup-ok' > "$out" + ;; + *) + printf 'unexpected url: %s\n' "$url" >&2 + exit 22 + ;; +esac +"# + } + + #[test] + fn downloads_launcher_package_from_backup_after_primary_terminal_failure() { + let temp = TempDir::new().unwrap(); + let curl_path = temp.path().join("curl"); + write_shell_script(&curl_path, fake_primary_404_backup_ok_curl_script()); + let service = + OfficialGameMainConfigBootstrapService::with_commands("1.17.0", &curl_path, "unzip"); + let destination = temp.path().join("resources.assets"); + let primary_url = "https://launcher-pkg-ba-jp.yo-star.com/prod/manifest/resources.assets"; + + service + .download_file_with_fallback( + primary_url, + "https://launcher-pkg-ba-jp-bk.yo-star.com", + "prod/manifest/resources.assets", + &destination, + ) + .unwrap(); + + assert_eq!(fs::read_to_string(&destination).unwrap(), "backup-ok"); + let urls = fs::read_to_string(curl_path.with_extension("urls")).unwrap(); + assert!( + urls.contains("https://launcher-pkg-ba-jp.yo-star.com/prod/manifest/resources.assets") + ); + assert!(urls + .contains("https://launcher-pkg-ba-jp-bk.yo-star.com/prod/manifest/resources.assets")); + } +} diff --git a/infrastructure/src/official_launcher.rs b/infrastructure/src/official_launcher.rs index eb39063..d7df5d8 100644 --- a/infrastructure/src/official_launcher.rs +++ b/infrastructure/src/official_launcher.rs @@ -5,9 +5,11 @@ //! discover the latest Windows client package and manifest, while resource //! updates still come from the game client's server-info flow. +use crate::curl_transfer::run_curl_with_retry as run_curl_command_with_retry; use bat_adapters::official::launcher::{YostarJpLauncherManifestFile, YOSTAR_JP_GAME_TAG}; use md5::{Digest, Md5}; use serde::Deserialize; +use std::path::Path; use std::process::{Command, Output}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -165,8 +167,8 @@ impl OfficialLauncherBootstrapService { return Err(format!("拒绝拉取非官方启动器 manifest URL：{manifest_url}")); } - let output = self.run_curl_with_retry( - || { + let output = self + .run_curl_with_retry(manifest_url, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") @@ -176,15 +178,8 @@ impl OfficialLauncherBootstrapService { .arg("--url") .arg(manifest_url); command - }, - |output| { - let stderr = String::from_utf8_lossy(&output.stderr); - format!( - "curl 拉取启动器 manifest 失败 {manifest_url}：{}", - stderr.trim() - ) - }, - )?; + }) + .map_err(|error| format!("curl 拉取启动器 manifest 失败 {manifest_url}：{error}"))?; let manifest: YostarJpLauncherRemoteManifest = serde_json::from_slice(&output.stdout) .map_err(|error| format!("解析官方启动器 manifest 失败：{error}"))?; @@ -222,8 +217,8 @@ impl OfficialLauncherBootstrapService { let url = launcher_api_url(path)?; let authorization = launcher_authorization_header(&self.launcher_version, "", None)?; - let output = self.run_curl_with_retry( - || { + let output = self + .run_curl_with_retry(&url, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") @@ -237,12 +232,8 @@ impl OfficialLauncherBootstrapService { .arg("--url") .arg(&url); command - }, - |output| { - let stderr = String::from_utf8_lossy(&output.stderr); - format!("curl 拉取失败 {url}：{}", stderr.trim()) - }, - )?; + }) + .map_err(|error| format!("curl 拉取失败 {url}：{error}"))?; let envelope: LauncherEnvelope = serde_json::from_slice(&output.stdout) .map_err(|error| format!("解析官方启动器 API 响应失败：{error}"))?; @@ -264,30 +255,17 @@ impl OfficialLauncherBootstrapService { fn run_curl_with_retry( &self, + url: &str, mut build_command: impl FnMut() -> Command, - failure_message: impl Fn(&Output) -> String, ) -> Result { - let attempts = self.retry_attempts.max(1); - let mut last_error = None; - for attempt in 1..=attempts { - match build_command().output() { - Ok(output) if output.status.success() => return Ok(output), - Ok(output) => { - last_error = Some(format!( - "第 {attempt}/{attempts} 次尝试失败：{}", - failure_message(&output) - )); - } - Err(error) => { - last_error = Some(format!( - "第 {attempt}/{attempts} 次尝试失败：启动 curl 命令失败 {}：{error}", - self.curl_command - )); - } - } - } - - Err(last_error.unwrap_or_else(|| format!("curl 命令未执行：{}", self.curl_command))) + run_curl_command_with_retry( + Path::new(&self.curl_command), + url, + None, + self.retry_attempts, + &mut build_command, + ) + .map_err(|error| error.to_string()) } } diff --git a/infrastructure/src/official_update.rs b/infrastructure/src/official_update.rs index 274e612..10d090c 100644 --- a/infrastructure/src/official_update.rs +++ b/infrastructure/src/official_update.rs @@ -474,6 +474,16 @@ pub struct OfficialUpdateProgress { pub download_bytes: Option, /// Bytes transferred in this run for the URL when known. pub download_transferred_bytes: Option, + /// Stable failure kind when a URL failed. + pub download_failure_kind: Option, + /// HTTP status parsed from curl stderr when a URL failed. + pub download_failure_http_status: Option, + /// Whether the final failure was retryable. + pub download_failure_retryable: Option, + /// Number of transfer attempts executed before failure. + pub download_failure_attempts: Option, + /// Whether the URL was recorded in the quarantine manifest. + pub download_quarantined: Option, } impl OfficialUpdateProgress { @@ -488,6 +498,11 @@ impl OfficialUpdateProgress { download_status: None, download_bytes: None, download_transferred_bytes: None, + download_failure_kind: None, + download_failure_http_status: None, + download_failure_retryable: None, + download_failure_attempts: None, + download_quarantined: None, } } @@ -495,9 +510,21 @@ impl OfficialUpdateProgress { self.download_index = Some(event.index); self.download_total = Some(event.total); self.download_url = Some(event.url.clone()); - self.download_status = event.status.map(|status| status.as_str().to_string()); + self.download_status = event + .status + .map(|status| status.as_str().to_string()) + .or_else(|| { + (event.kind == OfficialResourcePullProgressKind::Failed) + .then(|| "failed".to_string()) + }); self.download_bytes = event.bytes; self.download_transferred_bytes = event.transferred_bytes; + self.download_failure_kind = event.failure_kind.clone(); + self.download_failure_http_status = event.failure_http_status; + self.download_failure_retryable = event.failure_retryable; + self.download_failure_attempts = event.failure_attempts; + self.download_quarantined = + (event.kind == OfficialResourcePullProgressKind::Failed).then_some(event.quarantined); self } } @@ -1391,6 +1418,31 @@ fn progress_from_pull_event(event: OfficialResourcePullProgress) -> OfficialUpda ) .with_download_progress(&event) } + OfficialResourcePullProgressKind::Failed => OfficialUpdateProgress::new( + "download", + format!( + "下载中断：总体 {}/{} ({:.1}%)；失败类型={} HTTP={} 可重试={} 尝试次数={} quarantine={}；本轮跳过该 URL 且不会发布不完整资源 URL={}", + event.index, + event.total, + download_progress_percent(event.index, event.total), + event.failure_kind.as_deref().unwrap_or("unknown"), + event + .failure_http_status + .map(|status| status.to_string()) + .unwrap_or_else(|| "none".to_string()), + event + .failure_retryable + .map(|retryable| retryable.to_string()) + .unwrap_or_else(|| "unknown".to_string()), + event + .failure_attempts + .map(|attempts| attempts.to_string()) + .unwrap_or_else(|| "0".to_string()), + event.quarantined, + event.url + ), + ) + .with_download_progress(&event), } }