From 3e9bb20d791633466e663b00a8c5fc3489b14fe9 Mon Sep 17 00:00:00 2001 From: Yuyi-Oak <1722157266@qq.com> Date: Sat, 25 Jul 2026 20:51:01 +0800 Subject: [PATCH] =?UTF-8?q?feat(assetbundle):=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E5=AE=98=E6=96=B9=E8=B5=84=E6=BA=90=E8=A7=A3=E6=9E=90=E4=B8=8E?= =?UTF-8?q?=E5=8F=8C=E7=9B=AE=E5=BD=95=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 +- CURRENT_STATUS.md | 98 +- Cargo.lock | 20 +- Cargo.toml | 2 +- DOCS_INDEX.md | 10 +- PROJECT_PLAN.md | 78 +- README.md | 21 +- adapters/Cargo.toml | 3 +- adapters/src/manifest/addressables.rs | 380 ++++-- adapters/src/official/game_main_config.rs | 5 +- adapters/src/unity.rs | 7 +- adapters/src/unity/adapter.rs | 96 +- adapters/src/unity/serialized_file.rs | 526 +-------- adapters/src/unity/unity_2021_3.rs | 475 +------- adapters/tests/golden/real_addressables.json | 4 + adapters/tests/real_unityfs_local.rs | 2 + crates/bat-assetbundle/Cargo.toml | 5 +- crates/bat-assetbundle/src/error.rs | 40 +- crates/bat-assetbundle/src/lib.rs | 11 + crates/bat-assetbundle/src/parser.rs | 1025 ++++++++++++++++- crates/bat-assetbundle/src/serialized.rs | 878 ++++++++++++++ crates/bat-assetbundle/src/types.rs | 183 ++- .../bluearchive-toolkit-official-sync.service | 3 +- deployments/systemd/official-sync.env.example | 4 +- docs/architecture/assetbundle.md | 199 ++++ .../architecture/official-resource-backend.md | 32 +- docs/architecture/resource-release-layout.md | 311 +++++ docs/guides/deployment.md | 14 +- docs/guides/development.md | 17 +- docs/guides/official-resource-test-pull.md | 9 +- docs/reports/CURRENT_GAPS.md | 147 ++- infrastructure/Cargo.toml | 1 + infrastructure/src/bin/bat_official_sync.rs | 107 +- infrastructure/src/import.rs | 156 +++ infrastructure/src/lib.rs | 17 +- infrastructure/src/official_parse.rs | 950 +++++++++++++++ infrastructure/src/official_update.rs | 152 ++- 37 files changed, 4663 insertions(+), 1333 deletions(-) create mode 100644 crates/bat-assetbundle/src/serialized.rs create mode 100644 docs/architecture/assetbundle.md create mode 100644 docs/architecture/resource-release-layout.md create mode 100644 infrastructure/src/official_parse.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8839cb3..1d806e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,15 +9,19 @@ ### 新增 - Addressables catalog 提取 `m_Crc`（bundle IEEE CRC-32）：`ResourceEntry` 新增 `crc` 字段（compact/expanded 两种形态均解析），SQLite 持久化并对旧库幂等迁移补列；core 新增 `crc32_ieee` 与 `ResourceEntry::verify_downloaded_bytes`（按声明的 size/CRC 校验字节）（issue #2） - UnityFS 解析新增目录条目越界校验：directory 的 `offset+size` 必须落在解压数据区内，截断/损坏 bundle 的越界目录条目不再被静默接受（issue #3） -- 官方资源下载回归顺序执行：manifest/quarantine 簿记与 seed `.hash` 校验保持串行，`fail-fast` 与「不发布不完整资源」不变量不变（issue #17） +- 官方资源下载回归顺序执行：manifest/quarantine 簿记与 seed `.hash` 校验保持串行，`fail-fast` 与「不发布不完整资源」不变量不变（issue #17，按 wontfix 关闭多线程下载目标） +- 新增 `cmd/bat-api` 资源分发 HTTP 服务（issue #19 / G-009 资源面）：经 `bat.sock` RPC（`daemon.status` → `daemon.doctor` → catalog/manifest）发现已发布 release，按官方 CDN host/path 只读提供资源；管理面 `/healthz` `/v1/release` `/v1/resources`；可选 server-info 仅改写 Addressables root；`.env` 配置监听端口/RPC socket/预留数据库键。资源自动拉取仍由 Rust `bat` 负责 +- 官方同步新增 `official-parse-cache.json`：校验发布后从下载 manifest 覆盖直接 UnityFS bundle、zip 内 UnityFS 条目和非候选资源记录；URL、相对路径、size 和 BLAKE3 未变化时跳过重复解析 +- 官方原版资源和汉化产物目录分离：`BAT_OUTPUT`/`--output` 默认 `./bat-resources`，`BAT_LOCALIZED_OUTPUT`/`--localized-output` 默认 `./bat-localized`；同步报告新增 `localized_release_status=not_localized`，后续 Patch 发布完成后才切换为 `localized` ### 修复 - 官方下载失败重试之间加入指数退避（网络类失败 200ms→400ms→800ms…，上限 5s） ### 计划 -- [ ] 实现 `bat-api`：仿 BlueArchive 官方 API 的 Go HTTP 服务（含鉴权/签名验签，issue #19） +- [ ] `bat-api` 后续：launcher 链（若需要）、API 持久化层接入预留 database/redis 配置 - [ ] 官方同步结果接入 CAS + ResourceRepository 的用户级工作流 - [ ] 官方下载/导入路径接入 CRC/size 校验（复用 `verify_downloaded_bytes`） +- [ ] 汉化 Patch 发布：维护 `localized-output/current`、`localized-version-state`、`localized` 状态切换和回滚 - [ ] 实现翻译系统 - [ ] 实现 Patch 引擎 - [ ] 实现 Web 管理后台 diff --git a/CURRENT_STATUS.md b/CURRENT_STATUS.md index 4810c41..d33d2cd 100644 --- a/CURRENT_STATUS.md +++ b/CURRENT_STATUS.md @@ -1,16 +1,17 @@ # BlueArchiveToolkit 当前工作区状态 -- **更新时间**：2026-07-20 +- **更新时间**：2026-07-24 - **状态来源**：本地工作区盘点、代码验证和最新提交 - **状态分支**：`experiment` - **最新已推送功能提交**：以当前 `git log --oneline -1` 为准 - **权威计划**：`PROJECT_PLAN.md` +- **Go 进度权威**：`docs/reports/GO_STATUS.md` --- ## 1. 总体判断 -当前项目处于 **稳定基线完成、CAS V1 已落地、Rust 官方资源同步链路已具备可持续生产运行形态、Go `bat-api` 已有 Rust daemon RPC client 但 CLI/API/Web 仍未形成产品入口** 阶段。 +当前项目处于 **稳定基线完成、CAS V1 已落地、Rust 官方资源同步链路已具备可持续生产运行形态、Go 侧以 `bat-api` 资源分发 MVP + `backendrpc` 为正式服务入口（同步/运维命令行仍为近乎全自动的 Rust `bat`）** 阶段。 Rust 侧官方日服资源链路已经从实验验证推进到正式入口： @@ -23,13 +24,13 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 7. 支持 curl 传输层本地代理：默认自动检测 `HTTPS_PROXY` / `ALL_PROXY` / `HTTP_PROXY` 及小写环境变量（带凭据的代理推荐用环境变量配置），也可用 `--proxy ` 显式指定或 `--no-proxy` 强制直连；代理决策会写入 progress log、daemon log 和 `bat doctor` 诊断输出。代理凭据不落世界可读位置：日志/`status` 脱敏，传给 curl 经 `ALL_PROXY` 环境变量而非 argv，`--daemon` 下经环境变量下传后台子进程、不进子进程 argv 或 `bat-status.json`，复用凭据存于 `bat-proxy.secret`（`0600`）且 `clean-stable` 会清除。 8. `bat --watch` 可常驻运行，`bat --daemon` 可后台运行并用 `bat status` / `bat stop` / `bat restart` / `bat reload` / `bat logs` 管理；daemon 使用 `bat.sock` Unix socket JSON-RPC 作为 live 控制通道，PID/状态/日志文件作为快照和 fallback，`bat-events.jsonl` 记录带轮转的结构化事件日志，`bat-control.lock` 串行化控制命令；正常检查默认每 1 小时一次；远端和本地一致时默认静默，失败后默认 60 秒快速重试；CLI 默认向 stdout 输出人类可读摘要，向 stderr 输出 ASCII banner、progress log、失败分类和 quarantine 状态，需要机器输出时使用 `--json --no-progress`。 9. 远端 snapshot 未变化但输出目录为空时，会按首次运行执行全量拉取；官方 seed `.hash` 校验失败时会清理对应 manifest 条目，避免失败产物被后续本地 audit 误判为可复用。 -10. 默认资源目录是 `./bat-resources`，默认后台状态目录是 `/tmp/bat-pid`；资源目录是发布根目录，包含 `current` symlink、`versions/` 和 `.staging/`，非 dry-run 会先写 staging，校验完成后发布 versioned 目录并原子切换 `current`；如果上一轮同一 app version、bundle version 和 Addressables root 的 staging 失败但目录仍安全存在，下一轮会复用该 staging 并按 manifest 逐文件校验/补下载；后台状态目录包含 `bat.sock`、`bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl`、任务历史 `bat-tasks.json` 和短生命周期 `bat-control.lock`；非 dry-run 使用 `--output/.official-sync.lock` 防止并发写同一资源目录，live daemon 会阻止前台写命令直接修改它正在管理的同一目录。 +10. 默认官方原版资源目录是 `./bat-resources`，默认汉化产物目录是 `./bat-localized`，默认后台状态目录是 `/tmp/bat-pid`；官方资源目录是发布根目录，包含 `current` symlink、`versions/` 和 `.staging/`，非 dry-run 会先写 staging，校验完成后发布 versioned 目录并原子切换 `current`；如果上一轮同一 app version、bundle version 和 Addressables root 的 staging 失败但目录仍安全存在，下一轮会复用该 staging 并按 manifest 逐文件校验/补下载；后台状态目录包含 `bat.sock`、`bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl`、任务历史 `bat-tasks.json` 和短生命周期 `bat-control.lock`；非 dry-run 使用 `--output/.official-sync.lock` 防止并发写同一资源目录，live daemon 会阻止前台写命令直接修改它正在管理的同一目录。 11. 官方同步会拒绝危险输出目录、路径逃逸和现有 symlink 路径组件；下载目标、`.part`、manifest、snapshot、PID、status、log 和控制锁文件不会跟随 symlink，daemon 状态类文件默认以 `0600` 权限创建。 12. `/official-version-state.json` 会明确保存当前已完成版本、正在拉取版本、上一个可用版本和失败版本；同一 app version、bundle version 和 Addressables root 的失败只保留最新一条，同一版本开始重新拉取或后续发布成功时会清理对应失败记录；`bat status` 会显示最后成功时间、下次检查时间、最后错误摘要、当前阶段、当前下载 URL 进度、版本状态摘要、最近历史失败版本和原因、结构化日志路径和轮转日志路径，人类可读输出不会把完整版本状态 JSON 内联打印。 13. 资源导入链路已支持 CAS + `ResourceRepository` 索引写入，AssetBundle 导入会记录 UnityFS 摘要，TextAsset/Table/Media 会按类型分类；当前/上一个/结构变化 catalog、失败 staging 复用、403/404、hash mismatch、CRC 与 UnityFS 边界校验均有离线回归 fixture 或单测覆盖。 -14. `bat` 首次启动会在二进制所在目录释放 `.env` 配置模板（`0600`），之后每次启动自动加载（不覆盖已存在的环境变量），支持 `BAT_OUTPUT`/`BAT_STATE_DIR`/`BAT_AUTO_DISCOVER`/`BAT_WATCH`/`BAT_DAEMON`/`BAT_PROXY` 等键，实现编辑 `.env` 后无参启动；优先级为命令行参数 > 进程环境变量 > `.env` > 内置默认值，`BAT_SKIP_ENV_FILE=1` 可整体禁用；Redis 键为预留。daemon 任务历史持久化在 `/bat-tasks.json`（版本化、`0600` 原子写），重启后任务经 `task.*` 仍可查，中断任务标记 `task_interrupted`（`BAT-ERR-700005`）。 +14. 非 dry-run 官方同步在校验完成并发布后，会在 active release 下刷新 `official-parse-cache.json`；每轮根据 `official-download-manifest.json` 的 URL、相对路径、size 和 BLAKE3 判断是否复用旧解析结果，本地文件未变化时跳过重复解析。解析入口覆盖下载 manifest 中的全部官方资源：直接 UnityFS bundle、zip 内 UnityFS 条目会进入解析，其它 catalog/hash/media 文件记录为不支持而不是失败。官方同步报告默认 `localized_release_status=not_localized`，含义是原版资源已经发布、汉化资源未发布；后续 Patch 发布完成后才应切换为 `localized`，表示原版和汉化两套资源都可发布。`bat` 首次启动会在二进制所在目录释放 `.env` 配置模板（`0600`），之后每次启动自动加载（不覆盖已存在的环境变量），支持 `BAT_OUTPUT`/`BAT_LOCALIZED_OUTPUT`/`BAT_STATE_DIR`/`BAT_AUTO_DISCOVER`/`BAT_WATCH`/`BAT_DAEMON`/`BAT_PROXY` 等键，实现编辑 `.env` 后无参启动；优先级为命令行参数 > 进程环境变量 > `.env` > 内置默认值，`BAT_SKIP_ENV_FILE=1` 可整体禁用；Redis 键为预留。daemon 任务历史持久化在 `/bat-tasks.json`（版本化、`0600` 原子写），重启后任务经 `task.*` 仍可查，中断任务标记 `task_interrupted`（`BAT-ERR-700005`）。 -仍需明确：这不是完整产品完成。Go 产品入口、完整 AssetBundle 引擎解析、Patch、翻译系统、API Server 和 Web 仍是后续工作；真实官方网络全量拉取 smoke 已固化为可重复脚本和 runbook（G-018 已关闭），当前正在进行长期运行测试，运行报告将在后续提供；真实大文件产物与运行报告默认保存在 `/tmp` 隔离目录，不纳入 Git。 +仍需明确：这不是完整产品完成。完整 AssetBundle 引擎、Patch、翻译、Web、以及 `bat-api` 的服务器联调/可选业务扩展仍是后续工作；G-008（产品级 Go 同步 CLI）已决策关闭。真实官方网络全量拉取 smoke 已固化（G-018 已关闭）；真实大文件与运行报告默认在 `/tmp` 隔离目录，不纳入 Git。Go 细节见 `docs/reports/GO_STATUS.md`。 --- @@ -41,6 +42,7 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： - `docs/guides/official-resource-test-pull.md`：官方资源拉取与自动更新用户指南。 - `docs/guides/official-full-pull-smoke.md`：真实官方全量拉取 smoke runbook 和可重复命令。 - `docs/architecture/official-resource-backend.md`：官方资源后端设计和审核说明。 +- `docs/architecture/assetbundle.md`：解析补全路线图，覆盖 Addressables、UnityFS、Serialized 字段级解析、文本提取、CAS 接入和 Patch 发布前置。 - `docs/reports/CURRENT_GAPS.md`：当前缺口和关闭顺序。 历史 Week 2/Week 3 报告只作追溯，不再代表当前状态。 @@ -96,7 +98,7 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： 待完成： -- `crates/bat-assetbundle` 仍是占位 crate，完整 UnityFS/对象表/TypeTree 引擎未实现。 +- `crates/bat-assetbundle` 已具备 UnityFS 容器基础解析，完整对象表/TypeTree/TextAsset/MonoBehaviour/ScriptableObject 引擎未实现。 - Addressables parser 已覆盖当前真实形态 fixture/golden 与 `m_Crc`，但仍需继续覆盖二进制/压缩字段组合和更细失败诊断。 - 客户端发现、备份、应用补丁流程尚未连接真实实现。 @@ -144,20 +146,23 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： ### `bat-assetbundle` -状态：**占位** +状态：**UnityFS 解包和 TextAsset 提取已起步；MonoBehaviour/ScriptableObject 字段级解析未完成** -当前只有： +当前已有： -- Parser trait 仍是占位。 -- AssetType 仍是占位。 -- 错误类型骨架可用，但没有完整解析引擎。 +- `UnityFsParser`、`UnityFsBundle`、`ParsedAssetBundle`、`RawAssetBundle` 等正式类型。 +- UnityFS header、block info、directory 解析。 +- block info at end、LZ4/LZMA block info 解压、LZ4/LZMA 数据 block 解压、directory 文件提取、压缩/解压数据区大小和 directory 越界诊断。 +- Unity serialized file header、type table、TypeTree node 元数据、object table 和 TextAsset bytes 提取。 +- `ResourceImportService` 的 UnityFS 摘要已包含解包文件数、serialized file 数、TextAsset 数量和名称。 +- `bat-adapters` 的 Unity 2021.3 adapter 已改为版本选择薄层，复用 `bat-assetbundle`，避免两套 UnityFS parser。 待完成： -- UnityFS header、block、directory、metadata、object table。 -- LZ4/LZMA 解压。 -- TypeTree 解析。 -- TextAsset、MonoBehaviour、ScriptableObject 解析入口。 +- MonoBehaviour、ScriptableObject 的 TypeTree 字段级反序列化。 +- 修改 TextAsset/字段值后的 AssetBundle 重打包或 Patch 生成。 +- 真实资源 fixture 覆盖对象级解析和文本提取。 +- 详细补全顺序见 `docs/architecture/assetbundle.md`。 ### `bat-patch` @@ -198,15 +203,19 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： ### Go / API / Web -状态：**Go 产品入口仍未完成；`bat-api` 已有 typed Rust daemon RPC client、可选 CGO 兼容包装和试验性 `cmd/bat` 骨架** +状态：**边界已冻结；资源分发 MVP 已落地。权威细节见 `docs/reports/GO_STATUS.md`。** -当前情况： +| 角色 | 所有者 | 状态 | +|---|---|---| +| 同步/运维命令行（近乎全自动） | Rust `bat` | 产品入口 | +| 资源分发 HTTP | Go `cmd/bat-api` | CDN MVP | +| daemon RPC client | `internal/backendrpc` | 完成 | +| 试验 CLI | `cmd/bat` → `bin/bat-go` | 非产品 | +| FFI | `internal/ffi` | 可选 | +| 空目录 `api/` `pkg/` 等 | 占位 | 无实现 | +| Web | `web/` | 空（G-010） | -- `internal/backendrpc` 已提供 Go typed Unix socket JSON-RPC client，作为 `bat-api` 调用 Rust daemon 的默认路径。 -- `internal/ffi/ffi.go` 已存在。 -- Go CLI 的稳定集成方向仍应优先通过 Rust daemon RPC 或 Rust `bat --json` one-shot 进程边界；`cmd/bat` 目前只是试验性骨架，不代表产品级 CLI 已完成。 -- `cmd/`、`pkg/`、`api/`、`web/` 仍无可用产品入口，`cmd/bat` 目前只覆盖 `doctor`、`manifest inspect`、`sync plan` 这类最小演示能力。 -- `go test ./...` 目前只有空测试包结果，`go vet ./...` 可作为基础门禁。 +默认 Go 门禁：`make test-go-api`、`make build-go-api`（无 FFI）。 --- @@ -216,27 +225,16 @@ Rust 侧官方日服资源链路已经从实验验证推进到正式入口： ```bash cargo test --workspace --quiet -cargo clippy --workspace --all-targets -- -D warnings -go test ./... -go vet ./... -go build -o /tmp/bat-go-cli ./cmd/bat -target/debug/bat --help -git diff --check +make test-go-api +make build-go-api +go vet ./internal/api/... ./internal/backendrpc/... ./cmd/bat-api/... ``` -同步确认： +未执行 / 后置： -```bash -git status --short --branch -``` - -结果：工作区干净。 - -未执行： - -- 本次状态更新未执行一次性真实官方网络全量下载 smoke；该流程已由 `docs/guides/official-full-pull-smoke.md` 和 `scripts/official-full-pull-smoke.sh` 固化并关闭（G-018），当前处于长期运行测试阶段，运行报告将在后续提供。 -- Go CLI 端到端测试，因为 Go 产品入口尚未完成；`internal/backendrpc` 已有 fake transport 单测覆盖。 -- Web/API 测试，因为 Web/API 尚未实现。 +- 真实官方全量 smoke 长期运行报告（G-018 命令已固化）。 +- `bat-api` 对服务器全量 release 的 SSH 联调（等连接信息）。 +- Web（G-010）。 --- @@ -265,15 +263,15 @@ cargo run -p bat-infrastructure --bin bat -- \ ## 6. 当前阻塞项 -GitHub issue 状态：#1 已升为 P0；#17 的实现已合入 HEAD，但 issue 本身尚未关闭，需在验收后再同步关闭。其他 open issue 的实时标签以 GitHub 为准。 +GitHub issue 状态：#1 已关闭；#17 已按 wontfix 关闭（多线程下载入口已移除，下载回归顺序执行并保留指数退避与单调进度上报，子 issue #20–#23 均已关闭）。其他 open issue 的实时标签以 GitHub 为准。 下一阶段必须优先完成： 1. Issue #1（P0，主体已实现）：`bat.sock` Unix socket JSON-RPC 已扩展为面向 Go 服务层的 Rust Resource Backend API。统一 envelope（`ok`、`status`、`error`、`data`、`request_id`）与 `BAT-ERR` 错误码模型已落地；`daemon.*`（status/logs/stop/reload/refresh/doctor）、`resource.*`（state/sync/verify/repair/manifest/list）、`catalog.*`（status/refresh/diff/versions）、`task.*`（status/list/cancel/logs）已实现，长任务返回 `task_id` 可轮询（任务执行器单 worker FIFO，与 watch 循环互斥；任务历史持久化于 `/bat-tasks.json`，daemon 重启后仍可查，中断任务标记 `task_interrupted`）；错误码已接入下载、launcher/metadata、server-info/marker 与配置校验路径。剩余：`patch.*` / `unityfs.*`（被引擎阻塞）、`task.create`（按设计由语义方法创建）、`daemon.restart` / `daemon.clean-stable`（由 CLI 侧按进程生命周期显式执行，live RPC 内不做自重启或在线清理）、Redis 任务后端（`.env` 已预留配置键，接入时机另议）。Go 层通过 RPC 调用 Rust backend，不走 FFI（FFI 降级说明见 `docs/architecture/official-resource-backend.md` §7）。 -2. `bat-api` Go 侧：`internal/backendrpc` typed RPC client 已起步，`cmd/bat` 当前只实现 `doctor`、`manifest inspect` 和 `sync plan` 这类试验性入口，不能视作产品级 CLI；是否继续作为长期产品入口需要单独收敛。 +2. Go 侧：进度见 `docs/reports/GO_STATUS.md`。G-008 已关闭；`bat-api` 资源分发 MVP 已落地；联调与 USERGUIDE 专章后置。 3. 官方同步结果接入 CAS + ResourceRepository 的用户级工作流（G-011 剩余部分：自动导入触发、schema 迁移、CLI 查询）。 -4. Issue #3（P2）：AssetBundle UnityFS 基础解析校验。 -5. Issue #2（P2）：继续逆向 Addressables catalog，提取 bundle hash/size/CRC 等可校验字段。 +4. Issue #3（P1）：AssetBundle UnityFS 基础解析校验已具备离线和隔离真实样本覆盖；对象级解析继续跟踪 G-005。 +5. Issue #2（P1）：继续逆向 Addressables catalog，提取 bundle hash/size/CRC 等可校验字段。 6. Patch 和翻译系统仍应后置。 非阻塞跟踪项：官方同步长期运行测试正在进行，运行报告将在后续提供。 @@ -285,12 +283,12 @@ GitHub issue 状态：#1 已升为 P0；#17 的实现已合入 HEAD，但 issue 立即任务： 1. Issue #1 收尾：协议基础设施、最小方法集、`catalog.*`、`task.*`、`resource.repair`、任务持久化、错误码模型与文档（USERGUIDE §5/§6、架构文档 §7）均已完成；剩余 `patch.*`/`unityfs.*`（待引擎）以及 `task.create`、`daemon.restart`、`daemon.clean-stable` 的设计边界确认。 -2. 明确 Go 产品入口的边界：是继续推进独立 `bat` CLI，还是保留当前 Rust `bat` 为用户 CLI、Go 只做服务层与 `bat-api`。 -3. 跟进官方同步长期运行测试，收集并归档运行报告。 -4. 开始 AssetBundle parser 的 UnityFS header/block/directory（issue #3），并继续扩展 Addressables catalog 可校验字段（issue #2）。 +2. `bat-api` 与全量 release / 服务器 daemon 联调（issue #19 剩余）。 +3. 跟进官方同步长期运行测试报告。 +4. AssetBundle / Addressables（issue #3 / #2）；CAS 用户级导入（G-011）。 --- -- **当前总体完成度**：不再固定写单一百分比，以各模块状态和 issue 为准。 -- **当前基线状态**：Rust 官方资源同步链路已具备可运行闭环；Go `bat-api` 已有 Rust daemon RPC client，但产品级 CLI/API、CAS 用户级导入和完整 AssetBundle 引擎仍未完成。 -- **下一工程里程碑**：Rust Resource Backend RPC API 收尾、Go 产品入口收敛、官方同步结果接入 CAS/ResourceRepository、AssetBundle 解析起步。 +- **当前总体完成度**：不再固定写单一百分比，以各模块状态、`GO_STATUS.md` 和 issue 为准。 +- **当前基线状态**：Rust `bat` 同步闭环可用；Go `bat-api` 资源分发 MVP + `backendrpc` 可用；CAS 用户级导入与完整 AssetBundle 引擎未完成。 +- **下一工程里程碑**：bat-api 联调、CAS/ResourceRepository 用户流、AssetBundle 解析。 diff --git a/Cargo.lock b/Cargo.lock index 4230c43..65d5cbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,14 +66,13 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bat-adapters" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", "base64", + "bat-assetbundle", "bat-core", - "lz4", - "lzma-rs", "serde", "serde_json", "tempfile", @@ -83,10 +82,12 @@ dependencies = [ [[package]] name = "bat-assetbundle" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "hex", + "lz4", + "lzma-rs", "serde", "serde_json", "thiserror", @@ -95,7 +96,7 @@ dependencies = [ [[package]] name = "bat-cas-engine" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -112,7 +113,7 @@ dependencies = [ [[package]] name = "bat-core" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", @@ -125,7 +126,7 @@ dependencies = [ [[package]] name = "bat-ffi" -version = "0.2.0" +version = "1.0.0" dependencies = [ "bat-adapters", "bat-infrastructure", @@ -136,11 +137,12 @@ dependencies = [ [[package]] name = "bat-infrastructure" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", "bat-adapters", + "bat-assetbundle", "bat-cas-engine", "bat-core", "blake3", @@ -157,7 +159,7 @@ dependencies = [ [[package]] name = "bat-patch" -version = "0.2.0" +version = "1.0.0" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index 64f9d6e..ae626a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ ] [workspace.package] -version = "0.2.0" +version = "1.0.0" edition = "2021" authors = ["BlueArchive Toolkit Team"] license = "MIT" diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index 97ef9b0..24f1073 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -11,9 +11,12 @@ - `PROJECT_PLAN.md`：完整开发计划和最终目标路线图。 - `CURRENT_STATUS.md`：当前工作区真实状态。 - `docs/reports/CURRENT_GAPS.md`：当前实现缺口和关闭顺序。 +- `docs/reports/GO_STATUS.md`：Go 侧边界、约定与组件进度（权威）。 - `docs/guides/official-resource-test-pull.md`：官方资源拉取与自动更新用户指南。 - `docs/guides/official-full-pull-smoke.md`：真实官方全量拉取 smoke runbook 和可重复命令。 - `docs/architecture/official-resource-backend.md`：官方资源后端职责、工作原理和审核说明。 +- `docs/architecture/resource-release-layout.md`：release 布局、URL 映射、seed 规则、bat-api 分发契约（资源侧逆向权威）。 +- `docs/architecture/assetbundle.md`：AssetBundle、Addressables、Serialized File、文本提取和 Patch 前置解析路线图。 - `docs/reference/rpc-backend-api.md`：Rust Resource Backend JSON-RPC 稳定 contract。 - `CHANGELOG.md`：版本变更记录。 - `AGENTS.md`：AI agent 和自动化开发助手长期规则。 @@ -40,7 +43,6 @@ 后续建议新增： - `docs/architecture/cas.md`：CAS 生产级设计。 -- `docs/architecture/assetbundle.md`：AssetBundle 解析设计。 - `docs/architecture/translation.md`：翻译系统设计。 --- @@ -101,14 +103,18 @@ - CAS V1：原子写入、BLAKE3 校验、引用计数、GC、并发测试和损坏检测。 - 文档整理和路线图重制。 - Rust 官方资源同步闭环：`bat`、`--auto-discover`、`--watch`、`--daemon`、Unix socket JSON-RPC 后台控制、`status`、`stop`、`restart`、`reload`、`refresh`、`logs`、`verify`、`repair`、`doctor`、`clean-stable`、北京时间固定强制刷新、snapshot、manifest audit/repair、官方 seed `.hash` 校验。 +- 官方原版资源与汉化产物目录分离：`./bat-resources` 只承载原版 release，`./bat-localized` 承载后续汉化 release；当前官方同步报告 `not_localized`，Patch 发布完成后才进入 `localized`。 +- 官方 release 会维护 `official-parse-cache.json`，用于跳过未变化资源的重复解析。 - `bat-api/internal/backendrpc` typed Unix socket JSON-RPC client。 +- `cmd/bat-api` 资源分发 HTTP MVP（进度见 `docs/reports/GO_STATUS.md`）。 - 真实官方网络全量拉取 smoke 已固化为 `scripts/official-full-pull-smoke.sh` 和 `make official-smoke`，默认写入 `/tmp` 隔离目录并输出本地运行报告。 - `bat` 运行时 progress log 已覆盖下载已完成计数、单文件下载进度和校验结果摘要。 - Addressables 当前真实形态 fixture/golden 覆盖。 +- 解析补全路线图已固化到 `docs/architecture/assetbundle.md`：解析缓存、Addressables、UnityFS、Serialized 字段级解析、文本提取、CAS 接入和 Patch 发布前置。 - SQLite Resource Repository 和可选无状态 `bat-ffi` JSON 兼容接口。 优先待办： -- 收敛 Go 产品入口的最终形态，避免把试验性 `cmd/bat` 误当作完成品。 +- `bat-api` 与全量 release / 服务器 daemon 联调（issue #19 剩余）。 - 将官方同步结果接入 CAS + ResourceRepository 的用户级流程。 - 推进 AssetBundle UnityFS 引擎级解析。 diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index e091335..dcf5ece 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -34,27 +34,29 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 6. `bat-infrastructure` 已提供官方资源 pull/update 服务，正式入口是 Rust binary `bat`。 7. `bat` 支持 `--auto-discover`、`--watch`、`--daemon`、默认 1 小时间隔、本地 manifest audit/repair、官方 seed `.hash` 校验、snapshot/cache，以及基于 Unix socket JSON-RPC 的 live control/backend 方法（`daemon.status/logs/stop/reload/refresh/doctor`、`resource.sync/verify/repair/state/manifest/list`、`catalog.*`、`task.*`）；`restart` 和 `clean-stable` 仍由 CLI 侧按进程生命周期显式执行。 8. `bat-ffi` 已提供 Manifest inspect 和官方 sync plan 的可选无状态粗粒度 JSON C ABI helper。 -9. 文档已整理：根目录保留入口文档，历史报告进入 `docs/reports/historical/`，误嵌套的 `docs/docs` 已合并。 +9. 官方原版资源默认发布到 `./bat-resources`，汉化产物默认发布到独立的 `./bat-localized`；当前官方同步报告会标记 `localized_release_status=not_localized`，表示原版资源已发布、汉化资源未发布。 +10. 官方同步校验完成后会在 active release 下生成 `official-parse-cache.json`，未变化文件按 URL、相对路径、size 和 BLAKE3 复用解析结果。 +11. 文档已整理：根目录保留入口文档，历史报告进入 `docs/reports/historical/`，误嵌套的 `docs/docs` 已合并。 ### 仍是骨架或占位 -1. `bat-assetbundle` 仍是占位 crate；完整 UnityFS、压缩块、TypeTree 或对象表解析未完成。 +1. `bat-assetbundle` 已具备 UnityFS 解包和 TextAsset 提取基础能力（header/block info/directory、LZ4/LZMA block info 与数据 block、directory 文件提取、serialized file object table、TypeTree node 元数据、TextAsset bytes），但 MonoBehaviour/ScriptableObject 字段级解析、重打包和 Patch 仍未完成。 2. `bat-patch` 的 Binary/JSON 模块仍返回明确的未实现错误，不具备真实补丁能力。 -3. Go CLI/API/SDK 仍没有产品级入口；当前 `bat-api` module 只有 `internal/backendrpc` Rust daemon RPC client、`cmd/bat` 试验骨架与 `internal/ffi` 兼容包装。 +3. Go 侧边界已冻结（见 `docs/reports/GO_STATUS.md`）：同步/运维命令行 = Rust `bat`；资源分发 = `cmd/bat-api` MVP；`internal/backendrpc` 完成；`cmd/bat` 仅为试验（`bin/bat-go`）。完整游戏业务 API / Web / SDK 仍未完成。 4. Addressables parser 已覆盖当前真实形态 fixture/golden，但还不是完整 Unity Addressables/SBP catalog 兼容层。 5. 官方同步结果尚未作为用户级流程自动导入 CAS + ResourceRepository。 -6. 真实官方网络全量下载 smoke 已固化为可重复脚本和 runbook（G-018 已关闭）；真实运行记录处于长期运行测试阶段，报告待后续提供。 -7. Web、数据库迁移、OpenAPI、插件加载机制尚未实现。 -8. 原 Git 历史未恢复；当前仓库以新初始化基线为准。 +6. 汉化 Patch 发布流程尚未完成；`localized` 发布状态、`localized-output/current` 切换、回滚和完整性校验仍需由 Patch 阶段落地。 +7. 真实官方网络全量下载 smoke 已固化为可重复脚本和 runbook（G-018 已关闭）；真实运行记录处于长期运行测试阶段，报告待后续提供。 +8. Web、数据库迁移、OpenAPI、插件加载机制尚未实现。 +9. 原 Git 历史未恢复；当前仓库以新初始化基线为准。 ### 已验证 1. `cargo test --workspace --quiet` 通过。 2. `cargo clippy --workspace --all-targets -- -D warnings` 通过。 -3. `go test ./...` 通过，但目前没有 Go 产品级测试覆盖。 -4. `go vet ./...` 通过。 -5. `go build -o /tmp/bat-go-cli ./cmd/bat` 通过。 -6. `target/debug/bat --help` 可用。 +3. `make test-go-api` / `make build-go-api` 覆盖 `internal/api` 与 `internal/backendrpc`。 +4. `go vet` 覆盖 bat-api 相关包。 +5. `target/debug/bat --help`（Rust）可用。 --- @@ -141,7 +143,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 **目标**：完成可长期使用的 Content Addressable Storage。 -**当前状态**：已完成 CAS V1。Go CLI/API 产品入口尚未完成；`bat-api` 已有 `internal/backendrpc` typed RPC client 和 `cmd/bat` 试验骨架；Rust 继续承载完整资源拉取与更新检查核心逻辑；`bat-ffi` 仅保留为可选兼容层。 +**当前状态**：已完成 CAS V1。Rust 承载完整资源拉取与更新检查；Go 以 `bat-api` 资源分发 MVP + `backendrpc` 为服务入口（`GO_STATUS.md`）；`bat-ffi` 仅可选兼容层。 交付物： @@ -174,11 +176,13 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 2. 资源版本、区域、渠道、远端 URL、Hash、大小、依赖关系模型：**部分完成**。`Resource` 和官方 endpoint/snapshot 模型已扩展；仍需冻结 Go CLI/API 可见模型。 3. Rust 官方下载器：**已完成当前生产入口需要的核心能力**。包含官方 URL 校验、`.part` 续传、重试、本地 manifest size+BLAKE3 校验、官方 seed `.hash` 校验和 repair。 4. Rust 自动更新入口：**已完成当前生产入口**。`bat` 支持 snapshot、marker diff、bootstrap cache、one-shot、`--watch`、`--daemon`、默认 1 小时间隔、北京时间固定强制刷新，以及 Unix socket JSON-RPC 后台运维命令返回。 -5. Go 产品入口：**未完成**。`internal/backendrpc` 已提供 Go 调 Rust daemon 的 typed RPC client；当前 `cmd/bat` 仅有 `doctor`、`manifest inspect`、`sync plan` 试验能力，尚不构成产品级 CLI；若要继续由 Go 承担用户入口，需要单独收敛命令集和调用边界。 +5. Go 入口边界：**已冻结**。同步命令行 = Rust `bat`（G-008 关闭）；资源分发 = `bat-api` MVP（G-009 部分完成）。详见 `docs/reports/GO_STATUS.md`。 6. 用户级 `sync`、`manifest inspect`、`cache status`：**未完成**。Rust `bat --json` 是当前稳定进程边界；`bat-ffi` 只提供可选兼容用的 Manifest inspect 和 sync plan JSON helper。 7. 下载结果写入 CAS + ResourceRepository：**部分完成**。CAS 和 SQLite ResourceRepository 已存在，官方同步入口尚未把完整下载结果自动作为用户级流程导入。 8. Linux 生产同步不依赖已安装官方启动器：**已完成当前 Rust 入口**。`--auto-discover` 只使用官方 HTTP metadata 和临时目录解析 `GameMainConfig`。 9. 真实官方网络全量下载 smoke test：**命令已固化（G-018 已关闭）**。`scripts/official-full-pull-smoke.sh` / `make official-smoke` 已固化 dry-run、首次下载、二次 up-to-date 和本地损坏 repair 的可重复流程；真实运行处于长期运行测试阶段，报告待后续提供。 +10. 官方发布后的解析缓存：**已完成基础入口**。`official-parse-cache.json` 基于下载 manifest 覆盖直接 UnityFS bundle、zip 内 UnityFS 条目和非候选资源记录；本地文件未变化时跳过重复解析。 +11. 汉化发布状态：**已完成状态模型准备**。官方同步默认报告 `not_localized`，表示只发布原版资源；后续 Patch 阶段发布汉化资源后才切换为 `localized`。 验收标准： @@ -190,6 +194,8 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 6. 自动更新入口必须做到无变化不下载，有变化下载成功后才写入新 snapshot。 7. `--watch` 模式必须在 Rust 内部保持持久检查能力，外部 supervisor 只负责进程守护。 8. 真实官方网络 smoke 必须记录输出目录、命令、结果摘要和未纳入仓库的大文件位置。 +9. 官方原版资源目录和汉化产物目录必须物理分离，不能相同或互相嵌套。 +10. 官方同步完成后必须能区分 `not_localized` 和 `localized`，不能把原版资源发布状态与汉化产物发布状态混为一谈。 --- @@ -199,19 +205,25 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 交付物： -1. 解析 UnityFS header、blocks、directory、metadata、objects。 -2. 支持 LZ4/LZMA 解压，记录压缩块校验。 -3. 实现 TypeTree/ObjectInfo 读取。 -4. 实现 TextAsset、MonoBehaviour、ScriptableObject 的可扩展解析入口。 -5. 增加解析器注册表和版本适配器。 -6. 编写 `bundle inspect`、`bundle extract`。 +1. **解析缓存闭环**：官方同步发布后生成 `official-parse-cache.json`，覆盖 manifest 全部条目、直接 bundle、zip 内 bundle、非候选资源和解析失败诊断；未变化文件按 URL、相对路径、size 和 BLAKE3 复用解析结果。 +2. **Addressables 完整化**：覆盖 Windows/Android JSON、compact JSON 和后续二进制 catalog 入口，解析 provider、internal id、primary key、dependency、bundle name、hash、size、CRC 和资源类型。 +3. **UnityFS 容器层**：继续完善 header、block info、directory、data block、压缩、alignment、边界错误、directory 文件提取和真实样本回归。 +4. **Serialized file 层**：稳定 Unity serialized file header、type table、TypeTree node、object table、path id、class id 和 raw object bytes 表示。 +5. **字段级解析层**：实现 TypeTree 字段 reader，支持 bool、integer、float、string、bytes、array、map、PPtr 和 managed reference 诊断占位。 +6. **文本对象入口**：实现 TextAsset、MonoBehaviour、ScriptableObject 的可扩展提取入口，输出可追溯到 bundle、serialized file、path id 和 field path 的文本定位。 +7. **工具与接口**：编写 `bundle inspect`、`bundle extract`、`text extract` 的最小稳定入口；CLI/RPC/API 使用解析器输出，不直接耦合解析内部结构。 +8. **汉化发布前置**：解析结果必须能作为 Patch 输入；Patch 发布阶段才写 `localized-output` 并切换 `localized` 状态。 验收标准： -1. 能解析真实样本或明确结构化测试样本。 -2. 错误报告包含 bundle 名称、偏移、字段和 Unity 版本。 -3. 解析器和业务流程解耦。 -4. 不支持的 Unity 版本返回明确错误，不做隐式猜测。 +1. 能解析结构化测试样本、离线回归 fixture 和隔离真实样本。 +2. 错误报告包含 URL/路径、archive entry、UnityFS directory、object path id、class id、field path、offset 和 Unity 版本。 +3. 解析器和业务流程解耦；解析器不直接写 `bat-resources` 或 `bat-localized`。 +4. 不支持的 Unity 版本或 TypeTree 结构返回明确错误，不做隐式猜测。 +5. `official-parse-cache.json` 能跳过未变化资源的重复解析，且不会影响官方原版资源发布。 +6. 文本提取结果能追溯到原始资源位置，并可作为后续 Patch manifest 输入。 + +详细分层路线图见 `docs/architecture/assetbundle.md`。 --- @@ -296,6 +308,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 2. 任一步失败都能回滚到补丁前状态。 3. 不直接覆盖未经备份的客户端文件。 4. Patch 生成与应用有端到端测试。 +5. 汉化产物写入独立 `localized-output`，保留官方相对目录结构；只有完整 Patch 发布并通过校验后才切换为 `localized`。 --- @@ -366,7 +379,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 ## 5. 推荐执行顺序 -近期不要直接跳到 Web 或 AI Provider。项目当前的真实瓶颈是 Go 产品入口边界、资源解析、同步结果进入 CAS/ResourceRepository，以及真实端到端验证。 +近期不要直接跳到 Web 或 AI Provider。项目当前的真实瓶颈是资源解析、同步结果进入 CAS/ResourceRepository、`bat-api` 与全量 release 联调，以及真实端到端验证。 建议顺序： @@ -380,15 +393,14 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 ## 6. 近期具体任务 -优先完善 Rust `bat` 后端，并同步收敛 Go 产品入口边界。当前事实是 Rust `bat` 已承担可用的资源同步/运维入口，`bat-api/internal/backendrpc` 已提供 Go 到 Rust daemon 的本机 RPC client，Go `cmd/bat` 仍只是试验骨架，`bat-api` HTTP 服务仍是独立目标（issue #19 / G-009）： +优先完善 Rust 解析与资源库接入，并联调 Go 资源分发。边界见 `docs/reports/GO_STATUS.md`： -1. 对 issue #17 做验收并关闭或更新范围：多线程下载与指数退避实现已合入，但 GitHub issue 仍 open。 -2. 继续逆向 Addressables catalog，扩大 bundle hash/size/CRC 等可校验字段覆盖（issue #2）。 -3. 对 AssetBundle/UnityFS 做引擎级解析：header/block/directory/metadata/object table（issue #3 / G-005）。 -4. 将官方同步下载结果接入 CAS + `SqliteResourceRepository` 的用户级流程（G-011）。 -5. 收敛 Go 产品入口：明确继续推进最小 Go CLI，或把用户 CLI 固化为 Rust `bat` 并把 Go 侧集中到 `bat-api`。 -6. 实现 `bat-api`（仿官方 API 的 Go HTTP 服务，含鉴权/签名验签，issue #19 / G-009）。 -7. 为 CAS 增加 `doctor cas` 诊断入口。 +1. issue #17 已关闭（顺序下载 + 指数退避）。 +2. G-008 已决策关闭：同步/运维命令行 = 近乎全自动的 Rust `bat`。 +3. G-009 / issue #19：`bat-api` 资源分发 MVP 已落地；优先服务器联调；拉取仍在 Rust `bat`。 +4. 继续 Addressables（issue #2）与 UnityFS（issue #3 / G-005）。 +5. 官方同步结果接入 CAS + ResourceRepository 用户级流程（G-011）。 +6. 为 CAS 增加 `doctor cas` 诊断入口。 --- @@ -416,7 +428,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 处理策略： 1. Rust 提供稳定引擎能力，并在当前阶段承担可生产运行的官方资源同步 CLI、watch 和 daemon。 -2. Go 的长期职责包括用户命令、最小稳定 CLI、服务编排、网络和 Provider；当前 Go 产品入口尚未完成，不能把 `cmd/bat` 试验骨架视为完成。 +2. Go 的长期职责包括资源分发 HTTP（`bat-api`）、服务编排、网络和 Provider；同步/运维命令行由近乎全自动的 Rust `bat` 承担。不能把试验性 `cmd/bat` 视为产品 CLI。 3. 跨边界优先进程或 SDK，FFI 只作为可选的粗粒度、无状态、安全、可测试兼容 API。 4. Rust 不需要被强制写成 Go 调用库；当前 `bat --watch` / `bat --daemon` 是允许长期运行的 Rust 生产任务。 @@ -442,7 +454,7 @@ BlueArchiveToolkit 不是一次性脚本，也不是演示项目。最终交付 按最终目标计算，当前总体完成度不再固定写单一百分比，以模块状态和 issue 收敛情况为准。 -已完成的是稳定基线、架构骨架、部分接口、CAS V1 和 Rust 官方资源同步闭环，不是完整产品能力。下一阶段的关键不是继续堆目录，而是把 Go 产品入口边界、官方同步端到端验证、CAS/ResourceRepository 编排和 AssetBundle 解析链路做实。 +已完成的是稳定基线、架构骨架、部分接口、CAS V1、Rust 官方资源同步闭环，以及 Go `bat-api` 资源分发 MVP。下一阶段的关键是官方同步结果进入 CAS/ResourceRepository、AssetBundle 解析，以及 bat-api 与全量 release 联调。 --- diff --git a/README.md b/README.md index 07e7324..8e3a6d3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **BlueArchiveToolkit** 是一个面向长期维护的 Blue Archive 资源管理、解析、翻译和补丁工具套件。 -当前仓库仍不是完整产品，但 Rust 侧已经具备一条可运行的官方日服资源同步链路：可以在 Linux 上通过官方 HTTP metadata 自动发现资源入口，拉取 Windows + Android 官方资源，保存同步 snapshot，校验本地下载清单，并用 `--watch` 常驻定期检查更新。Go module 名为 `bat-api`，目前包含试验性的 `cmd/bat` 骨架、`internal/backendrpc` Rust daemon RPC client 和 `internal/ffi` 兼容包装；产品级 CLI、API Server、Web、完整 AssetBundle 解析、翻译系统和 Patch 系统仍在后续阶段。 +当前仓库仍不是完整产品，但 Rust 侧已经具备一条可运行的官方日服资源同步链路：可以在 Linux 上通过官方 HTTP metadata 自动发现资源入口，拉取 Windows + Android 官方资源，保存同步 snapshot，校验本地下载清单，并用近乎全自动的 `--watch` / `--daemon` 常驻更新。Go module 名为 `bat-api`：正式 Go 入口是资源分发服务 `cmd/bat-api`（经 `bat.sock` RPC 发现 release，CDN path 只读提供）；`internal/backendrpc` 为 RPC client；`cmd/bat` 仅为试验骨架（产物 `bin/bat-go`，不是产品 CLI）。边界与进度见 [`docs/reports/GO_STATUS.md`](docs/reports/GO_STATUS.md)。完整游戏业务 API、Web、AssetBundle 引擎、翻译和 Patch 仍在后续阶段。 --- @@ -10,27 +10,30 @@ - Rust workspace 和 monorepo 结构。 - `bat-core` 领域对象和仓储接口骨架。 -- `bat-adapters` Unity、Manifest、Client 集成框架，以及当前真实形态 Addressables catalog 解析覆盖，含 `m_Crc` 提取和 UnityFS 基础校验。 +- `bat-adapters` Unity、Manifest、Client 集成框架，以及当前真实形态 Addressables catalog 解析覆盖，含 `m_Crc` 提取和 UnityFS 解包/TextAsset 提取基础校验。 - `bat-cas-engine` CAS V1：原子写入、BLAKE3 校验、引用计数、GC、并发写入测试、损坏检测。 - `bat-infrastructure` CAS 适配层、SQLite Resource Repository、资源导入服务、官方资源 pull/update 服务。 - `bat`：官方资源自动发现、全量拉取、原子发布到 `current -> versions/`、本地 manifest audit/repair、`.part` 断点续传、403/404/5xx 分类重试、指数退避、顺序下载、下载 quarantine 诊断、ZIP 结构校验、官方 seed `.hash` 校验、snapshot/cache、`--watch` 常驻更新、`--daemon` 后台运行，以及 Unix socket JSON-RPC live control/backend 方法（`daemon.status/logs/stop/reload/refresh/doctor`、`resource.sync/verify/repair/state/manifest/list`、`catalog.*`、`task.*`）。 - `internal/backendrpc`：Go 侧 typed Unix socket JSON-RPC client，是 `bat-api` 调用 Rust daemon 的默认路径。 +- `cmd/bat-api`：资源分发 HTTP MVP（issue #19 / G-009）；`.env` 配置端口/RPC socket；不负责自动拉取。 +- Go 边界权威说明：[`docs/reports/GO_STATUS.md`](docs/reports/GO_STATUS.md)（G-008 已关闭：同步 CLI = Rust `bat`）。 - 官方同步会维护 `/official-version-state.json`，明确记录当前已完成版本、正在拉取版本、上一个可用版本和失败版本。 -- 资源导入链路可将 manifest 条目写入 CAS + `ResourceRepository`，AssetBundle 会记录 UnityFS 摘要，TextAsset/Table/Media 会按类型分类索引。 +- 资源导入链路可将 manifest 条目写入 CAS + `ResourceRepository`，AssetBundle 会记录 UnityFS 摘要、解包文件数、serialized file 数和 TextAsset 名称，TextAsset/Table/Media 会按类型分类索引。 - `bat-ffi` 可选无状态 C ABI 兼容层：仅保留 Manifest inspect 和官方 sync plan 的粗粒度 JSON helper，不作为 Go CLI 或生产同步的主集成边界。 - 文档路线图、当前状态、缺口清单、官方资源运行指南。 仍未完成： -- Go CLI 产品入口（当前仅有试验性 `cmd/bat` 骨架）。 -- 完整 UnityFS / AssetBundle 引擎解析。 +- `bat-api` 完整游戏业务 API / launcher 全链（资源 CDN MVP 已可用）。 +- 完整 AssetBundle 对象级解析（UnityFS 解包、object table、TypeTree node 元数据和 TextAsset bytes 已起步；MonoBehaviour/ScriptableObject 字段级解析、重打包和 Patch 仍未完成）。 - 真实 Patch apply/diff。 - Translation Memory、Glossary、AI Provider。 -- API Server、SDK、Web 管理后台。 +- SDK、Web 管理后台。 详细状态见： - [当前状态](CURRENT_STATUS.md) +- [Go 侧进度与边界](docs/reports/GO_STATUS.md) - [完整开发计划](PROJECT_PLAN.md) - [文档索引](DOCS_INDEX.md) - [当前缺口清单](docs/reports/CURRENT_GAPS.md) @@ -81,7 +84,7 @@ cargo run -p bat-infrastructure --bin bat -- \ --error-retry 60s ``` -后台自动运行可以把 `--watch` 换成 `--daemon`。默认资源目录是 `./bat-resources`，默认后台状态目录是 `/tmp/bat-pid`。daemon 会在状态目录下创建 `bat.sock` 作为 Unix socket JSON-RPC 控制通道，同时写入 `bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl` 和短生命周期的 `bat-control.lock`；`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志，`bat-status.json` 保存最后成功时间、下次检查时间、最后错误摘要和当前下载进度，`bat-control.lock` 用于串行化 `status/stop/restart/reload/logs/refresh/repair` 等控制命令： +后台自动运行可以把 `--watch` 换成 `--daemon`。默认官方原版资源目录是 `./bat-resources`，默认汉化产物目录是 `./bat-localized`，默认后台状态目录是 `/tmp/bat-pid`。daemon 会在状态目录下创建 `bat.sock` 作为 Unix socket JSON-RPC 控制通道，同时写入 `bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl` 和短生命周期的 `bat-control.lock`；`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志，`bat-status.json` 保存最后成功时间、下次检查时间、最后错误摘要和当前下载进度，`bat-control.lock` 用于串行化 `status/stop/restart/reload/logs/refresh/repair` 等控制命令： ```bash cargo run -p bat-infrastructure --bin bat -- \ @@ -99,7 +102,7 @@ cargo run -p bat-infrastructure --bin bat -- stop `bat` 会拒绝危险输出目录、路径逃逸和现有 symlink 路径组件；下载目标、`.part`、manifest、snapshot、PID、status、log 和控制锁文件不会跟随 symlink，daemon 状态类文件默认以 `0600` 权限创建。 -非 dry-run 同步不会把新文件直接写进生产可读目录。资源会先下载到 `/.staging/`，完成 manifest、BLAKE3、ZIP 和官方 `.hash` 校验后移动到 `/versions/`，再原子切换 `/current` symlink；生产读取方应只读取 `/current`。同步过程会更新 `/official-version-state.json`：下载开始时写入 `in_progress_version`，发布成功后写入 `current_completed_version` 和 `previous_available_version`，失败或中断时写入 `failed_versions`。 +非 dry-run 同步不会把新文件直接写进生产可读目录。官方原版资源会先下载到 `/.staging/`，完成 manifest、BLAKE3、ZIP 和官方 `.hash` 校验后移动到 `/versions/`，再原子切换 `/current` symlink；生产读取方应只读取 `/current`。同步过程会更新 `/official-version-state.json`：下载开始时写入 `in_progress_version`，发布成功后写入 `current_completed_version` 和 `previous_available_version`，失败或中断时写入 `failed_versions`。校验完成后会在 published release 下刷新 `official-parse-cache.json`；本地文件未变化时复用缓存并跳过重复解析。官方同步报告默认 `localized_release_status=not_localized`，表示原版资源已发布、汉化资源未发布；后续 Patch/导出默认写入 `./bat-localized`，保留官方相对目录结构，完成后才应切换为 `localized`。 资源操作命令默认输出人类可读摘要，并在没有显式 metadata 参数时默认走官方自动发现。脚本或上层程序需要稳定结构化输出时加 `--json`： @@ -129,7 +132,7 @@ make official-smoke 该 smoke 会执行 dry-run plan、首次全量拉取、二次 `up_to_date` 检查、本地文件破坏后的 `repair`、repair 后 `verify`，并在 `report/SMOKE_REPORT.md` 记录命令、输出目录、active release、文件数量、release 大小和被破坏文件。大型官方资源文件不纳入 Git。 -生产资源输出目录必须使用独立目录，不要指向已有客户端目录，也不要指向 `/home/wanye/D/BlueArchive` 这类人工维护或开发资源目录。需要覆盖默认位置时，用 `--output <资源目录>`；需要覆盖后台状态目录时，用 `--state-dir <状态目录>`。 +生产官方资源输出目录和汉化产物目录都必须使用独立目录，不要指向已有客户端目录，也不要指向 `/home/wanye/D/BlueArchive` 这类人工维护或开发资源目录。需要覆盖官方原版资源位置时，用 `--output <资源目录>`；需要覆盖汉化产物位置时，用 `--localized-output <目录>` 或 `.env` 中的 `BAT_LOCALIZED_OUTPUT`；需要覆盖后台状态目录时，用 `--state-dir <状态目录>`。 --- diff --git a/adapters/Cargo.toml b/adapters/Cargo.toml index 214030d..9eed211 100644 --- a/adapters/Cargo.toml +++ b/adapters/Cargo.toml @@ -6,6 +6,7 @@ authors.workspace = true license.workspace = true [dependencies] +bat-assetbundle = { path = "../crates/bat-assetbundle" } bat-core = { path = "../core" } anyhow.workspace = true thiserror.workspace = true @@ -13,8 +14,6 @@ serde.workspace = true serde_json.workspace = true async-trait.workspace = true tokio.workspace = true -lz4 = "1.28" -lzma-rs = "0.3" base64 = "0.22" [dev-dependencies] diff --git a/adapters/src/manifest/addressables.rs b/adapters/src/manifest/addressables.rs index d0463ae..e476e39 100644 --- a/adapters/src/manifest/addressables.rs +++ b/adapters/src/manifest/addressables.rs @@ -428,74 +428,105 @@ impl AddressablesCatalogDriver { resources } - fn compact_entry_resources(json: &Value) -> Vec { - let Some(internal_ids) = Self::string_array(json, "m_InternalIds") else { - return Vec::new(); - }; - let Some(provider_ids) = Self::string_array(json, "m_ProviderIds") else { - return Vec::new(); - }; - let Some(key_bytes) = Self::blob_bytes(json, "m_KeyDataString") else { - return Vec::new(); - }; - let Some(entry_records) = Self::compact_entry_records(json) else { - return Vec::new(); - }; - let Some(buckets) = Self::compact_buckets(json) else { - return Vec::new(); - }; + fn compact_entry_resources(json: &Value) -> Result, String> { + let internal_ids = Self::string_array(json, "m_InternalIds") + .ok_or_else(|| "compact catalog missing string array m_InternalIds".to_string())?; + let provider_ids = Self::string_array(json, "m_ProviderIds") + .ok_or_else(|| "compact catalog missing string array m_ProviderIds".to_string())?; + let key_bytes = Self::blob_bytes(json, "m_KeyDataString") + .ok_or_else(|| "compact catalog missing decodable m_KeyDataString".to_string())?; + let entry_records = Self::compact_entry_records(json) + .ok_or_else(|| "failed to decode m_EntryDataString compact records".to_string())?; + let buckets = Self::compact_buckets(json) + .ok_or_else(|| "failed to decode m_BucketDataString compact buckets".to_string())?; let keys = Self::compact_keys(&key_bytes, &buckets); if keys.is_empty() { - return Vec::new(); + return Err("compact catalog contains no decodable key buckets".to_string()); } let internal_id_prefixes = Self::string_array(json, "m_InternalIdPrefixes") .unwrap_or_default() .into_iter() .collect::>(); - let extra_data = Self::blob_bytes(json, "m_ExtraDataString").unwrap_or_default(); + let extra_data = if json.get("m_ExtraDataString").is_some() { + Self::blob_bytes(json, "m_ExtraDataString") + .ok_or_else(|| "compact catalog has undecodable m_ExtraDataString".to_string())? + } else { + Vec::new() + }; - entry_records - .iter() - .enumerate() - .filter_map(|(index, record)| { - let internal_id = internal_ids.get(record.internal_id as usize)?; - provider_ids.get(record.provider_index as usize)?; - let primary_key = keys - .get(record.primary_key_index as usize) - .and_then(|key| key.as_ref()) - .and_then(AddressablesObject::key_string) - .unwrap_or_else(|| format!("addressable_{}", index)); - let path = Self::normalize_internal_id(&internal_id_prefixes, internal_id); - let extra = Self::extra_data_at(&extra_data, record.data_index); - let resource_type_name = Self::resource_type_name(json, record.resource_type_index); - let dependencies = - Self::compact_dependencies(record, &entry_records, &buckets, &keys); - let hash = extra - .hash - .filter(|value| !value.is_empty()) - .or_else(|| extra.bundle_name.filter(|value| !value.is_empty())) - .unwrap_or_else(|| format!("addressable_{}", index)); + let mut resources = Vec::with_capacity(entry_records.len()); + for (index, record) in entry_records.iter().enumerate() { + if record.internal_id < 0 { + return Err(format!("compact entry {index} has negative internal_id")); + } + if record.provider_index < 0 { + return Err(format!("compact entry {index} has negative provider_index")); + } + if record.primary_key_index < 0 { + return Err(format!( + "compact entry {index} has negative primary_key_index" + )); + } - Some(ResourceEntry { - path: path.clone(), - hash, - size: extra.bundle_size.unwrap_or_default(), - resource_type: Self::resource_type_for_compact_entry( - &path, - resource_type_name.as_deref(), - ), - address: if primary_key.is_empty() { - None - } else { - Some(primary_key) - }, - dependencies, - crc: extra.crc, - }) - }) - .collect() + let internal_id = internal_ids + .get(record.internal_id as usize) + .ok_or_else(|| { + format!( + "compact entry {index} internal_id index {} out of range {}", + record.internal_id, + internal_ids.len() + ) + })?; + provider_ids + .get(record.provider_index as usize) + .ok_or_else(|| { + format!( + "compact entry {index} provider_index {} out of range {}", + record.provider_index, + provider_ids.len() + ) + })?; + let primary_key = keys + .get(record.primary_key_index as usize) + .and_then(|key| key.as_ref()) + .and_then(AddressablesObject::key_string) + .ok_or_else(|| { + format!( + "compact entry {index} primary_key_index {} has no decodable key", + record.primary_key_index + ) + })?; + let path = Self::normalize_internal_id(&internal_id_prefixes, internal_id); + let extra = Self::extra_data_at(&extra_data, record.data_index); + let resource_type_name = Self::resource_type_name(json, record.resource_type_index); + let dependencies = Self::compact_dependencies(record, &entry_records, &buckets, &keys); + let hash = extra + .hash + .filter(|value| !value.is_empty()) + .or_else(|| extra.bundle_name.filter(|value| !value.is_empty())) + .unwrap_or_else(|| format!("addressable_{}", index)); + + resources.push(ResourceEntry { + path: path.clone(), + hash, + size: extra.bundle_size.unwrap_or_default(), + resource_type: Self::resource_type_for_compact_entry( + &path, + resource_type_name.as_deref(), + ), + address: if primary_key.is_empty() { + None + } else { + Some(primary_key) + }, + dependencies, + crc: extra.crc, + }); + } + + Ok(resources) } fn string_array(json: &Value, field: &str) -> Option> { @@ -677,15 +708,30 @@ impl AddressablesCatalogDriver { format!("{prefix}{path}") } - fn resources(json: &Value) -> Vec { + fn has_compact_catalog_fields(json: &Value) -> bool { + [ + "m_ProviderIds", + "m_KeyDataString", + "m_BucketDataString", + "m_EntryDataString", + "m_ExtraDataString", + "m_resourceTypes", + ] + .iter() + .any(|field| json.get(field).is_some()) + } + + fn resources(json: &Value) -> Result, String> { let entry_resources = Self::entry_resources(json); if !entry_resources.is_empty() { - return entry_resources; + return Ok(entry_resources); } - let compact_resources = Self::compact_entry_resources(json); - if !compact_resources.is_empty() { - return compact_resources; + if Self::has_compact_catalog_fields(json) { + let compact_resources = Self::compact_entry_resources(json)?; + if !compact_resources.is_empty() { + return Ok(compact_resources); + } } let key_resources = Self::key_data_resources(json); @@ -694,13 +740,13 @@ impl AddressablesCatalogDriver { .map(|count| key_resources.len() >= count) .unwrap_or(true) { - return key_resources; + return Ok(key_resources); } - Self::internal_id_resources(json) + Ok(Self::internal_id_resources(json)) } - fn extra_metadata(json: &Value) -> HashMap { + fn extra_metadata(json: &Value, resources: &[ResourceEntry]) -> HashMap { let mut extra = HashMap::new(); Self::insert_array_len(&mut extra, json, "m_InternalIds", "internal_id_count"); Self::insert_array_len(&mut extra, json, "m_Entries", "entry_count"); @@ -732,7 +778,7 @@ impl AddressablesCatalogDriver { "m_ExtraDataString", "extra_data_string_len", ); - Self::insert_dependency_count(&mut extra, json); + Self::insert_resource_summary(&mut extra, resources); extra } @@ -772,13 +818,56 @@ impl AddressablesCatalogDriver { } } - fn insert_dependency_count(extra: &mut HashMap, json: &Value) { - let count = Self::entry_resources(json) - .into_iter() + fn insert_resource_summary(extra: &mut HashMap, resources: &[ResourceEntry]) { + extra.insert("resource_count".to_string(), resources.len().to_string()); + + let asset_bundle_count = resources + .iter() + .filter(|entry| entry.resource_type == ResourceType::AssetBundle) + .count(); + if asset_bundle_count > 0 { + extra.insert( + "asset_bundle_count".to_string(), + asset_bundle_count.to_string(), + ); + } + + let declared_size_count = resources.iter().filter(|entry| entry.size != 0).count(); + if declared_size_count > 0 { + extra.insert( + "declared_size_count".to_string(), + declared_size_count.to_string(), + ); + } + + let declared_crc_count = resources + .iter() + .filter(|entry| entry.declared_crc().is_some()) + .count(); + if declared_crc_count > 0 { + extra.insert( + "declared_crc_count".to_string(), + declared_crc_count.to_string(), + ); + } + + let dependency_count = resources + .iter() .map(|entry| entry.dependencies.len()) .sum::(); - if count > 0 { - extra.insert("dependency_count".to_string(), count.to_string()); + if dependency_count > 0 { + extra.insert("dependency_count".to_string(), dependency_count.to_string()); + } + + let fallback_hash_count = resources + .iter() + .filter(|entry| entry.hash.starts_with("addressable_")) + .count(); + if fallback_hash_count > 0 { + extra.insert( + "fallback_hash_count".to_string(), + fallback_hash_count.to_string(), + ); } } } @@ -886,16 +975,17 @@ impl ManifestDriver for AddressablesCatalogDriver { async fn parse(&self, raw_data: &[u8]) -> Result { let json = Self::parse_json(raw_data)?; + let resources = Self::resources(&json)?; let metadata = ManifestMetadata { locator_id: Self::locator_id(&json), cdn_prefixes: Self::cdn_prefixes(&json), - extra: Self::extra_metadata(&json), + extra: Self::extra_metadata(&json, &resources), }; Ok(GenericManifest { format: ManifestFormat::AddressablesCatalog, - resources: Self::resources(&json), + resources, metadata, }) } @@ -904,6 +994,91 @@ impl ManifestDriver for AddressablesCatalogDriver { #[cfg(test)] mod tests { use super::*; + use base64::engine::general_purpose::STANDARD; + use base64::Engine; + + fn push_u32_le(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i32_le(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_serialized_string(data: &mut Vec, value: &str) -> usize { + let offset = data.len(); + data.push(0); + push_u32_le(data, value.len() as u32); + data.extend_from_slice(value.as_bytes()); + offset + } + + fn serialized_json_object(json_text: &str) -> Vec { + let assembly_name = + "Unity.ResourceManager, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null"; + let class_name = + "UnityEngine.ResourceManagement.ResourceProviders.AssetBundleRequestOptions"; + let mut json_bytes = Vec::new(); + for unit in json_text.encode_utf16() { + json_bytes.extend_from_slice(&unit.to_le_bytes()); + } + + let mut data = Vec::new(); + data.push(7); + data.push(assembly_name.len() as u8); + data.extend_from_slice(assembly_name.as_bytes()); + data.push(class_name.len() as u8); + data.extend_from_slice(class_name.as_bytes()); + push_u32_le(&mut data, json_bytes.len() as u32); + data.extend_from_slice(&json_bytes); + data + } + + fn compact_catalog_json(extra_json: &str) -> String { + let mut key_data = Vec::new(); + push_u32_le(&mut key_data, 1); + let key_offset = push_serialized_string(&mut key_data, "synthetic.bundle"); + + let mut bucket_data = Vec::new(); + push_u32_le(&mut bucket_data, 1); + push_i32_le(&mut bucket_data, key_offset as i32); + push_i32_le(&mut bucket_data, 1); + push_i32_le(&mut bucket_data, 0); + + let mut entry_data = Vec::new(); + push_u32_le(&mut entry_data, 1); + push_i32_le(&mut entry_data, 0); // internal_id + push_i32_le(&mut entry_data, 0); // provider_index + push_i32_le(&mut entry_data, -1); // dependency_key_index + push_i32_le(&mut entry_data, 0); // reserved/unused + push_i32_le(&mut entry_data, 0); // data_index + push_i32_le(&mut entry_data, 0); // primary_key_index + push_i32_le(&mut entry_data, 0); // resource_type_index + + let extra_data = serialized_json_object(extra_json); + + serde_json::json!({ + "m_LocatorId": "AddressablesMainContentCatalog", + "m_InternalIdPrefixes": [], + "m_ProviderIds": [ + "UnityEngine.ResourceManagement.ResourceProviders.AssetBundleProvider" + ], + "m_InternalIds": [ + "{PlatformUtils.AddressableLoadPath}\\synthetic.bundle" + ], + "m_resourceTypes": [ + { + "m_AssemblyName": "Unity.ResourceManager, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", + "m_ClassName": "UnityEngine.ResourceManagement.ResourceProviders.IAssetBundleResource" + } + ], + "m_KeyDataString": STANDARD.encode(key_data), + "m_BucketDataString": STANDARD.encode(bucket_data), + "m_EntryDataString": STANDARD.encode(entry_data), + "m_ExtraDataString": STANDARD.encode(extra_data) + }) + .to_string() + } #[test] fn test_can_parse_valid_catalog() { @@ -1048,6 +1223,65 @@ mod tests { assert!(error.contains("Invalid JSON at line")); } + #[tokio::test] + async fn test_parse_compact_catalog_extracts_verification_fields() { + let driver = AddressablesCatalogDriver::new(); + let catalog_json = compact_catalog_json( + r#"{ + "m_Hash":"hash-compact", + "m_Crc":305419896, + "m_BundleName":"synthetic-bundle-name", + "m_BundleSize":42 + }"#, + ); + + let manifest = driver.parse(catalog_json.as_bytes()).await.unwrap(); + + assert_eq!(manifest.resources.len(), 1); + let resource = &manifest.resources[0]; + assert_eq!(resource.path, "synthetic.bundle"); + assert_eq!(resource.hash, "hash-compact"); + assert_eq!(resource.size, 42); + assert_eq!(resource.crc, Some(0x1234_5678)); + assert_eq!(resource.resource_type, ResourceType::AssetBundle); + assert_eq!(resource.address.as_deref(), Some("synthetic.bundle")); + assert_eq!( + manifest.metadata.extra.get("resource_count"), + Some(&"1".to_string()) + ); + assert_eq!( + manifest.metadata.extra.get("asset_bundle_count"), + Some(&"1".to_string()) + ); + assert_eq!( + manifest.metadata.extra.get("declared_size_count"), + Some(&"1".to_string()) + ); + assert_eq!( + manifest.metadata.extra.get("declared_crc_count"), + Some(&"1".to_string()) + ); + } + + #[tokio::test] + async fn test_parse_compact_catalog_reports_blob_decode_failure() { + let driver = AddressablesCatalogDriver::new(); + let catalog_json = r#"{ + "m_LocatorId": "AddressablesMainContentCatalog", + "m_ProviderIds": ["UnityEngine.ResourceManagement.ResourceProviders.AssetBundleProvider"], + "m_InternalIds": ["synthetic.bundle"], + "m_KeyDataString": "not-base64", + "m_BucketDataString": "not-base64", + "m_EntryDataString": "not-base64", + "m_ExtraDataString": "not-base64" + }"#; + + let error = driver.parse(catalog_json.as_bytes()).await.unwrap_err(); + + assert!(error.contains("compact catalog"), "{error}"); + assert!(error.contains("m_KeyDataString"), "{error}"); + } + #[tokio::test] async fn test_parse_table_bundle_resource_types() { let driver = AddressablesCatalogDriver::new(); diff --git a/adapters/src/official/game_main_config.rs b/adapters/src/official/game_main_config.rs index c6096d2..9a6bf4f 100644 --- a/adapters/src/official/game_main_config.rs +++ b/adapters/src/official/game_main_config.rs @@ -38,13 +38,14 @@ pub struct YostarJpGameMainConfig { impl YostarJpGameMainConfig { /// Reads and decrypts `GameMainConfig` from a Unity serialized file. pub fn from_resources_assets(path: impl AsRef) -> Result { - let serialized = UnitySerializedFile::from_path(path)?; + let serialized = UnitySerializedFile::from_path(path).map_err(|error| error.to_string())?; Self::from_serialized_file(&serialized) } /// Reads and decrypts `GameMainConfig` from serialized file bytes. pub fn from_resources_assets_bytes(bytes: &[u8]) -> Result { - let serialized = UnitySerializedFile::from_slice(bytes)?; + let serialized = + UnitySerializedFile::from_slice(bytes).map_err(|error| error.to_string())?; Self::from_serialized_file(&serialized) } diff --git a/adapters/src/unity.rs b/adapters/src/unity.rs index 0d7b698..3cbe89b 100644 --- a/adapters/src/unity.rs +++ b/adapters/src/unity.rs @@ -9,8 +9,11 @@ pub mod unity_2021_3; pub use adapter::{ ParsedAssetBundle, RawAssetBundle, UnityAdapter, UnityFsBlockInfo, UnityFsCompression, - UnityFsDirectoryInfo, UnityFsHeader, VersionRange, + UnityFsDirectoryInfo, UnityFsFile, UnityFsHeader, UnitySerializedParseError, VersionRange, }; pub use registry::UnityAdapterRegistry; -pub use serialized_file::{UnitySerializedFile, UnitySerializedTextAsset}; +pub use serialized_file::{ + UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, + UnityTypeTreeNode, +}; pub use unity_2021_3::Unity2021_3Adapter; diff --git a/adapters/src/unity/adapter.rs b/adapters/src/unity/adapter.rs index d3f1c5c..f35ccf1 100644 --- a/adapters/src/unity/adapter.rs +++ b/adapters/src/unity/adapter.rs @@ -1,6 +1,10 @@ //! Unity Adapter 接口定义 use async_trait::async_trait; +pub use bat_assetbundle::{ + ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsCompression, UnityFsDirectoryInfo, + UnityFsFile, UnityFsHeader, UnitySerializedParseError, +}; /// Unity 版本范围 #[derive(Debug, Clone)] @@ -56,92 +60,6 @@ fn parse_version_components(version: &str) -> Option<(u64, u64, u64)> { Some((major, minor, patch)) } -/// 原始 AssetBundle 数据 -#[derive(Debug)] -pub struct RawAssetBundle { - /// 文件数据 - pub data: Vec, - /// 文件路径（可选） - pub path: Option, -} - -/// 解析后的 AssetBundle -#[derive(Debug)] -pub struct ParsedAssetBundle { - /// Unity 版本 - pub unity_version: String, - /// 资源列表（简化表示） - pub assets: Vec, - /// 原始数据（保留用于序列化） - pub raw_data: Vec, - /// UnityFS 文件头信息。 - pub unityfs_header: Option, - /// UnityFS 压缩块信息。 - pub blocks: Vec, - /// UnityFS 目录信息。 - pub directories: Vec, -} - -/// UnityFS 文件头。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UnityFsHeader { - /// UnityFS 格式版本。 - pub format_version: u32, - /// Bundle 目标版本字符串，例如 `5.x.x`。 - pub target_version: String, - /// Unity 编辑器版本字符串。 - pub unity_version: String, - /// 文件总大小。 - pub total_size: u64, - /// 压缩后的 block info 大小。 - pub compressed_blocks_info_size: u32, - /// 解压后的 block info 大小。 - pub uncompressed_blocks_info_size: u32, - /// UnityFS flags 原始值。 - pub flags: u32, -} - -/// UnityFS 块压缩类型。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum UnityFsCompression { - /// 未压缩。 - None, - /// LZMA 压缩。 - Lzma, - /// LZ4 压缩。 - Lz4, - /// LZ4HC 压缩。 - Lz4Hc, - /// 当前版本未识别的压缩类型。 - Unknown(u16), -} - -/// UnityFS 压缩块信息。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UnityFsBlockInfo { - /// 解压后大小。 - pub uncompressed_size: u32, - /// 压缩后大小。 - pub compressed_size: u32, - /// 块 flags 原始值。 - pub flags: u16, - /// 解析出的压缩类型。 - pub compression: UnityFsCompression, -} - -/// UnityFS 目录条目。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UnityFsDirectoryInfo { - /// 条目在数据区中的偏移。 - pub offset: u64, - /// 条目大小。 - pub size: u64, - /// 条目 flags 原始值。 - pub flags: u32, - /// 条目路径。 - pub path: String, -} - /// Unity Adapter 接口 /// /// 用于解析不同 Unity 版本的 AssetBundle @@ -172,8 +90,7 @@ pub trait UnityAdapter: Send + Sync { /// - 成功：返回解析后的 AssetBundle /// - 失败：返回错误 /// - /// # 注意 - /// Phase 1 中标记为 TODO，Phase 2 实现 + /// 当前 UnityFS 容器解析由具体适配器委托给 `bat-assetbundle`。 async fn parse(&self, bundle: &RawAssetBundle) -> Result; /// 序列化 AssetBundle @@ -185,8 +102,7 @@ pub trait UnityAdapter: Send + Sync { /// - 成功：返回序列化后的数据 /// - 失败：返回错误 /// - /// # 注意 - /// Phase 1 中标记为 TODO，Phase 2 实现 + /// 当前阶段只定义接口；具体序列化能力尚未进入实现范围。 async fn serialize(&self, parsed: &ParsedAssetBundle) -> Result, String>; } diff --git a/adapters/src/unity/serialized_file.rs b/adapters/src/unity/serialized_file.rs index 4375c64..9db2f3d 100644 --- a/adapters/src/unity/serialized_file.rs +++ b/adapters/src/unity/serialized_file.rs @@ -1,521 +1,9 @@ -//! Unity serialized file reader. +//! Compatibility exports for Unity serialized file parsing. //! -//! This module is intentionally narrow: it extracts `TextAsset` payloads from -//! Unity serialized files such as `resources.assets` and -//! `globalgamemanagers.assets`. +//! The implementation lives in `bat-assetbundle`; adapters keep this module so +//! existing call sites can continue to import through `bat_adapters::unity`. -use std::fs; -use std::path::Path; - -/// One extracted Unity `TextAsset`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UnitySerializedTextAsset { - /// Unity path ID of the object. - pub path_id: i64, - /// Asset name stored in the serialized object. - pub name: String, - /// Raw bytes stored by the `TextAsset`. - pub bytes: Vec, -} - -/// Parsed Unity serialized file summary. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UnitySerializedFile { - /// Serialized file format version. - pub version: u32, - /// Unity editor version stored in the file. - pub unity_version: String, - /// Target platform value from the file header. - pub platform: i32, - text_assets: Vec, -} - -impl UnitySerializedFile { - /// Parses a serialized file from raw bytes. - pub fn from_slice(data: &[u8]) -> Result { - let mut reader = Reader::new(data); - - let _metadata_size = reader.read_u32_be("metadata_size")?; - let _file_size = reader.read_u32_be("file_size")?; - let version = reader.read_u32_be("version")?; - let _data_offset = reader.read_u32_be("data_offset")?; - let endian_flag = reader.read_u8("endian_flag")?; - reader.read_bytes(3, "reserved")?; - - let (metadata_size, file_size, data_offset) = if version >= 22 { - let metadata_size = reader.read_u32_be("metadata_size_2")?; - let file_size = reader.read_u64_be("file_size_2")?; - let data_offset = reader.read_u64_be("data_offset_2")? as usize; - let _unknown = reader.read_u64_be("unknown_2")?; - (metadata_size, file_size, data_offset) - } else { - (_metadata_size, _file_size as u64, _data_offset as usize) - }; - let _ = metadata_size; - let _ = file_size; - - let endian = if endian_flag == 0 { - Endian::Little - } else { - Endian::Big - }; - reader.set_endian(endian); - - let unity_version = reader.read_c_string("unity_version")?; - let platform = reader.read_i32("platform")?; - let enable_type_tree = reader.read_u8("enable_type_tree")?; - let type_count = reader.read_i32("type_count")?; - if type_count < 0 { - return Err(format!("Invalid Unity type count: {}", type_count)); - } - - let mut class_ids = Vec::with_capacity(type_count as usize); - for _ in 0..type_count { - class_ids.push(read_serialized_type( - &mut reader, - version, - enable_type_tree, - )?); - } - - let big_id_enabled = if (11..14).contains(&version) { - reader.read_i32("big_id_enabled")? - } else { - 0 - }; - - let object_count = reader.read_i32("object_count")?; - if object_count < 0 { - return Err(format!("Invalid Unity object count: {}", object_count)); - } - - let mut text_assets = Vec::new(); - for _ in 0..object_count { - if version >= 14 { - reader.align(4)?; - } - - let path_id = if big_id_enabled != 0 { - reader.read_i64("path_id")? - } else if version < 14 { - reader.read_i32("path_id")? as i64 - } else { - reader.read_i64("path_id")? - }; - - let byte_start = if version >= 22 { - reader.read_u64("byte_start")? as usize - } else { - reader.read_u32("byte_start")? as usize - }; - let byte_size = reader.read_u32("byte_size")? as usize; - let type_id = reader.read_i32("type_id")?; - if version < 16 { - reader.read_u16("class_id")?; - } - if version < 11 { - reader.read_u16("is_destroyed")?; - } - if (11..17).contains(&version) { - reader.read_i16("script_type_index")?; - } - if version == 15 || version == 16 { - reader.read_u8("stripped")?; - } - - let class_id = class_ids - .get(type_id as usize) - .copied() - .ok_or_else(|| format!("Invalid Unity type index: {}", type_id))?; - if class_id == 49 { - let object_start = data_offset - .checked_add(byte_start) - .ok_or_else(|| "Unity object offset overflow".to_string())?; - let object_end = object_start - .checked_add(byte_size) - .ok_or_else(|| "Unity object size overflow".to_string())?; - if object_end > data.len() { - return Err(format!( - "Unity object exceeds file size: start={}, size={}, file_size={}", - object_start, - byte_size, - data.len() - )); - } - - let asset = parse_text_asset(path_id, &data[object_start..object_end], endian)?; - text_assets.push(asset); - } - } - - Ok(Self { - version, - unity_version, - platform, - text_assets, - }) - } - - /// Parses a serialized file from disk. - pub fn from_path(path: impl AsRef) -> Result { - let path = path.as_ref(); - let bytes = fs::read(path) - .map_err(|error| format!("Failed to read {}: {error}", path.display()))?; - Self::from_slice(&bytes) - } - - /// Returns all extracted text assets. - pub fn text_assets(&self) -> &[UnitySerializedTextAsset] { - &self.text_assets - } - - /// Returns one extracted text asset by name. - pub fn text_asset(&self, name: &str) -> Option<&UnitySerializedTextAsset> { - self.text_assets.iter().find(|asset| asset.name == name) - } -} - -fn parse_text_asset( - path_id: i64, - data: &[u8], - endian: Endian, -) -> Result { - let mut reader = Reader::new(data); - reader.set_endian(endian); - let name = reader.read_len_prefixed_string("text_asset_name")?; - reader.align(4)?; - let bytes_len = reader.read_u32("text_asset_bytes_len")? as usize; - let bytes = reader.read_bytes(bytes_len, "text_asset_bytes")?.to_vec(); - - Ok(UnitySerializedTextAsset { - path_id, - name, - bytes, - }) -} - -fn read_serialized_type( - reader: &mut Reader<'_>, - version: u32, - enable_type_tree: u8, -) -> Result { - let class_id = reader.read_i32("type_class_id")?; - - if version >= 16 { - reader.read_u8("type_is_stripped")?; - } - if version >= 17 { - reader.read_i16("type_script_index")?; - } - if version >= 13 { - if (version < 16 && class_id < 0) || (version >= 16 && class_id == 114) { - reader.read_bytes(16, "type_script_id")?; - } - reader.read_bytes(16, "type_hash")?; - } - - if enable_type_tree != 0 { - if version >= 12 || version == 10 { - let node_count = reader.read_i32("type_tree_node_count")?; - if node_count < 0 { - return Err(format!( - "Invalid Unity type tree node count: {}", - node_count - )); - } - let string_buffer_size = reader.read_i32("type_tree_string_buffer_size")?; - if string_buffer_size < 0 { - return Err(format!( - "Invalid Unity type tree string buffer size: {}", - string_buffer_size - )); - } - - let node_size = 2 + 1 + 1 + 4 + 4 + 4 + 4 + 4 + if version >= 19 { 8 } else { 0 }; - reader.read_bytes(node_count as usize * node_size, "type_tree_nodes")?; - reader.read_bytes(string_buffer_size as usize, "type_tree_strings")?; - } - - if version >= 21 { - let dependency_count = reader.read_i32("type_tree_dependency_count")?; - if dependency_count < 0 { - return Err(format!( - "Invalid Unity type tree dependency count: {}", - dependency_count - )); - } - reader.read_bytes(dependency_count as usize * 4, "type_tree_dependencies")?; - } - } - - Ok(class_id) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Endian { - Little, - Big, -} - -struct Reader<'a> { - data: &'a [u8], - offset: usize, - endian: Endian, -} - -impl<'a> Reader<'a> { - fn new(data: &'a [u8]) -> Self { - Self { - data, - offset: 0, - endian: Endian::Big, - } - } - - fn set_endian(&mut self, endian: Endian) { - self.endian = endian; - } - - fn read_bytes(&mut self, len: usize, field: &str) -> Result<&'a [u8], String> { - let end = self - .offset - .checked_add(len) - .ok_or_else(|| format!("{field} length overflow at offset {}", self.offset))?; - if end > self.data.len() { - return Err(format!( - "Unexpected end while reading {field} at offset {}: need {}, have {}", - self.offset, - len, - self.data.len().saturating_sub(self.offset) - )); - } - - let bytes = &self.data[self.offset..end]; - self.offset = end; - Ok(bytes) - } - - fn align(&mut self, alignment: usize) -> Result<(), String> { - if alignment == 0 { - return Ok(()); - } - - let remainder = self.offset % alignment; - if remainder == 0 { - return Ok(()); - } - - let padding = alignment - remainder; - self.read_bytes(padding, "alignment padding").map(|_| ()) - } - - fn read_u8(&mut self, field: &str) -> Result { - Ok(self.read_bytes(1, field)?[0]) - } - - fn read_u16(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(2, field)?; - Ok(match self.endian { - Endian::Little => u16::from_le_bytes([bytes[0], bytes[1]]), - Endian::Big => u16::from_be_bytes([bytes[0], bytes[1]]), - }) - } - - fn read_i16(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(2, field)?; - Ok(match self.endian { - Endian::Little => i16::from_le_bytes([bytes[0], bytes[1]]), - Endian::Big => i16::from_be_bytes([bytes[0], bytes[1]]), - }) - } - - fn read_u32(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(4, field)?; - Ok(match self.endian { - Endian::Little => u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), - Endian::Big => u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), - }) - } - - fn read_u32_be(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(4, field)?; - Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) - } - - fn read_i32(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(4, field)?; - Ok(match self.endian { - Endian::Little => i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), - Endian::Big => i32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), - }) - } - - fn read_u64(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(8, field)?; - Ok(match self.endian { - Endian::Little => u64::from_le_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]), - Endian::Big => u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]), - }) - } - - fn read_u64_be(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(8, field)?; - Ok(u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ])) - } - - fn read_i64(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(8, field)?; - Ok(match self.endian { - Endian::Little => i64::from_le_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]), - Endian::Big => i64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]), - }) - } - - fn read_c_string(&mut self, field: &str) -> Result { - let remaining = &self.data[self.offset..]; - let Some(length) = remaining.iter().position(|&byte| byte == 0) else { - return Err(format!( - "Missing null terminator while reading {field} at offset {}", - self.offset - )); - }; - let bytes = self.read_bytes(length, field)?; - self.offset += 1; - std::str::from_utf8(bytes) - .map(ToOwned::to_owned) - .map_err(|error| format!("Invalid UTF-8 in {field}: {error}")) - } - - fn read_len_prefixed_string(&mut self, field: &str) -> Result { - let len = self.read_u32(field)? as usize; - let bytes = self.read_bytes(len, field)?; - std::str::from_utf8(bytes) - .map(ToOwned::to_owned) - .map_err(|error| format!("Invalid UTF-8 in {field}: {error}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn push_i16_le(data: &mut Vec, value: i16) { - data.extend_from_slice(&value.to_le_bytes()); - } - - fn push_u32_le(data: &mut Vec, value: u32) { - data.extend_from_slice(&value.to_le_bytes()); - } - - fn push_i32_le(data: &mut Vec, value: i32) { - data.extend_from_slice(&value.to_le_bytes()); - } - - fn push_i64_le(data: &mut Vec, value: i64) { - data.extend_from_slice(&value.to_le_bytes()); - } - - fn push_u64_le(data: &mut Vec, value: u64) { - data.extend_from_slice(&value.to_le_bytes()); - } - - fn push_u32_be(data: &mut Vec, value: u32) { - data.extend_from_slice(&value.to_be_bytes()); - } - - fn push_u64_be(data: &mut Vec, value: u64) { - data.extend_from_slice(&value.to_be_bytes()); - } - - fn align(data: &mut Vec, alignment: usize) { - let remainder = data.len() % alignment; - if remainder != 0 { - data.resize(data.len() + alignment - remainder, 0); - } - } - - fn synthetic_serialized_file() -> Vec { - let mut object_data = Vec::new(); - push_u32_le(&mut object_data, 14); - object_data.extend_from_slice(b"GameMainConfig"); - align(&mut object_data, 4); - push_u32_le(&mut object_data, 5); - object_data.extend_from_slice(b"hello"); - - let mut metadata = Vec::new(); - metadata.extend_from_slice(b"2021.3.56f2\0"); - push_i32_le(&mut metadata, 19); - metadata.push(0); - push_i32_le(&mut metadata, 1); - push_i32_le(&mut metadata, 49); - metadata.push(0); - push_i16_le(&mut metadata, 0); - metadata.extend_from_slice(&[0; 16]); - push_i32_le(&mut metadata, 1); - align(&mut metadata, 4); - push_i64_le(&mut metadata, 1); - push_u64_le(&mut metadata, 0); - push_u32_le(&mut metadata, object_data.len() as u32); - push_i32_le(&mut metadata, 0); - - let header_len = 48usize; - let data_offset = header_len + metadata.len(); - let file_size = data_offset + object_data.len(); - - let mut file = Vec::new(); - push_u32_be(&mut file, metadata.len() as u32); - push_u32_be(&mut file, file_size as u32); - push_u32_be(&mut file, 22); - push_u32_be(&mut file, 0); - file.push(0); - file.extend_from_slice(&[0, 0, 0]); - push_u32_be(&mut file, metadata.len() as u32); - push_u64_be(&mut file, file_size as u64); - push_u64_be(&mut file, data_offset as u64); - push_u64_be(&mut file, 0); - file.extend_from_slice(&metadata); - file.extend_from_slice(&object_data); - file - } - - #[test] - fn parses_synthetic_text_asset() { - let file = synthetic_serialized_file(); - let parsed = UnitySerializedFile::from_slice(&file).unwrap(); - - assert_eq!(parsed.version, 22); - assert_eq!(parsed.unity_version, "2021.3.56f2"); - assert_eq!(parsed.platform, 19); - assert_eq!(parsed.text_assets.len(), 1); - - let asset = parsed.text_asset("GameMainConfig").unwrap(); - assert_eq!(asset.name, "GameMainConfig"); - assert_eq!(asset.bytes, b"hello"); - } - - #[test] - #[ignore = "requires BAT_REAL_RESOURCES_ASSETS pointing at a local resources.assets"] - fn reads_text_asset_from_real_resource_file() { - let path = std::env::var("BAT_REAL_RESOURCES_ASSETS") - .expect("BAT_REAL_RESOURCES_ASSETS must be set"); - - let parsed = - UnitySerializedFile::from_path(Path::new(&path)).expect("parse local resources.assets"); - let asset = parsed - .text_asset("GameMainConfig") - .expect("GameMainConfig TextAsset present"); - - assert_eq!(asset.name, "GameMainConfig"); - assert!(!asset.bytes.is_empty()); - } -} +pub use bat_assetbundle::{ + UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, + UnityTypeTreeNode, +}; diff --git a/adapters/src/unity/unity_2021_3.rs b/adapters/src/unity/unity_2021_3.rs index 1b87b77..5fb5b9d 100644 --- a/adapters/src/unity/unity_2021_3.rs +++ b/adapters/src/unity/unity_2021_3.rs @@ -1,358 +1,24 @@ -//! Unity 2021.3 Adapter +//! Unity 2021.3 adapter. //! -//! 支持 Unity 2021.3.x 版本的 AssetBundle +//! 该层只负责 Unity 版本选择；UnityFS 容器解析由 `bat-assetbundle` 引擎承担。 -use super::adapter::{ - ParsedAssetBundle, RawAssetBundle, UnityAdapter, UnityFsBlockInfo, UnityFsCompression, - UnityFsDirectoryInfo, UnityFsHeader, VersionRange, -}; +use super::adapter::{ParsedAssetBundle, RawAssetBundle, UnityAdapter, VersionRange}; use async_trait::async_trait; -use std::io::Cursor; +use bat_assetbundle::UnityFsParser; -const SERIALIZE_NOT_IMPLEMENTED: &str = "serialize() 将在 Phase 2 实现"; -const UNITYFS_COMPRESSION_MASK: u32 = 0x3f; -const UNITYFS_BLOCK_INFO_AT_END_FLAG: u32 = 0x80; -const UNITYFS_ALIGNMENT: usize = 16; +const SERIALIZE_NOT_IMPLEMENTED: &str = "serialize() 尚未实现"; -/// Unity 2021.3 Adapter +/// Unity 2021.3 adapter. pub struct Unity2021_3Adapter; impl Unity2021_3Adapter { - /// 创建新的适配器实例 + /// Creates an adapter instance. pub fn new() -> Self { Self } - fn unity_version_bytes(data: &[u8]) -> Option<&[u8]> { - if data.len() < 20 { - return None; - } - - if &data[0..7] != b"UnityFS" { - return None; - } - - let version_start = data.windows(7).position(|window| window == b"2021.3.")?; - let version_bytes = &data[version_start..]; - let version_end = version_bytes - .iter() - .position(|&byte| byte == 0 || !byte.is_ascii())?; - - Some(&version_bytes[..version_end]) - } - - /// 检测 Unity 版本（从文件头） fn detect_unity_version(data: &[u8]) -> Option { - let version_bytes = Self::unity_version_bytes(data)?; - std::str::from_utf8(version_bytes) - .map(ToOwned::to_owned) - .ok() - } - - fn parse_unityfs(data: &[u8]) -> Result { - let mut reader = UnityFsReader::new(data); - let signature = reader.read_c_string("signature")?; - if signature != "UnityFS" { - return Err(format!("Unsupported AssetBundle signature: {}", signature)); - } - - let format_version = reader.read_u32("format_version")?; - let target_version = reader.read_c_string("target_version")?; - let unity_version = reader.read_c_string("unity_version")?; - let total_size = reader.read_u64("total_size")?; - let compressed_blocks_info_size = reader.read_u32("compressed_blocks_info_size")?; - let uncompressed_blocks_info_size = reader.read_u32("uncompressed_blocks_info_size")?; - let flags = reader.read_u32("flags")?; - - let header = UnityFsHeader { - format_version, - target_version, - unity_version, - total_size, - compressed_blocks_info_size, - uncompressed_blocks_info_size, - flags, - }; - - if format_version >= 7 { - reader.align(UNITYFS_ALIGNMENT)?; - } - - let blocks_info_bytes = read_blocks_info_bytes(data, &mut reader, &header)?; - let block_info = decompress_blocks_info( - blocks_info_bytes, - compressed_blocks_info_size, - uncompressed_blocks_info_size, - flags, - )?; - let (blocks, directories) = Self::parse_blocks_info(&block_info)?; - validate_directory_bounds(&blocks, &directories)?; - - Ok(ParsedAssetBundle { - unity_version: header.unity_version.clone(), - assets: directories - .iter() - .map(|directory| directory.path.clone()) - .collect(), - raw_data: data.to_vec(), - unityfs_header: Some(header), - blocks, - directories, - }) - } - - fn parse_blocks_info( - data: &[u8], - ) -> Result<(Vec, Vec), String> { - let mut reader = UnityFsReader::new(data); - let _hash = reader.read_bytes(16, "blocks_info_hash")?; - let block_count = reader.read_i32("block_count")?; - if block_count < 0 { - return Err(format!("Invalid UnityFS block count: {}", block_count)); - } - - let mut blocks = Vec::with_capacity(block_count as usize); - for _ in 0..block_count { - let uncompressed_size = reader.read_u32("block_uncompressed_size")?; - let compressed_size = reader.read_u32("block_compressed_size")?; - let flags = reader.read_u16("block_flags")?; - blocks.push(UnityFsBlockInfo { - uncompressed_size, - compressed_size, - flags, - compression: compression_from_flags(flags), - }); - } - - let directory_count = reader.read_i32("directory_count")?; - if directory_count < 0 { - return Err(format!( - "Invalid UnityFS directory count: {}", - directory_count - )); - } - - let mut directories = Vec::with_capacity(directory_count as usize); - for _ in 0..directory_count { - directories.push(UnityFsDirectoryInfo { - offset: reader.read_u64("directory_offset")?, - size: reader.read_u64("directory_size")?, - flags: reader.read_u32("directory_flags")?, - path: reader.read_c_string("directory_path")?, - }); - } - - Ok((blocks, directories)) - } -} - -/// 校验目录条目落在解压数据区内。 -/// -/// UnityFS 的 directory 是解压后（所有 block 的 uncompressed 数据依次拼接而成的） -/// 连续数据区上的 `[offset, offset + size)` 切片。解析阶段只按结构读取这些数值， -/// 并不保证它们不越界；截断或损坏的 bundle 会给出指向数据区之外的目录条目， -/// 若不校验就静默接受，后续按 offset/size 取数据时才会出错或读到错误内容。 -/// 这里把每个目录条目与「各 block 解压大小之和」比对，越界即报错并带上下文。 -fn validate_directory_bounds( - blocks: &[UnityFsBlockInfo], - directories: &[UnityFsDirectoryInfo], -) -> Result<(), String> { - let mut data_region_size: u64 = 0; - for (index, block) in blocks.iter().enumerate() { - data_region_size = data_region_size - .checked_add(u64::from(block.uncompressed_size)) - .ok_or_else(|| { - format!( - "UnityFS 解压数据区大小溢出：累加到第 {index} 个 block（uncompressed_size={}）时超过 u64", - block.uncompressed_size - ) - })?; - } - - for (index, directory) in directories.iter().enumerate() { - let end = directory - .offset - .checked_add(directory.size) - .ok_or_else(|| { - format!( - "UnityFS 目录条目 {} 的 offset({}) + size({}) 溢出 u64", - directory.path, directory.offset, directory.size - ) - })?; - if end > data_region_size { - return Err(format!( - "UnityFS 目录条目 {}（第 {index} 项）越界：offset({}) + size({}) = {} 超过解压数据区大小 {}", - directory.path, directory.offset, directory.size, end, data_region_size - )); - } - } - - Ok(()) -} - -fn read_blocks_info_bytes<'a>( - data: &'a [u8], - reader: &mut UnityFsReader<'a>, - header: &UnityFsHeader, -) -> Result<&'a [u8], String> { - let len = header.compressed_blocks_info_size as usize; - if blocks_info_at_end(header.flags) { - let start = data.len().checked_sub(len).ok_or_else(|| { - format!( - "UnityFS block info at end underflow: compressed size {}, file size {}", - len, - data.len() - ) - })?; - return Ok(&data[start..]); - } - - reader.read_bytes(len, "blocks_info") -} - -fn blocks_info_at_end(flags: u32) -> bool { - flags & UNITYFS_BLOCK_INFO_AT_END_FLAG != 0 -} - -fn decompress_blocks_info( - data: &[u8], - compressed_size: u32, - uncompressed_size: u32, - flags: u32, -) -> Result, String> { - if data.len() != compressed_size as usize { - return Err(format!( - "UnityFS block info size mismatch: header says {}, read {}", - compressed_size, - data.len() - )); - } - - let compression = compression_from_flags((flags & UNITYFS_COMPRESSION_MASK) as u16); - match compression { - UnityFsCompression::None => { - if compressed_size != uncompressed_size { - return Err(format!( - "Uncompressed UnityFS block info size mismatch: compressed {} != uncompressed {}", - compressed_size, uncompressed_size - )); - } - Ok(data.to_vec()) - } - UnityFsCompression::Lz4 | UnityFsCompression::Lz4Hc => { - lz4::block::decompress(data, Some(uncompressed_size as i32)) - .map_err(|error| format!("Failed to decompress UnityFS LZ4 block info: {}", error)) - } - UnityFsCompression::Lzma => { - let mut output = Vec::with_capacity(uncompressed_size as usize); - lzma_rs::lzma_decompress(&mut Cursor::new(data), &mut output).map_err(|error| { - format!("Failed to decompress UnityFS LZMA block info: {}", error) - })?; - if output.len() != uncompressed_size as usize { - return Err(format!( - "UnityFS LZMA block info size mismatch: expected {}, got {}", - uncompressed_size, - output.len() - )); - } - Ok(output) - } - UnityFsCompression::Unknown(value) => Err(format!( - "Unsupported UnityFS block info compression flag: {}", - value - )), - } -} - -fn compression_from_flags(flags: u16) -> UnityFsCompression { - match flags & UNITYFS_COMPRESSION_MASK as u16 { - 0 => UnityFsCompression::None, - 1 => UnityFsCompression::Lzma, - 2 => UnityFsCompression::Lz4, - 3 | 4 => UnityFsCompression::Lz4Hc, - value => UnityFsCompression::Unknown(value), - } -} - -struct UnityFsReader<'a> { - data: &'a [u8], - offset: usize, -} - -impl<'a> UnityFsReader<'a> { - fn new(data: &'a [u8]) -> Self { - Self { data, offset: 0 } - } - - fn read_bytes(&mut self, len: usize, field: &str) -> Result<&'a [u8], String> { - let end = self - .offset - .checked_add(len) - .ok_or_else(|| format!("{} length overflow at offset {}", field, self.offset))?; - if end > self.data.len() { - return Err(format!( - "Unexpected end while reading {} at offset {}: need {}, have {}", - field, - self.offset, - len, - self.data.len().saturating_sub(self.offset) - )); - } - - let bytes = &self.data[self.offset..end]; - self.offset = end; - Ok(bytes) - } - - fn align(&mut self, alignment: usize) -> Result<(), String> { - if alignment == 0 { - return Ok(()); - } - - let remainder = self.offset % alignment; - if remainder == 0 { - return Ok(()); - } - - let padding = alignment - remainder; - self.read_bytes(padding, "alignment padding").map(|_| ()) - } - - fn read_u16(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(2, field)?; - Ok(u16::from_be_bytes([bytes[0], bytes[1]])) - } - - fn read_u32(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(4, field)?; - Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) - } - - fn read_i32(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(4, field)?; - Ok(i32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) - } - - fn read_u64(&mut self, field: &str) -> Result { - let bytes = self.read_bytes(8, field)?; - Ok(u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ])) - } - - fn read_c_string(&mut self, field: &str) -> Result { - let remaining = &self.data[self.offset..]; - let Some(length) = remaining.iter().position(|&byte| byte == 0) else { - return Err(format!( - "Missing null terminator while reading {} at offset {}", - field, self.offset - )); - }; - let bytes = self.read_bytes(length, field)?; - self.offset += 1; - std::str::from_utf8(bytes) - .map(ToOwned::to_owned) - .map_err(|error| format!("Invalid UTF-8 in {}: {}", field, error)) + UnityFsParser::detect_unity_version(data) } } @@ -373,16 +39,15 @@ impl UnityAdapter for Unity2021_3Adapter { } fn can_handle(&self, bundle: &RawAssetBundle) -> bool { - // 检测 Unity 版本 - if let Some(version) = Self::detect_unity_version(&bundle.data) { - self.supported_versions().contains(&version) - } else { - false - } + Self::detect_unity_version(&bundle.data) + .map(|version| self.supported_versions().contains(&version)) + .unwrap_or(false) } async fn parse(&self, bundle: &RawAssetBundle) -> Result { - let parsed = Self::parse_unityfs(&bundle.data)?; + let parsed = UnityFsParser::new() + .parse_asset_bundle(bundle) + .map_err(|error| error.to_string())?; if !self.supported_versions().contains(&parsed.unity_version) { return Err(format!( "Unsupported Unity version for {}: {}", @@ -394,12 +59,6 @@ impl UnityAdapter for Unity2021_3Adapter { } async fn serialize(&self, _parsed: &ParsedAssetBundle) -> Result, String> { - // TODO: Phase 2 实现 - // 需要： - // 1. 序列化 Asset 对象 - // 2. 重新构建 TypeTree - // 3. 压缩数据块 - // 4. 写入 UnityFS 文件头 Err(SERIALIZE_NOT_IMPLEMENTED.to_string()) } } @@ -407,6 +66,9 @@ impl UnityAdapter for Unity2021_3Adapter { #[cfg(test)] mod tests { use super::*; + use crate::unity::UnityFsCompression; + + const UNITYFS_ALIGNMENT: usize = 16; fn push_c_string(data: &mut Vec, value: &str) { data.extend_from_slice(value.as_bytes()); @@ -436,7 +98,7 @@ mod tests { } } - fn synthetic_minimal_unityfs_bundle() -> Vec { + fn synthetic_minimal_unityfs_bundle(unity_version: &str) -> Vec { let mut blocks_info = Vec::new(); blocks_info.extend_from_slice(&[0; 16]); push_i32(&mut blocks_info, 1); @@ -453,7 +115,7 @@ mod tests { push_c_string(&mut data, "UnityFS"); push_u32(&mut data, 8); push_c_string(&mut data, "5.x.x"); - push_c_string(&mut data, "2021.3.56f2"); + push_c_string(&mut data, unity_version); push_u64(&mut data, 0); push_u32(&mut data, blocks_info.len() as u32); push_u32(&mut data, blocks_info.len() as u32); @@ -463,7 +125,7 @@ mod tests { data.extend_from_slice(b"data"); let total_size = data.len() as u64; - let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + unity_version.len() + 1; data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); data } @@ -490,7 +152,7 @@ mod tests { let adapter = Unity2021_3Adapter::new(); let bundle = RawAssetBundle { - data: synthetic_minimal_unityfs_bundle(), + data: synthetic_minimal_unityfs_bundle("2021.3.56f2"), path: Some("synthetic-minimal.bundle".to_string()), }; @@ -514,7 +176,7 @@ mod tests { let adapter = Unity2021_3Adapter::new(); let bundle = RawAssetBundle { - data: synthetic_minimal_unityfs_bundle(), + data: synthetic_minimal_unityfs_bundle("2021.3.56f2"), path: Some("synthetic-minimal.bundle".to_string()), }; @@ -538,85 +200,34 @@ mod tests { path: None, }; - let result = adapter.parse(&bundle).await; - assert!(result.is_err()); - assert!(result.unwrap_err().contains("signature")); - } - - fn block(uncompressed_size: u32) -> UnityFsBlockInfo { - UnityFsBlockInfo { - uncompressed_size, - compressed_size: uncompressed_size, - flags: 0, - compression: UnityFsCompression::None, - } - } - - fn directory(offset: u64, size: u64) -> UnityFsDirectoryInfo { - UnityFsDirectoryInfo { - offset, - size, - flags: 0, - path: "CAB-test".to_string(), - } - } - - #[test] - fn directory_within_data_region_is_accepted() { - // 两个 block 共 12 字节解压数据区；目录条目正好覆盖尾部，合法。 - let result = - validate_directory_bounds(&[block(8), block(4)], &[directory(0, 8), directory(8, 4)]); - assert!(result.is_ok(), "{result:?}"); - } - - #[test] - fn directory_past_data_region_is_rejected() { - // 解压数据区仅 4 字节，目录声称 [0, 8) 越界，应被拒绝并带上下文。 - let error = validate_directory_bounds(&[block(4)], &[directory(0, 8)]).unwrap_err(); - assert!(error.contains("越界"), "{error}"); - assert!(error.contains("解压数据区大小 4"), "{error}"); - } - - #[test] - fn directory_offset_size_overflow_is_rejected() { - let error = validate_directory_bounds(&[block(4)], &[directory(u64::MAX, 1)]).unwrap_err(); - assert!(error.contains("溢出"), "{error}"); + let error = adapter.parse(&bundle).await.unwrap_err(); + assert!(error.contains("signature"), "{error}"); } #[tokio::test] - async fn test_parse_rejects_out_of_bounds_directory() { - // 构造一个 directory.size 超过 block 解压大小的 bundle，端到端验证被拒。 - let mut blocks_info = Vec::new(); - blocks_info.extend_from_slice(&[0; 16]); - push_i32(&mut blocks_info, 1); - push_u32(&mut blocks_info, 4); // block uncompressed_size = 4 - push_u32(&mut blocks_info, 4); - push_u16(&mut blocks_info, 0); - push_i32(&mut blocks_info, 1); - push_u64(&mut blocks_info, 0); - push_u64(&mut blocks_info, 99); // directory size 99 远超数据区 - push_u32(&mut blocks_info, 0); - push_c_string(&mut blocks_info, "CAB-test"); + async fn test_parse_rejects_unsupported_unity_version() { + let adapter = Unity2021_3Adapter::new(); - let mut data = Vec::new(); - push_c_string(&mut data, "UnityFS"); - push_u32(&mut data, 8); - push_c_string(&mut data, "5.x.x"); - push_c_string(&mut data, "2021.3.56f2"); - push_u64(&mut data, 0); - push_u32(&mut data, blocks_info.len() as u32); - push_u32(&mut data, blocks_info.len() as u32); - push_u32(&mut data, 0); - align(&mut data, UNITYFS_ALIGNMENT); - data.extend_from_slice(&blocks_info); - data.extend_from_slice(b"data"); + let bundle = RawAssetBundle { + data: synthetic_minimal_unityfs_bundle("2022.3.1f1"), + path: Some("unsupported.bundle".to_string()), + }; + let error = adapter.parse(&bundle).await.unwrap_err(); + assert!(error.contains("Unsupported Unity version"), "{error}"); + } + + #[tokio::test] + async fn serialize_returns_explicit_not_implemented_error() { let adapter = Unity2021_3Adapter::new(); let bundle = RawAssetBundle { - data, - path: Some("out-of-bounds.bundle".to_string()), + data: synthetic_minimal_unityfs_bundle("2021.3.56f2"), + path: Some("synthetic-minimal.bundle".to_string()), }; - let error = adapter.parse(&bundle).await.unwrap_err(); - assert!(error.contains("越界"), "{error}"); + let parsed = adapter.parse(&bundle).await.unwrap(); + + let error = adapter.serialize(&parsed).await.unwrap_err(); + + assert_eq!(error, SERIALIZE_NOT_IMPLEMENTED); } } diff --git a/adapters/tests/golden/real_addressables.json b/adapters/tests/golden/real_addressables.json index 5c5efd0..0894331 100644 --- a/adapters/tests/golden/real_addressables.json +++ b/adapters/tests/golden/real_addressables.json @@ -35,7 +35,11 @@ } ], "metadata": { + "asset_bundle_count": "3", + "declared_size_count": "3", + "dependency_count": "1", "internal_id_count": "3", + "resource_count": "3", "resource_type_count": "1", "key_object_count": "4", "bucket_record_count": "4", diff --git a/adapters/tests/real_unityfs_local.rs b/adapters/tests/real_unityfs_local.rs index 03ca8b0..b649354 100644 --- a/adapters/tests/real_unityfs_local.rs +++ b/adapters/tests/real_unityfs_local.rs @@ -21,4 +21,6 @@ async fn parses_local_real_unityfs_bundle() { assert_eq!(parsed.unity_version, "2021.3.56f2"); assert!(!parsed.blocks.is_empty()); assert!(!parsed.directories.is_empty()); + assert_eq!(parsed.files.len(), parsed.directories.len()); + assert!(parsed.files.iter().all(|file| !file.data.is_empty())); } diff --git a/crates/bat-assetbundle/Cargo.toml b/crates/bat-assetbundle/Cargo.toml index 7525d87..4b91b89 100644 --- a/crates/bat-assetbundle/Cargo.toml +++ b/crates/bat-assetbundle/Cargo.toml @@ -11,9 +11,8 @@ thiserror.workspace = true serde.workspace = true serde_json.workspace = true tracing.workspace = true - -# 注意：byteorder、lz4、lzma-rs 等 UnityFS 解析/解压依赖待解析器真正实现时 -# 再按需引入，避免占位阶段白增编译负担。 +lz4 = "1.28" +lzma-rs = "0.3" [dev-dependencies] hex = "0.4" diff --git a/crates/bat-assetbundle/src/error.rs b/crates/bat-assetbundle/src/error.rs index 66ecc49..f94f669 100644 --- a/crates/bat-assetbundle/src/error.rs +++ b/crates/bat-assetbundle/src/error.rs @@ -1,26 +1,52 @@ -//! AssetBundle 错误类型定义 +//! AssetBundle error types. use thiserror::Error; -/// AssetBundle 错误类型 +/// AssetBundle parser error. #[derive(Error, Debug)] pub enum AssetBundleError { - /// IO 错误 + /// I/O error. #[error("IO error: {0}")] Io(#[from] std::io::Error), - /// 解析错误 + /// Parser reached malformed data while reading a named field. + #[error("Parse error at offset {offset} while reading {field}: {message}")] + ParseField { + /// Field or structure name being read. + field: String, + /// Byte offset where parsing failed. + offset: usize, + /// Human-readable diagnostic. + message: String, + }, + + /// General parser error. #[error("Parse error: {0}")] Parse(String), - /// 不支持的格式 + /// Unsupported format or compression mode. #[error("Unsupported format: {0}")] UnsupportedFormat(String), - /// 其他错误 + /// Other error. #[error(transparent)] Other(#[from] anyhow::Error), } -/// AssetBundle Result 类型 +impl AssetBundleError { + /// Creates a field-scoped parser error with byte offset context. + pub fn parse_field( + field: impl Into, + offset: usize, + message: impl Into, + ) -> Self { + Self::ParseField { + field: field.into(), + offset, + message: message.into(), + } + } +} + +/// AssetBundle result type. pub type Result = std::result::Result; diff --git a/crates/bat-assetbundle/src/lib.rs b/crates/bat-assetbundle/src/lib.rs index ce3fe1d..e38d29f 100644 --- a/crates/bat-assetbundle/src/lib.rs +++ b/crates/bat-assetbundle/src/lib.rs @@ -9,9 +9,20 @@ pub mod error; pub mod parser; +pub mod serialized; pub mod types; pub use error::{AssetBundleError, Result}; +pub use parser::{compression_from_flags, Parser, UnityFsParser}; +pub use serialized::{ + UnitySerializedFile, UnitySerializedObject, UnitySerializedTextAsset, UnitySerializedType, + UnityTypeTreeNode, +}; +pub use types::{ + AssetType, ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle, + UnityFsCompression, UnityFsDirectoryInfo, UnityFsFile, UnityFsHeader, + UnitySerializedParseError, +}; /// AssetBundle 解析器版本号 pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/bat-assetbundle/src/parser.rs b/crates/bat-assetbundle/src/parser.rs index 0e035cf..6798c93 100644 --- a/crates/bat-assetbundle/src/parser.rs +++ b/crates/bat-assetbundle/src/parser.rs @@ -1,7 +1,1026 @@ -//! 解析器模块占位 +//! Unity AssetBundle parser implementations. -/// AssetBundle 解析器接口（待实现） +use crate::error::{AssetBundleError, Result}; +use crate::serialized::UnitySerializedFile; +use crate::types::{ + ParsedAssetBundle, RawAssetBundle, UnityFsBlockInfo, UnityFsBundle, UnityFsCompression, + UnityFsDirectoryInfo, UnityFsFile, UnityFsHeader, UnitySerializedParseError, +}; +use std::io::Cursor; + +const UNITYFS_COMPRESSION_MASK: u32 = 0x3f; +const UNITYFS_COMPRESSION_MASK_U16: u16 = 0x3f; +const UNITYFS_BLOCK_INFO_AT_END_FLAG: u32 = 0x80; +const UNITYFS_BLOCK_DATA_ALIGNED_FLAG: u32 = 0x200; +const UNITYFS_ALIGNMENT: usize = 16; + +/// AssetBundle parser interface. pub trait Parser { - /// 解析器名称 + /// Parser name. fn name(&self) -> &str; + + /// Returns true when this parser can handle the bytes. + fn can_parse(&self, data: &[u8]) -> bool; + + /// Parses raw bytes into a structured bundle summary. + fn parse(&self, data: &[u8]) -> Result; +} + +/// UnityFS container parser. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnityFsParser; + +impl UnityFsParser { + /// Creates a parser instance. + pub fn new() -> Self { + Self + } + + /// Returns true if the byte stream starts with the UnityFS signature. + pub fn has_unityfs_signature(data: &[u8]) -> bool { + data.starts_with(b"UnityFS\0") + } + + /// Detects the Unity version string from a UnityFS header without parsing + /// the full block table. + pub fn detect_unity_version(data: &[u8]) -> Option { + if !Self::has_unityfs_signature(data) { + return None; + } + + let mut reader = UnityFsReader::new(data); + let signature = reader.read_c_string("signature").ok()?; + if signature != "UnityFS" { + return None; + } + reader.read_u32("format_version").ok()?; + reader.read_c_string("target_version").ok()?; + reader.read_c_string("unity_version").ok() + } + + /// Parses a raw bundle wrapper. + pub fn parse_asset_bundle(&self, bundle: &RawAssetBundle) -> Result { + self.parse(&bundle.data) + } + + /// Parses UnityFS bytes into the engine-level container representation. + pub fn parse_bytes(&self, data: &[u8]) -> Result { + parse_unityfs(data) + } +} + +impl Parser for UnityFsParser { + fn name(&self) -> &str { + "UnityFS" + } + + fn can_parse(&self, data: &[u8]) -> bool { + Self::has_unityfs_signature(data) + } + + fn parse(&self, data: &[u8]) -> Result { + self.parse_bytes(data).map(Into::into) + } +} + +/// Decodes UnityFS compression bits from raw flags. +pub fn compression_from_flags(flags: u16) -> UnityFsCompression { + match flags & UNITYFS_COMPRESSION_MASK_U16 { + 0 => UnityFsCompression::None, + 1 => UnityFsCompression::Lzma, + 2 => UnityFsCompression::Lz4, + 3 | 4 => UnityFsCompression::Lz4Hc, + value => UnityFsCompression::Unknown(value), + } +} + +fn parse_unityfs(data: &[u8]) -> Result { + let mut reader = UnityFsReader::new(data); + let signature = reader.read_c_string("signature")?; + if signature != "UnityFS" { + return Err(AssetBundleError::UnsupportedFormat(format!( + "unsupported AssetBundle signature: {signature}" + ))); + } + + let header = UnityFsHeader { + format_version: reader.read_u32("format_version")?, + target_version: reader.read_c_string("target_version")?, + unity_version: reader.read_c_string("unity_version")?, + total_size: reader.read_u64("total_size")?, + compressed_blocks_info_size: reader.read_u32("compressed_blocks_info_size")?, + uncompressed_blocks_info_size: reader.read_u32("uncompressed_blocks_info_size")?, + flags: reader.read_u32("flags")?, + }; + + if header.total_size > data.len() as u64 { + return Err(AssetBundleError::Parse(format!( + "UnityFS total_size {} exceeds file size {}", + header.total_size, + data.len() + ))); + } + + if header.format_version >= 7 { + reader.align(UNITYFS_ALIGNMENT)?; + } + + let blocks_info_inline_offset = reader.offset(); + let blocks_info_bytes = read_blocks_info_bytes(data, &mut reader, &header)?; + let blocks_info_len = blocks_info_bytes.len(); + let block_info = decompress_blocks_info( + blocks_info_bytes, + header.compressed_blocks_info_size, + header.uncompressed_blocks_info_size, + header.flags, + )?; + let (blocks_info_hash, blocks, directories) = parse_blocks_info(&block_info)?; + let mut data_start_offset = if blocks_info_at_end(header.flags) { + blocks_info_inline_offset + } else { + reader.offset() + }; + if block_data_aligned(header.flags) { + data_start_offset = align_offset(data_start_offset, UNITYFS_ALIGNMENT)?; + } + let compressed_data_size = sum_compressed_block_sizes(&blocks)?; + let uncompressed_data_size = sum_uncompressed_block_sizes(&blocks)?; + validate_compressed_region( + data.len(), + data_start_offset, + compressed_data_size, + blocks_info_at_end(header.flags), + blocks_info_len, + )?; + validate_directory_bounds(uncompressed_data_size, &directories)?; + let uncompressed_data = decompress_data_blocks(data, data_start_offset, &blocks)?; + if uncompressed_data.len() as u64 != uncompressed_data_size { + return Err(AssetBundleError::Parse(format!( + "UnityFS uncompressed data size mismatch: expected {}, got {}", + uncompressed_data_size, + uncompressed_data.len() + ))); + } + let files = extract_files(&uncompressed_data, &directories)?; + let (serialized_files, text_assets, serialized_parse_errors) = parse_serialized_files(&files); + + Ok(UnityFsBundle { + header, + blocks_info_hash, + blocks, + directories, + data_start_offset: data_start_offset as u64, + compressed_data_size, + uncompressed_data_size, + raw_data: data.to_vec(), + files, + serialized_files, + text_assets, + serialized_parse_errors, + }) +} + +fn read_blocks_info_bytes<'a>( + data: &'a [u8], + reader: &mut UnityFsReader<'a>, + header: &UnityFsHeader, +) -> Result<&'a [u8]> { + let len = header.compressed_blocks_info_size as usize; + if blocks_info_at_end(header.flags) { + let start = data.len().checked_sub(len).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS block info at end underflow: compressed size {}, file size {}", + len, + data.len() + )) + })?; + return Ok(&data[start..]); + } + + reader.read_bytes(len, "blocks_info") +} + +fn blocks_info_at_end(flags: u32) -> bool { + flags & UNITYFS_BLOCK_INFO_AT_END_FLAG != 0 +} + +fn block_data_aligned(flags: u32) -> bool { + flags & UNITYFS_BLOCK_DATA_ALIGNED_FLAG != 0 +} + +fn align_offset(offset: usize, alignment: usize) -> Result { + if alignment == 0 { + return Ok(offset); + } + let remainder = offset % alignment; + if remainder == 0 { + return Ok(offset); + } + offset.checked_add(alignment - remainder).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS aligned offset overflow: offset {offset}, alignment {alignment}" + )) + }) +} + +fn decompress_blocks_info( + data: &[u8], + compressed_size: u32, + uncompressed_size: u32, + flags: u32, +) -> Result> { + if data.len() != compressed_size as usize { + return Err(AssetBundleError::Parse(format!( + "UnityFS block info size mismatch: header says {}, read {}", + compressed_size, + data.len() + ))); + } + + decompress_unityfs_bytes( + data, + compressed_size, + uncompressed_size, + compression_from_flags((flags & UNITYFS_COMPRESSION_MASK) as u16), + "UnityFS block info", + ) +} + +fn decompress_unityfs_bytes( + data: &[u8], + compressed_size: u32, + uncompressed_size: u32, + compression: UnityFsCompression, + context: &str, +) -> Result> { + match compression { + UnityFsCompression::None => { + if compressed_size != uncompressed_size { + return Err(AssetBundleError::Parse(format!( + "{context} uncompressed size mismatch: compressed {} != uncompressed {}", + compressed_size, uncompressed_size + ))); + } + Ok(data.to_vec()) + } + UnityFsCompression::Lz4 | UnityFsCompression::Lz4Hc => { + lz4::block::decompress(data, Some(uncompressed_size as i32)).map_err(|error| { + AssetBundleError::Parse(format!("Failed to decompress {context} as LZ4: {error}")) + }) + } + UnityFsCompression::Lzma => { + let mut output = Vec::with_capacity(uncompressed_size as usize); + lzma_rs::lzma_decompress(&mut Cursor::new(data), &mut output).map_err(|error| { + AssetBundleError::Parse(format!("Failed to decompress {context} as LZMA: {error}")) + })?; + if output.len() != uncompressed_size as usize { + return Err(AssetBundleError::Parse(format!( + "{context} LZMA size mismatch: expected {}, got {}", + uncompressed_size, + output.len() + ))); + } + Ok(output) + } + UnityFsCompression::Unknown(value) => Err(AssetBundleError::UnsupportedFormat(format!( + "unsupported {context} compression flag: {value}" + ))), + } +} + +fn decompress_data_blocks( + data: &[u8], + data_start_offset: usize, + blocks: &[UnityFsBlockInfo], +) -> Result> { + let payload = data.get(data_start_offset..).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS data_start_offset {} exceeds file size {}", + data_start_offset, + data.len() + )) + })?; + let mut reader = UnityFsReader::new(payload); + let uncompressed_capacity = sum_uncompressed_block_sizes(blocks)?; + let mut output = Vec::with_capacity(usize::try_from(uncompressed_capacity).unwrap_or(0)); + + for (index, block) in blocks.iter().enumerate() { + let compressed = reader.read_bytes( + block.compressed_size as usize, + &format!("data_block_{index}"), + )?; + let bytes = decompress_unityfs_bytes( + compressed, + block.compressed_size, + block.uncompressed_size, + block.compression, + &format!("UnityFS data block {index}"), + )?; + if bytes.len() != block.uncompressed_size as usize { + return Err(AssetBundleError::Parse(format!( + "UnityFS data block {index} size mismatch: expected {}, got {}", + block.uncompressed_size, + bytes.len() + ))); + } + output.extend_from_slice(&bytes); + } + + Ok(output) +} + +fn extract_files( + uncompressed_data: &[u8], + directories: &[UnityFsDirectoryInfo], +) -> Result> { + let mut files = Vec::with_capacity(directories.len()); + for directory in directories { + let start = usize::try_from(directory.offset).map_err(|_| { + AssetBundleError::Parse(format!( + "UnityFS directory {} offset {} does not fit usize", + directory.path, directory.offset + )) + })?; + let size = usize::try_from(directory.size).map_err(|_| { + AssetBundleError::Parse(format!( + "UnityFS directory {} size {} does not fit usize", + directory.path, directory.size + )) + })?; + let end = start.checked_add(size).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS directory {} start + size overflows usize", + directory.path + )) + })?; + let data = uncompressed_data.get(start..end).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS directory {} byte range {}..{} exceeds uncompressed data {}", + directory.path, + start, + end, + uncompressed_data.len() + )) + })?; + files.push(UnityFsFile { + path: directory.path.clone(), + offset: directory.offset, + size: directory.size, + flags: directory.flags, + data: data.to_vec(), + }); + } + Ok(files) +} + +fn parse_serialized_files( + files: &[UnityFsFile], +) -> ( + Vec, + Vec, + Vec, +) { + let mut serialized_files = Vec::new(); + let mut text_assets = Vec::new(); + let mut serialized_parse_errors = Vec::new(); + + for file in files { + if !looks_like_serialized_file(&file.data) { + continue; + } + match UnitySerializedFile::from_named_slice(Some(file.path.clone()), &file.data) { + Ok(serialized_file) => { + text_assets.extend(serialized_file.text_assets().iter().cloned()); + serialized_files.push(serialized_file); + } + Err(error) => serialized_parse_errors.push(UnitySerializedParseError { + path: file.path.clone(), + error: error.to_string(), + }), + } + } + + (serialized_files, text_assets, serialized_parse_errors) +} + +fn looks_like_serialized_file(data: &[u8]) -> bool { + if data.len() < 20 { + return false; + } + + let version = u32::from_be_bytes([data[8], data[9], data[10], data[11]]); + if !(5..=30).contains(&version) { + return false; + } + + if version >= 22 { + if data.len() < 48 { + return false; + } + let file_size = u64::from_be_bytes([ + data[24], data[25], data[26], data[27], data[28], data[29], data[30], data[31], + ]); + let data_offset = u64::from_be_bytes([ + data[32], data[33], data[34], data[35], data[36], data[37], data[38], data[39], + ]); + file_size <= data.len() as u64 && data_offset <= data.len() as u64 + } else { + let file_size = u32::from_be_bytes([data[4], data[5], data[6], data[7]]); + let data_offset = u32::from_be_bytes([data[12], data[13], data[14], data[15]]); + file_size as usize <= data.len() && data_offset as usize <= data.len() + } +} + +fn parse_blocks_info( + data: &[u8], +) -> Result<([u8; 16], Vec, Vec)> { + let mut reader = UnityFsReader::new(data); + let hash = reader.read_bytes(16, "blocks_info_hash")?; + let blocks_info_hash: [u8; 16] = hash + .try_into() + .map_err(|_| AssetBundleError::parse_field("blocks_info_hash", 0, "expected 16 bytes"))?; + + let block_count = reader.read_i32("block_count")?; + if block_count < 0 { + return Err(AssetBundleError::Parse(format!( + "invalid UnityFS block count: {block_count}" + ))); + } + + let mut blocks = Vec::with_capacity(block_count as usize); + for _ in 0..block_count { + let uncompressed_size = reader.read_u32("block_uncompressed_size")?; + let compressed_size = reader.read_u32("block_compressed_size")?; + let flags = reader.read_u16("block_flags")?; + blocks.push(UnityFsBlockInfo { + uncompressed_size, + compressed_size, + flags, + compression: compression_from_flags(flags), + }); + } + + let directory_count = reader.read_i32("directory_count")?; + if directory_count < 0 { + return Err(AssetBundleError::Parse(format!( + "invalid UnityFS directory count: {directory_count}" + ))); + } + + let mut directories = Vec::with_capacity(directory_count as usize); + for _ in 0..directory_count { + directories.push(UnityFsDirectoryInfo { + offset: reader.read_u64("directory_offset")?, + size: reader.read_u64("directory_size")?, + flags: reader.read_u32("directory_flags")?, + path: reader.read_c_string("directory_path")?, + }); + } + + Ok((blocks_info_hash, blocks, directories)) +} + +fn sum_compressed_block_sizes(blocks: &[UnityFsBlockInfo]) -> Result { + let mut total = 0u64; + for (index, block) in blocks.iter().enumerate() { + total = total + .checked_add(u64::from(block.compressed_size)) + .ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS compressed data size overflow at block {index}" + )) + })?; + } + Ok(total) +} + +fn sum_uncompressed_block_sizes(blocks: &[UnityFsBlockInfo]) -> Result { + let mut total = 0u64; + for (index, block) in blocks.iter().enumerate() { + total = total + .checked_add(u64::from(block.uncompressed_size)) + .ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS uncompressed data size overflow at block {index}" + )) + })?; + } + Ok(total) +} + +fn validate_compressed_region( + file_size: usize, + data_start_offset: usize, + compressed_data_size: u64, + block_info_at_end: bool, + block_info_len: usize, +) -> Result<()> { + let end = data_start_offset + .checked_add(usize::try_from(compressed_data_size).map_err(|_| { + AssetBundleError::Parse(format!( + "UnityFS compressed data size {} does not fit usize", + compressed_data_size + )) + })?) + .ok_or_else(|| AssetBundleError::Parse("UnityFS compressed data end overflow".into()))?; + + let max_end = if block_info_at_end { + file_size.checked_sub(block_info_len).ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS block info size {block_info_len} exceeds file size {file_size}" + )) + })? + } else { + file_size + }; + + if end > max_end { + return Err(AssetBundleError::Parse(format!( + "UnityFS compressed data region exceeds file bounds: start {data_start_offset}, size {compressed_data_size}, max_end {max_end}" + ))); + } + Ok(()) +} + +fn validate_directory_bounds( + data_region_size: u64, + directories: &[UnityFsDirectoryInfo], +) -> Result<()> { + for (index, directory) in directories.iter().enumerate() { + let end = directory + .offset + .checked_add(directory.size) + .ok_or_else(|| { + AssetBundleError::Parse(format!( + "UnityFS directory {} offset({}) + size({}) overflows u64", + directory.path, directory.offset, directory.size + )) + })?; + if end > data_region_size { + return Err(AssetBundleError::Parse(format!( + "UnityFS directory {} (index {index}) out of bounds: offset({}) + size({}) = {} exceeds uncompressed data region size {}", + directory.path, directory.offset, directory.size, end, data_region_size + ))); + } + } + + Ok(()) +} + +struct UnityFsReader<'a> { + data: &'a [u8], + offset: usize, +} + +impl<'a> UnityFsReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, offset: 0 } + } + + fn offset(&self) -> usize { + self.offset + } + + fn read_bytes(&mut self, len: usize, field: &str) -> Result<&'a [u8]> { + let start = self.offset; + let end = self.offset.checked_add(len).ok_or_else(|| { + AssetBundleError::parse_field(field, start, "field length overflows usize") + })?; + if end > self.data.len() { + return Err(AssetBundleError::parse_field( + field, + start, + format!( + "unexpected end: need {}, have {}", + len, + self.data.len().saturating_sub(start) + ), + )); + } + + let bytes = &self.data[self.offset..end]; + self.offset = end; + Ok(bytes) + } + + fn align(&mut self, alignment: usize) -> Result<()> { + if alignment == 0 { + return Ok(()); + } + + let remainder = self.offset % alignment; + if remainder == 0 { + return Ok(()); + } + + let padding = alignment - remainder; + self.read_bytes(padding, "alignment_padding").map(|_| ()) + } + + fn read_u16(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(2, field)?; + Ok(u16::from_be_bytes([bytes[0], bytes[1]])) + } + + fn read_u32(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(4, field)?; + Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + fn read_i32(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(4, field)?; + Ok(i32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + fn read_u64(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(8, field)?; + Ok(u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ])) + } + + fn read_c_string(&mut self, field: &str) -> Result { + let start = self.offset; + let remaining = &self.data[self.offset..]; + let Some(length) = remaining.iter().position(|&byte| byte == 0) else { + return Err(AssetBundleError::parse_field( + field, + start, + "missing null terminator", + )); + }; + let bytes = self.read_bytes(length, field)?; + self.offset += 1; + std::str::from_utf8(bytes) + .map(ToOwned::to_owned) + .map_err(|error| { + AssetBundleError::parse_field(field, start, format!("invalid UTF-8: {error}")) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn push_c_string(data: &mut Vec, value: &str) { + data.extend_from_slice(value.as_bytes()); + data.push(0); + } + + fn push_u16(data: &mut Vec, value: u16) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u32(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_i32(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u64(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_i16_le(data: &mut Vec, value: i16) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u32_le(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i32_le(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i64_le(data: &mut Vec, value: i64) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u64_le(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn align(data: &mut Vec, alignment: usize) { + let remainder = data.len() % alignment; + if remainder != 0 { + data.resize(data.len() + alignment - remainder, 0); + } + } + + fn blocks_info(directory_size: u64) -> Vec { + let mut blocks_info = Vec::new(); + blocks_info.extend_from_slice(&[0xAB; 16]); + push_i32(&mut blocks_info, 1); + push_u32(&mut blocks_info, 4); + push_u32(&mut blocks_info, 4); + push_u16(&mut blocks_info, 0); + push_i32(&mut blocks_info, 1); + push_u64(&mut blocks_info, 0); + push_u64(&mut blocks_info, directory_size); + push_u32(&mut blocks_info, 0); + push_c_string(&mut blocks_info, "CAB-test"); + blocks_info + } + + fn blocks_info_for_payload( + directory_path: &str, + uncompressed_size: u32, + compressed_size: u32, + block_flags: u16, + ) -> Vec { + let mut blocks_info = Vec::new(); + blocks_info.extend_from_slice(&[0xCD; 16]); + push_i32(&mut blocks_info, 1); + push_u32(&mut blocks_info, uncompressed_size); + push_u32(&mut blocks_info, compressed_size); + push_u16(&mut blocks_info, block_flags); + push_i32(&mut blocks_info, 1); + push_u64(&mut blocks_info, 0); + push_u64(&mut blocks_info, u64::from(uncompressed_size)); + push_u32(&mut blocks_info, 0); + push_c_string(&mut blocks_info, directory_path); + blocks_info + } + + fn synthetic_unityfs_bundle( + blocks_info: &[u8], + header_flags: u32, + block_info_at_end: bool, + ) -> Vec { + let mut data = Vec::new(); + push_c_string(&mut data, "UnityFS"); + push_u32(&mut data, 8); + push_c_string(&mut data, "5.x.x"); + push_c_string(&mut data, "2021.3.56f2"); + push_u64(&mut data, 0); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, header_flags); + align(&mut data, UNITYFS_ALIGNMENT); + + if block_info_at_end { + data.extend_from_slice(b"data"); + data.extend_from_slice(blocks_info); + } else { + data.extend_from_slice(blocks_info); + data.extend_from_slice(b"data"); + } + + let total_size = data.len() as u64; + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); + data + } + + fn synthetic_unityfs_bundle_with_payload( + directory_path: &str, + payload: &[u8], + block_payload: &[u8], + block_flags: u16, + ) -> Vec { + synthetic_unityfs_bundle_with_payload_and_header_flags( + directory_path, + payload, + block_payload, + block_flags, + 0, + ) + } + + fn synthetic_unityfs_bundle_with_payload_and_header_flags( + directory_path: &str, + payload: &[u8], + block_payload: &[u8], + block_flags: u16, + header_flags: u32, + ) -> Vec { + let blocks_info = blocks_info_for_payload( + directory_path, + payload.len() as u32, + block_payload.len() as u32, + block_flags, + ); + let mut data = Vec::new(); + push_c_string(&mut data, "UnityFS"); + push_u32(&mut data, 8); + push_c_string(&mut data, "5.x.x"); + push_c_string(&mut data, "2021.3.56f2"); + push_u64(&mut data, 0); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, header_flags); + align(&mut data, UNITYFS_ALIGNMENT); + data.extend_from_slice(&blocks_info); + if block_data_aligned(header_flags) { + align(&mut data, UNITYFS_ALIGNMENT); + } + data.extend_from_slice(block_payload); + + let total_size = data.len() as u64; + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); + data + } + + fn synthetic_serialized_text_asset() -> Vec { + let mut object_data = Vec::new(); + push_u32_le(&mut object_data, 8); + object_data.extend_from_slice(b"Scenario"); + align(&mut object_data, 4); + push_u32_le(&mut object_data, 15); + object_data.extend_from_slice("こんにちは".as_bytes()); + + let mut metadata = Vec::new(); + metadata.extend_from_slice(b"2021.3.56f2\0"); + push_i32_le(&mut metadata, 19); + metadata.push(0); + push_i32_le(&mut metadata, 1); + push_i32_le(&mut metadata, 49); + metadata.push(0); + push_i16_le(&mut metadata, 0); + metadata.extend_from_slice(&[0; 16]); + push_i32_le(&mut metadata, 1); + align(&mut metadata, 4); + push_i64_le(&mut metadata, 9); + push_u64_le(&mut metadata, 0); + push_u32_le(&mut metadata, object_data.len() as u32); + push_i32_le(&mut metadata, 0); + + let header_len = 48usize; + let data_offset = header_len + metadata.len(); + let file_size = data_offset + object_data.len(); + + let mut file = Vec::new(); + push_u32(&mut file, metadata.len() as u32); + push_u32(&mut file, file_size as u32); + push_u32(&mut file, 22); + push_u32(&mut file, 0); + file.push(0); + file.extend_from_slice(&[0, 0, 0]); + push_u32(&mut file, metadata.len() as u32); + push_u64(&mut file, file_size as u64); + push_u64(&mut file, data_offset as u64); + push_u64(&mut file, 0); + file.extend_from_slice(&metadata); + file.extend_from_slice(&object_data); + file + } + + #[test] + fn parses_uncompressed_unityfs_metadata() { + let parser = UnityFsParser::new(); + let data = synthetic_unityfs_bundle(&blocks_info(4), 0, false); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.header.format_version, 8); + assert_eq!(parsed.header.unity_version, "2021.3.56f2"); + assert_eq!(parsed.blocks_info_hash, [0xAB; 16]); + assert_eq!(parsed.blocks.len(), 1); + assert_eq!(parsed.blocks[0].compression, UnityFsCompression::None); + assert_eq!(parsed.directories.len(), 1); + assert_eq!(parsed.directories[0].path, "CAB-test"); + assert_eq!(parsed.uncompressed_data_size, 4); + assert_eq!(parsed.files.len(), 1); + assert_eq!(parsed.files[0].data, b"data"); + assert_eq!(parser.parse(&data).unwrap().assets, vec!["CAB-test"]); + } + + #[test] + fn parses_block_info_at_end() { + let parser = UnityFsParser::new(); + let data = synthetic_unityfs_bundle(&blocks_info(4), UNITYFS_BLOCK_INFO_AT_END_FLAG, true); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.directories[0].path, "CAB-test"); + assert_eq!(parsed.compressed_data_size, 4); + } + + #[test] + fn parses_lz4_compressed_block_info() { + let parser = UnityFsParser::new(); + let plain = blocks_info(4); + let compressed = lz4::block::compress(&plain, None, false).unwrap(); + let mut data = Vec::new(); + push_c_string(&mut data, "UnityFS"); + push_u32(&mut data, 8); + push_c_string(&mut data, "5.x.x"); + push_c_string(&mut data, "2021.3.56f2"); + push_u64(&mut data, 0); + push_u32(&mut data, compressed.len() as u32); + push_u32(&mut data, plain.len() as u32); + push_u32(&mut data, 2); + align(&mut data, UNITYFS_ALIGNMENT); + data.extend_from_slice(&compressed); + data.extend_from_slice(b"data"); + let total_size = data.len() as u64; + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.directories[0].path, "CAB-test"); + } + + #[test] + fn extracts_files_and_text_assets_from_unityfs_payload() { + let parser = UnityFsParser::new(); + let serialized_file = synthetic_serialized_text_asset(); + let data = synthetic_unityfs_bundle_with_payload( + "CAB-scenario", + &serialized_file, + &serialized_file, + 0, + ); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.files.len(), 1); + assert_eq!(parsed.files[0].path, "CAB-scenario"); + assert_eq!(parsed.files[0].data, serialized_file); + assert_eq!(parsed.serialized_files.len(), 1); + assert!(parsed.serialized_parse_errors.is_empty()); + assert_eq!(parsed.text_assets.len(), 1); + assert_eq!( + parsed.text_assets[0].source_path.as_deref(), + Some("CAB-scenario") + ); + assert_eq!(parsed.text_assets[0].name, "Scenario"); + assert_eq!(parsed.text_assets[0].bytes, "こんにちは".as_bytes()); + + let summary = parser.parse(&data).unwrap(); + assert_eq!(summary.text_assets.len(), 1); + assert_eq!(summary.files.len(), 1); + } + + #[test] + fn extracts_lz4_compressed_data_block() { + let parser = UnityFsParser::new(); + let payload = b"localized-text-payload"; + let compressed = lz4::block::compress(payload, None, false).unwrap(); + let data = synthetic_unityfs_bundle_with_payload("CAB-lz4", payload, &compressed, 2); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.files.len(), 1); + assert_eq!(parsed.files[0].path, "CAB-lz4"); + assert_eq!(parsed.files[0].data, payload); + } + + #[test] + fn extracts_data_block_after_block_info_alignment_padding() { + let parser = UnityFsParser::new(); + let payload = b"aligned-payload"; + let data = synthetic_unityfs_bundle_with_payload_and_header_flags( + "CAB-aligned", + payload, + payload, + 0, + UNITYFS_BLOCK_DATA_ALIGNED_FLAG, + ); + + let parsed = parser.parse_bytes(&data).unwrap(); + + assert_eq!(parsed.files.len(), 1); + assert_eq!(parsed.files[0].data, payload); + assert_eq!(parsed.data_start_offset % UNITYFS_ALIGNMENT as u64, 0); + } + + #[test] + fn rejects_directory_past_uncompressed_data_region() { + let parser = UnityFsParser::new(); + let data = synthetic_unityfs_bundle(&blocks_info(99), 0, false); + + let error = parser.parse_bytes(&data).unwrap_err().to_string(); + + assert!(error.contains("out of bounds"), "{error}"); + assert!(error.contains("uncompressed data region size 4"), "{error}"); + } + + #[test] + fn rejects_truncated_header_with_field_context() { + let parser = UnityFsParser::new(); + let error = parser.parse_bytes(b"UnityFS\0").unwrap_err().to_string(); + + assert!(error.contains("format_version"), "{error}"); + assert!(error.contains("offset"), "{error}"); + } + + #[test] + fn rejects_unknown_block_info_compression() { + let parser = UnityFsParser::new(); + let data = synthetic_unityfs_bundle(&blocks_info(4), 5, false); + + let error = parser.parse_bytes(&data).unwrap_err().to_string(); + + assert!( + error.contains("unsupported UnityFS block info compression"), + "{error}" + ); + } } diff --git a/crates/bat-assetbundle/src/serialized.rs b/crates/bat-assetbundle/src/serialized.rs new file mode 100644 index 0000000..2f55cd9 --- /dev/null +++ b/crates/bat-assetbundle/src/serialized.rs @@ -0,0 +1,878 @@ +//! Unity serialized file parser. +//! +//! The parser is intentionally data-oriented: it reads the serialized file +//! header, type metadata, object table and `TextAsset` payloads. Field-level +//! deserialization for `MonoBehaviour` and `ScriptableObject` builds on the +//! type tree structures exposed here. + +use crate::error::{AssetBundleError, Result}; +use std::fs; +use std::path::Path; + +/// One extracted Unity `TextAsset`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnitySerializedTextAsset { + /// UnityFS directory path or standalone serialized file path. + pub source_path: Option, + /// Unity path ID of the object. + pub path_id: i64, + /// Asset name stored in the serialized object. + pub name: String, + /// Raw bytes stored by the `TextAsset`. + pub bytes: Vec, +} + +/// Parsed Unity serialized file summary. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnitySerializedFile { + /// UnityFS directory path or standalone path, when known. + pub source_path: Option, + /// Serialized file format version. + pub version: u32, + /// Unity editor version stored in the file. + pub unity_version: String, + /// Target platform value from the file header. + pub platform: i32, + /// Type metadata entries declared by the file. + pub types: Vec, + /// Object table entries declared by the file. + pub objects: Vec, + text_assets: Vec, +} + +impl UnitySerializedFile { + /// Parses a serialized file from raw bytes. + pub fn from_slice(data: &[u8]) -> Result { + Self::from_named_slice(None::, data) + } + + /// Parses a serialized file from raw bytes with a source path. + pub fn from_named_slice(path: impl Into>, data: &[u8]) -> Result { + let source_path = path.into(); + let mut reader = Reader::new(data); + + let metadata_size_legacy = reader.read_u32_be("metadata_size")?; + let file_size_legacy = reader.read_u32_be("file_size")?; + let version = reader.read_u32_be("version")?; + let data_offset_legacy = reader.read_u32_be("data_offset")?; + let endian_flag = reader.read_u8("endian_flag")?; + reader.read_bytes(3, "reserved")?; + + let (metadata_size, file_size, data_offset) = if version >= 22 { + let metadata_size = reader.read_u32_be("metadata_size_2")?; + let file_size = reader.read_u64_be("file_size_2")?; + let data_offset = reader.read_u64_be("data_offset_2")?; + let _unknown = reader.read_u64_be("unknown_2")?; + (u64::from(metadata_size), file_size, data_offset) + } else { + ( + u64::from(metadata_size_legacy), + u64::from(file_size_legacy), + u64::from(data_offset_legacy), + ) + }; + + if file_size > data.len() as u64 { + return Err(AssetBundleError::Parse(format!( + "Unity serialized file_size {} exceeds available bytes {}", + file_size, + data.len() + ))); + } + if data_offset > data.len() as u64 { + return Err(AssetBundleError::Parse(format!( + "Unity serialized data_offset {} exceeds available bytes {}", + data_offset, + data.len() + ))); + } + let metadata_end = reader + .offset() + .checked_add(metadata_size as usize) + .ok_or_else(|| { + AssetBundleError::Parse("Unity serialized metadata end overflow".to_string()) + })?; + if metadata_end > data.len() { + return Err(AssetBundleError::Parse(format!( + "Unity serialized metadata exceeds file: end {}, file {}", + metadata_end, + data.len() + ))); + } + + let endian = if endian_flag == 0 { + Endian::Little + } else { + Endian::Big + }; + reader.set_endian(endian); + + let unity_version = reader.read_c_string("unity_version")?; + let platform = reader.read_i32("platform")?; + let enable_type_tree = reader.read_u8("enable_type_tree")?; + let type_count = reader.read_i32("type_count")?; + if type_count < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity type count: {type_count}" + ))); + } + + let mut types = Vec::with_capacity(type_count as usize); + for index in 0..type_count { + types.push(read_serialized_type( + &mut reader, + version, + enable_type_tree, + index as usize, + )?); + } + + let big_id_enabled = if (11..14).contains(&version) { + reader.read_i32("big_id_enabled")? + } else { + 0 + }; + + let object_count = reader.read_i32("object_count")?; + if object_count < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity object count: {object_count}" + ))); + } + + let mut objects = Vec::with_capacity(object_count as usize); + let mut text_assets = Vec::new(); + for _ in 0..object_count { + if version >= 14 { + reader.align(4)?; + } + + let path_id = if big_id_enabled != 0 { + reader.read_i64("path_id")? + } else if version < 14 { + i64::from(reader.read_i32("path_id")?) + } else { + reader.read_i64("path_id")? + }; + + let byte_start = if version >= 22 { + reader.read_u64("byte_start")? + } else { + u64::from(reader.read_u32("byte_start")?) + }; + let byte_size = reader.read_u32("byte_size")?; + let type_index_raw = reader.read_i32("type_id")?; + if type_index_raw < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity type index: {type_index_raw}" + ))); + } + if version < 16 { + reader.read_u16("class_id")?; + } + if version < 11 { + reader.read_u16("is_destroyed")?; + } + if (11..17).contains(&version) { + reader.read_i16("script_type_index")?; + } + if version == 15 || version == 16 { + reader.read_u8("stripped")?; + } + + let type_index = type_index_raw as usize; + let class_id = types.get(type_index).map(|ty| ty.class_id).ok_or_else(|| { + AssetBundleError::Parse(format!( + "Invalid Unity type index: {type_index_raw}, type_count {}", + types.len() + )) + })?; + let object_start = data_offset.checked_add(byte_start).ok_or_else(|| { + AssetBundleError::Parse("Unity object offset overflow".to_string()) + })?; + let object_end = object_start + .checked_add(u64::from(byte_size)) + .ok_or_else(|| AssetBundleError::Parse("Unity object size overflow".to_string()))?; + if object_end > data.len() as u64 { + return Err(AssetBundleError::Parse(format!( + "Unity object exceeds file size: start={}, size={}, file_size={}", + object_start, + byte_size, + data.len() + ))); + } + + objects.push(UnitySerializedObject { + path_id, + byte_start, + byte_size, + type_index, + class_id, + }); + + if class_id == 49 { + let mut asset = parse_text_asset( + path_id, + &data[object_start as usize..object_end as usize], + endian, + )?; + asset.source_path.clone_from(&source_path); + text_assets.push(asset); + } + } + + Ok(Self { + source_path, + version, + unity_version, + platform, + types, + objects, + text_assets, + }) + } + + /// Parses a serialized file from disk. + pub fn from_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + let bytes = fs::read(path).map_err(AssetBundleError::Io)?; + Self::from_named_slice(Some(path.display().to_string()), &bytes) + } + + /// Returns all extracted text assets. + pub fn text_assets(&self) -> &[UnitySerializedTextAsset] { + &self.text_assets + } + + /// Returns one extracted text asset by name. + pub fn text_asset(&self, name: &str) -> Option<&UnitySerializedTextAsset> { + self.text_assets.iter().find(|asset| asset.name == name) + } +} + +/// Type metadata entry from a Unity serialized file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnitySerializedType { + /// Index in the file type table. + pub index: usize, + /// Unity class ID, for example `49` for `TextAsset`. + pub class_id: i32, + /// Whether this type is stripped. + pub is_stripped_type: bool, + /// Script type index, when available. + pub script_type_index: Option, + /// Script hash bytes for MonoBehaviour-like types. + pub script_id: Option<[u8; 16]>, + /// Type hash bytes. + pub old_type_hash: Option<[u8; 16]>, + /// TypeTree nodes, if the file embeds a type tree. + pub type_tree: Vec, +} + +/// One TypeTree node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnityTypeTreeNode { + /// TypeTree format version for the node. + pub version: u16, + /// Nesting depth. + pub level: u8, + /// Whether this field is an array. + pub is_array: bool, + /// Field type name. + pub type_name: String, + /// Field name. + pub name: String, + /// Declared byte size. + pub byte_size: i32, + /// Node index. + pub index: i32, + /// Unity metadata flags. + pub meta_flag: i32, + /// Referenced type hash for newer Unity versions. + pub ref_type_hash: Option, +} + +/// Object table entry from a Unity serialized file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnitySerializedObject { + /// Unity path ID of the object. + pub path_id: i64, + /// Object byte offset relative to data section. + pub byte_start: u64, + /// Object byte size. + pub byte_size: u32, + /// Index into the file type table. + pub type_index: usize, + /// Unity class ID resolved from `type_index`. + pub class_id: i32, +} + +fn parse_text_asset(path_id: i64, data: &[u8], endian: Endian) -> Result { + let mut reader = Reader::new(data); + reader.set_endian(endian); + let name = reader.read_len_prefixed_string("text_asset_name")?; + reader.align(4)?; + let bytes_len = reader.read_u32("text_asset_bytes_len")? as usize; + let bytes = reader.read_bytes(bytes_len, "text_asset_bytes")?.to_vec(); + + Ok(UnitySerializedTextAsset { + source_path: None, + path_id, + name, + bytes, + }) +} + +fn read_serialized_type( + reader: &mut Reader<'_>, + version: u32, + enable_type_tree: u8, + index: usize, +) -> Result { + let class_id = reader.read_i32("type_class_id")?; + + let is_stripped_type = if version >= 16 { + reader.read_u8("type_is_stripped")? != 0 + } else { + false + }; + let script_type_index = if version >= 17 { + Some(reader.read_i16("type_script_index")?) + } else { + None + }; + + let mut script_id = None; + let mut old_type_hash = None; + if version >= 13 { + if (version < 16 && class_id < 0) || (version >= 16 && class_id == 114) { + script_id = Some(read_hash16(reader, "type_script_id")?); + } + old_type_hash = Some(read_hash16(reader, "type_hash")?); + } + + let type_tree = if enable_type_tree != 0 { + read_type_tree(reader, version)? + } else { + Vec::new() + }; + + Ok(UnitySerializedType { + index, + class_id, + is_stripped_type, + script_type_index, + script_id, + old_type_hash, + type_tree, + }) +} + +fn read_hash16(reader: &mut Reader<'_>, field: &str) -> Result<[u8; 16]> { + reader + .read_bytes(16, field)? + .try_into() + .map_err(|_| AssetBundleError::parse_field(field, reader.offset(), "expected 16 bytes")) +} + +fn read_type_tree(reader: &mut Reader<'_>, version: u32) -> Result> { + if version >= 12 || version == 10 { + let node_count = reader.read_i32("type_tree_node_count")?; + if node_count < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity type tree node count: {node_count}" + ))); + } + let string_buffer_size = reader.read_i32("type_tree_string_buffer_size")?; + if string_buffer_size < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity type tree string buffer size: {string_buffer_size}" + ))); + } + + let mut raw_nodes = Vec::with_capacity(node_count as usize); + for _ in 0..node_count { + raw_nodes.push(RawTypeTreeNode { + version: reader.read_u16("type_tree_node_version")?, + level: reader.read_u8("type_tree_node_level")?, + is_array: reader.read_u8("type_tree_node_is_array")? != 0, + type_offset: reader.read_i32("type_tree_node_type_offset")?, + name_offset: reader.read_i32("type_tree_node_name_offset")?, + byte_size: reader.read_i32("type_tree_node_byte_size")?, + index: reader.read_i32("type_tree_node_index")?, + meta_flag: reader.read_i32("type_tree_node_meta_flag")?, + ref_type_hash: if version >= 19 { + Some(reader.read_u64("type_tree_node_ref_type_hash")?) + } else { + None + }, + }); + } + + let string_buffer = + reader.read_bytes(string_buffer_size as usize, "type_tree_string_buffer")?; + if version >= 21 { + let dependency_count = reader.read_i32("type_tree_dependency_count")?; + if dependency_count < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid Unity type tree dependency count: {dependency_count}" + ))); + } + reader.read_bytes(dependency_count as usize * 4, "type_tree_dependencies")?; + } + + Ok(raw_nodes + .into_iter() + .map(|node| UnityTypeTreeNode { + version: node.version, + level: node.level, + is_array: node.is_array, + type_name: type_tree_string(string_buffer, node.type_offset), + name: type_tree_string(string_buffer, node.name_offset), + byte_size: node.byte_size, + index: node.index, + meta_flag: node.meta_flag, + ref_type_hash: node.ref_type_hash, + }) + .collect()) + } else { + read_legacy_type_tree(reader, version) + } +} + +fn read_legacy_type_tree(reader: &mut Reader<'_>, version: u32) -> Result> { + let type_name = reader.read_c_string("legacy_type_tree_type")?; + let name = reader.read_c_string("legacy_type_tree_name")?; + let byte_size = reader.read_i32("legacy_type_tree_byte_size")?; + let index = if version == 2 { + 0 + } else { + reader.read_i32("legacy_type_tree_index")? + }; + let is_array = reader.read_i32("legacy_type_tree_is_array")? != 0; + let version_value = reader.read_i32("legacy_type_tree_version")?; + let meta_flag = reader.read_i32("legacy_type_tree_meta_flag")?; + let child_count = reader.read_i32("legacy_type_tree_child_count")?; + if child_count < 0 { + return Err(AssetBundleError::Parse(format!( + "Invalid legacy type tree child count: {child_count}" + ))); + } + + let mut nodes = vec![UnityTypeTreeNode { + version: version_value as u16, + level: 0, + is_array, + type_name, + name, + byte_size, + index, + meta_flag, + ref_type_hash: None, + }]; + for _ in 0..child_count { + nodes.extend(read_legacy_type_tree(reader, version)?); + } + Ok(nodes) +} + +struct RawTypeTreeNode { + version: u16, + level: u8, + is_array: bool, + type_offset: i32, + name_offset: i32, + byte_size: i32, + index: i32, + meta_flag: i32, + ref_type_hash: Option, +} + +fn type_tree_string(buffer: &[u8], offset: i32) -> String { + let raw = offset as u32; + if raw & 0x8000_0000 != 0 { + let common_index = raw & 0x7fff_ffff; + return common_type_tree_string(common_index) + .map(ToOwned::to_owned) + .unwrap_or_else(|| format!("unity_common_string_{common_index}")); + } + + let offset = raw as usize; + if offset >= buffer.len() { + return format!("invalid_string_offset_{offset}"); + } + let bytes = &buffer[offset..]; + let len = bytes + .iter() + .position(|&byte| byte == 0) + .unwrap_or(bytes.len()); + std::str::from_utf8(&bytes[..len]) + .map(ToOwned::to_owned) + .unwrap_or_else(|_| format!("invalid_utf8_string_offset_{offset}")) +} + +fn common_type_tree_string(index: u32) -> Option<&'static str> { + match index { + 0 => Some("AABB"), + 5 => Some("Array"), + 19 => Some("Base"), + 28 => Some("bool"), + 35 => Some("char"), + 41 => Some("ColorRGBA"), + 51 => Some("data"), + 56 => Some("deque"), + 62 => Some("double"), + 69 => Some("dynamic_array"), + 83 => Some("FastPropertyName"), + 100 => Some("first"), + 106 => Some("float"), + 112 => Some("Font"), + 117 => Some("GameObject"), + 128 => Some("Generic Mono"), + 141 => Some("GradientNEW"), + 153 => Some("GUID"), + 158 => Some("GUIStyle"), + 167 => Some("int"), + 171 => Some("list"), + 176 => Some("long long"), + 186 => Some("map"), + 190 => Some("Matrix4x4f"), + 200 => Some("m_ByteSize"), + 211 => Some("m_Curve"), + 219 => Some("m_EditorClassIdentifier"), + 243 => Some("m_EditorHideFlags"), + 261 => Some("m_Enabled"), + 271 => Some("m_ExtensionPtr"), + 286 => Some("m_GameObject"), + 299 => Some("m_Index"), + 307 => Some("m_IsArray"), + 317 => Some("m_IsStatic"), + 328 => Some("m_MetaFlag"), + 339 => Some("m_Name"), + 346 => Some("m_ObjectHideFlags"), + 364 => Some("m_PrefabInternal"), + 381 => Some("m_PrefabParentObject"), + 402 => Some("m_Script"), + 411 => Some("m_StaticEditorFlags"), + 431 => Some("m_Type"), + 438 => Some("m_Version"), + 448 => Some("Object"), + 455 => Some("pair"), + 460 => Some("PPtr"), + 476 => Some("PPtr"), + 493 => Some("PPtr"), + 508 => Some("PPtr"), + 528 => Some("PPtr"), + 546 => Some("PPtr"), + 559 => Some("PPtr"), + 572 => Some("PPtr"), + 585 => Some("PPtr"), + 601 => Some("PPtr"), + 615 => Some("PPtr"), + 631 => Some("PPtr"), + 647 => Some("Quaternionf"), + 659 => Some("Rectf"), + 665 => Some("RectInt"), + 673 => Some("second"), + 680 => Some("set"), + 684 => Some("short"), + 690 => Some("size"), + 695 => Some("SInt16"), + 702 => Some("SInt32"), + 709 => Some("SInt64"), + 716 => Some("SInt8"), + 722 => Some("staticvector"), + 735 => Some("string"), + 742 => Some("TextAsset"), + 752 => Some("Texture2D"), + 762 => Some("Transform"), + 772 => Some("TypelessData"), + 785 => Some("UInt16"), + 792 => Some("UInt32"), + 799 => Some("UInt64"), + 806 => Some("UInt8"), + 812 => Some("unsigned int"), + 825 => Some("unsigned long long"), + 844 => Some("unsigned short"), + 859 => Some("vector"), + 866 => Some("Vector2f"), + 875 => Some("Vector3f"), + 884 => Some("Vector4f"), + 893 => Some("m_ScriptingClassIdentifier"), + _ => None, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Endian { + Little, + Big, +} + +struct Reader<'a> { + data: &'a [u8], + offset: usize, + endian: Endian, +} + +impl<'a> Reader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { + data, + offset: 0, + endian: Endian::Big, + } + } + + fn offset(&self) -> usize { + self.offset + } + + fn set_endian(&mut self, endian: Endian) { + self.endian = endian; + } + + fn read_bytes(&mut self, len: usize, field: &str) -> Result<&'a [u8]> { + let start = self.offset; + let end = self.offset.checked_add(len).ok_or_else(|| { + AssetBundleError::parse_field(field, start, "field length overflows usize") + })?; + if end > self.data.len() { + return Err(AssetBundleError::parse_field( + field, + start, + format!( + "unexpected end: need {}, have {}", + len, + self.data.len().saturating_sub(start) + ), + )); + } + + let bytes = &self.data[self.offset..end]; + self.offset = end; + Ok(bytes) + } + + fn align(&mut self, alignment: usize) -> Result<()> { + if alignment == 0 { + return Ok(()); + } + + let remainder = self.offset % alignment; + if remainder == 0 { + return Ok(()); + } + + let padding = alignment - remainder; + self.read_bytes(padding, "alignment_padding").map(|_| ()) + } + + fn read_u8(&mut self, field: &str) -> Result { + Ok(self.read_bytes(1, field)?[0]) + } + + fn read_u16(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(2, field)?; + Ok(match self.endian { + Endian::Little => u16::from_le_bytes([bytes[0], bytes[1]]), + Endian::Big => u16::from_be_bytes([bytes[0], bytes[1]]), + }) + } + + fn read_i16(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(2, field)?; + Ok(match self.endian { + Endian::Little => i16::from_le_bytes([bytes[0], bytes[1]]), + Endian::Big => i16::from_be_bytes([bytes[0], bytes[1]]), + }) + } + + fn read_u32(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(4, field)?; + Ok(match self.endian { + Endian::Little => u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + Endian::Big => u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + }) + } + + fn read_u32_be(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(4, field)?; + Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + fn read_i32(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(4, field)?; + Ok(match self.endian { + Endian::Little => i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + Endian::Big => i32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + }) + } + + fn read_u64(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(8, field)?; + Ok(match self.endian { + Endian::Little => u64::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]), + Endian::Big => u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]), + }) + } + + fn read_u64_be(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(8, field)?; + Ok(u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ])) + } + + fn read_i64(&mut self, field: &str) -> Result { + let bytes = self.read_bytes(8, field)?; + Ok(match self.endian { + Endian::Little => i64::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]), + Endian::Big => i64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]), + }) + } + + fn read_c_string(&mut self, field: &str) -> Result { + let start = self.offset; + let remaining = &self.data[self.offset..]; + let Some(length) = remaining.iter().position(|&byte| byte == 0) else { + return Err(AssetBundleError::parse_field( + field, + start, + "missing null terminator", + )); + }; + let bytes = self.read_bytes(length, field)?; + self.offset += 1; + std::str::from_utf8(bytes) + .map(ToOwned::to_owned) + .map_err(|error| { + AssetBundleError::parse_field(field, start, format!("invalid UTF-8: {error}")) + }) + } + + fn read_len_prefixed_string(&mut self, field: &str) -> Result { + let len = self.read_u32(field)? as usize; + let bytes = self.read_bytes(len, field)?; + std::str::from_utf8(bytes) + .map(ToOwned::to_owned) + .map_err(|error| { + AssetBundleError::parse_field(field, self.offset, format!("invalid UTF-8: {error}")) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn push_i16_le(data: &mut Vec, value: i16) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u32_le(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i32_le(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i64_le(data: &mut Vec, value: i64) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u64_le(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u32_be(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u64_be(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn align(data: &mut Vec, alignment: usize) { + let remainder = data.len() % alignment; + if remainder != 0 { + data.resize(data.len() + alignment - remainder, 0); + } + } + + fn synthetic_serialized_file() -> Vec { + let mut object_data = Vec::new(); + push_u32_le(&mut object_data, 14); + object_data.extend_from_slice(b"GameMainConfig"); + align(&mut object_data, 4); + push_u32_le(&mut object_data, 5); + object_data.extend_from_slice(b"hello"); + + let mut metadata = Vec::new(); + metadata.extend_from_slice(b"2021.3.56f2\0"); + push_i32_le(&mut metadata, 19); + metadata.push(0); + push_i32_le(&mut metadata, 1); + push_i32_le(&mut metadata, 49); + metadata.push(0); + push_i16_le(&mut metadata, 0); + metadata.extend_from_slice(&[0; 16]); + push_i32_le(&mut metadata, 1); + align(&mut metadata, 4); + push_i64_le(&mut metadata, 1); + push_u64_le(&mut metadata, 0); + push_u32_le(&mut metadata, object_data.len() as u32); + push_i32_le(&mut metadata, 0); + + let header_len = 48usize; + let data_offset = header_len + metadata.len(); + let file_size = data_offset + object_data.len(); + + let mut file = Vec::new(); + push_u32_be(&mut file, metadata.len() as u32); + push_u32_be(&mut file, file_size as u32); + push_u32_be(&mut file, 22); + push_u32_be(&mut file, 0); + file.push(0); + file.extend_from_slice(&[0, 0, 0]); + push_u32_be(&mut file, metadata.len() as u32); + push_u64_be(&mut file, file_size as u64); + push_u64_be(&mut file, data_offset as u64); + push_u64_be(&mut file, 0); + file.extend_from_slice(&metadata); + file.extend_from_slice(&object_data); + file + } + + #[test] + fn parses_synthetic_text_asset_and_object_table() { + let file = synthetic_serialized_file(); + let parsed = + UnitySerializedFile::from_named_slice(Some("CAB-test".to_string()), &file).unwrap(); + + assert_eq!(parsed.source_path.as_deref(), Some("CAB-test")); + assert_eq!(parsed.version, 22); + assert_eq!(parsed.unity_version, "2021.3.56f2"); + assert_eq!(parsed.platform, 19); + assert_eq!(parsed.types.len(), 1); + assert_eq!(parsed.types[0].class_id, 49); + assert_eq!(parsed.objects.len(), 1); + assert_eq!(parsed.objects[0].path_id, 1); + assert_eq!(parsed.objects[0].class_id, 49); + assert_eq!(parsed.text_assets.len(), 1); + + let asset = parsed.text_asset("GameMainConfig").unwrap(); + assert_eq!(asset.source_path.as_deref(), Some("CAB-test")); + assert_eq!(asset.name, "GameMainConfig"); + assert_eq!(asset.bytes, b"hello"); + } +} diff --git a/crates/bat-assetbundle/src/types.rs b/crates/bat-assetbundle/src/types.rs index 024c761..e49c955 100644 --- a/crates/bat-assetbundle/src/types.rs +++ b/crates/bat-assetbundle/src/types.rs @@ -1,7 +1,184 @@ -//! AssetBundle 类型定义占位 +//! AssetBundle and UnityFS public types. -/// Asset 类型（待实现） +use crate::serialized::{UnitySerializedFile, UnitySerializedTextAsset}; + +/// Asset type extracted from a Unity bundle. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum AssetType { - /// 文本资源 + /// Unity TextAsset. TextAsset, } + +/// Raw AssetBundle bytes with optional source path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawAssetBundle { + /// File bytes. + pub data: Vec, + /// Source path or logical name, when known. + pub path: Option, +} + +/// Parsed AssetBundle summary used by higher layers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedAssetBundle { + /// Unity editor version declared by the bundle. + pub unity_version: String, + /// Directory paths exposed by the UnityFS container. + pub assets: Vec, + /// Original bytes retained for future serialization. + pub raw_data: Vec, + /// UnityFS header information. + pub unityfs_header: Option, + /// UnityFS compressed block entries. + pub blocks: Vec, + /// UnityFS directory entries. + pub directories: Vec, + /// Files extracted from the UnityFS uncompressed data region. + pub files: Vec, + /// Serialized files parsed from UnityFS directory files. + pub serialized_files: Vec, + /// TextAsset objects extracted from serialized files. + pub text_assets: Vec, + /// Non-fatal serialized-file parse diagnostics for extracted files. + pub serialized_parse_errors: Vec, +} + +/// Parsed UnityFS container. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnityFsBundle { + /// UnityFS header. + pub header: UnityFsHeader, + /// 16-byte block info hash stored before block entries. + pub blocks_info_hash: [u8; 16], + /// UnityFS compressed block entries. + pub blocks: Vec, + /// UnityFS directory entries. + pub directories: Vec, + /// Offset where compressed block payload bytes begin. + pub data_start_offset: u64, + /// Total compressed payload bytes declared by block entries. + pub compressed_data_size: u64, + /// Total uncompressed payload bytes declared by block entries. + pub uncompressed_data_size: u64, + /// Original bytes retained for future extraction/serialization. + pub raw_data: Vec, + /// Files extracted from the UnityFS uncompressed data region. + pub files: Vec, + /// Serialized files parsed from UnityFS directory files. + pub serialized_files: Vec, + /// TextAsset objects extracted from serialized files. + pub text_assets: Vec, + /// Non-fatal serialized-file parse diagnostics for extracted files. + pub serialized_parse_errors: Vec, +} + +impl UnityFsBundle { + /// Returns directory paths in stable order. + pub fn asset_paths(&self) -> Vec { + self.directories + .iter() + .map(|directory| directory.path.clone()) + .collect() + } +} + +impl From for ParsedAssetBundle { + fn from(bundle: UnityFsBundle) -> Self { + Self { + unity_version: bundle.header.unity_version.clone(), + assets: bundle.asset_paths(), + raw_data: bundle.raw_data, + unityfs_header: Some(bundle.header), + blocks: bundle.blocks, + directories: bundle.directories, + files: bundle.files, + serialized_files: bundle.serialized_files, + text_assets: bundle.text_assets, + serialized_parse_errors: bundle.serialized_parse_errors, + } + } +} + +/// UnityFS header. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct UnityFsHeader { + /// UnityFS format version. + pub format_version: u32, + /// Bundle target version, for example `5.x.x`. + pub target_version: String, + /// Unity editor version, for example `2021.3.56f2`. + pub unity_version: String, + /// Total file size declared by the header. + pub total_size: u64, + /// Compressed block info byte size. + pub compressed_blocks_info_size: u32, + /// Uncompressed block info byte size. + pub uncompressed_blocks_info_size: u32, + /// Raw UnityFS flags. + pub flags: u32, +} + +/// UnityFS compression mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum UnityFsCompression { + /// Uncompressed. + None, + /// LZMA compression. + Lzma, + /// LZ4 compression. + Lz4, + /// LZ4HC compression. + Lz4Hc, + /// Unknown compression mode. + Unknown(u16), +} + +/// UnityFS compressed block entry. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct UnityFsBlockInfo { + /// Uncompressed block size. + pub uncompressed_size: u32, + /// Compressed block size. + pub compressed_size: u32, + /// Raw block flags. + pub flags: u16, + /// Compression mode decoded from `flags`. + pub compression: UnityFsCompression, +} + +/// UnityFS directory entry. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct UnityFsDirectoryInfo { + /// Entry offset in the uncompressed data region. + pub offset: u64, + /// Entry byte size. + pub size: u64, + /// Raw directory flags. + pub flags: u32, + /// Entry path. + pub path: String, +} + +/// File extracted from a UnityFS directory entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnityFsFile { + /// Entry path from the UnityFS directory table. + pub path: String, + /// Offset in the uncompressed UnityFS data region. + pub offset: u64, + /// File byte size. + pub size: u64, + /// Raw directory flags. + pub flags: u32, + /// Extracted file bytes. + pub data: Vec, +} + +/// Non-fatal parse error for an extracted UnityFS file. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct UnitySerializedParseError { + /// UnityFS directory path that failed serialized-file parsing. + pub path: String, + /// Human-readable parser error. + pub error: String, +} diff --git a/deployments/systemd/bluearchive-toolkit-official-sync.service b/deployments/systemd/bluearchive-toolkit-official-sync.service index 51c214f..ba183b1 100644 --- a/deployments/systemd/bluearchive-toolkit-official-sync.service +++ b/deployments/systemd/bluearchive-toolkit-official-sync.service @@ -10,10 +10,11 @@ User=bat Group=bat WorkingDirectory=/var/lib/bluearchive-toolkit Environment=BAT_OUTPUT_ROOT=/var/lib/bluearchive-toolkit/official +Environment=BAT_LOCALIZED_OUTPUT_ROOT=/var/lib/bluearchive-toolkit/localized Environment=BAT_INTERVAL=1h Environment=BAT_ERROR_RETRY=60s EnvironmentFile=-/etc/bluearchive-toolkit/official-sync.env -ExecStart=/opt/bluearchive-toolkit/bin/bat --auto-discover --output ${BAT_OUTPUT_ROOT} --watch --interval ${BAT_INTERVAL} --error-retry ${BAT_ERROR_RETRY} --no-banner +ExecStart=/opt/bluearchive-toolkit/bin/bat --auto-discover --output ${BAT_OUTPUT_ROOT} --localized-output ${BAT_LOCALIZED_OUTPUT_ROOT} --watch --interval ${BAT_INTERVAL} --error-retry ${BAT_ERROR_RETRY} --no-banner Restart=on-failure RestartSec=30 TimeoutStopSec=60 diff --git a/deployments/systemd/official-sync.env.example b/deployments/systemd/official-sync.env.example index c6457cf..72c40a4 100644 --- a/deployments/systemd/official-sync.env.example +++ b/deployments/systemd/official-sync.env.example @@ -4,9 +4,11 @@ # sudo install -o root -g root -m 0644 deployments/systemd/official-sync.env.example /etc/bluearchive-toolkit/official-sync.env # # Paths are intentionally independent from any official launcher or game client -# install directory. Do not point BAT_OUTPUT_ROOT at an existing game directory. +# install directory. Do not point either root at an existing game directory, and +# keep the official and localized roots separate. BAT_OUTPUT_ROOT=/var/lib/bluearchive-toolkit/official +BAT_LOCALIZED_OUTPUT_ROOT=/var/lib/bluearchive-toolkit/localized BAT_INTERVAL=1h BAT_ERROR_RETRY=60s diff --git a/docs/architecture/assetbundle.md b/docs/architecture/assetbundle.md new file mode 100644 index 0000000..30a324e --- /dev/null +++ b/docs/architecture/assetbundle.md @@ -0,0 +1,199 @@ +# AssetBundle 与资源解析路线图 + +- **更新时间**：2026-07-25 +- **适用范围**：Rust 解析引擎、官方同步后的解析缓存、CAS/ResourceRepository 接入、后续文本提取和 Patch 发布。 +- **权威关联**：`PROJECT_PLAN.md` Milestone 3/4/5/8，`docs/reports/CURRENT_GAPS.md` G-005/G-007/G-011/G-011D。 + +--- + +## 1. 目标边界 + +解析系统的目标不是把下载流程写成一次性脚本，而是建立可长期维护的资源理解层： + +1. 官方资源同步负责拉取、校验和发布原版资源。 +2. 解析器只读取已发布或 staging 中已校验的资源，不修改原始文件。 +3. 解析结果写入派生缓存、CAS 索引或后续文本提取索引。 +4. 汉化产物只能由 Patch/发布阶段写入 `localized-output`，不能写回官方资源目录。 +5. 解析器必须与 CLI、daemon、Go API、Patch 业务流程解耦。 + +当前官方同步完成后会维护 `official-parse-cache.json`。它是官方 release 的派生索引，不是汉化产物；本地 URL、相对路径、size 和 BLAKE3 未变化时应复用旧解析结果并跳过重复解析。 + +--- + +## 2. 分层模型 + +解析能力按从外到内分层： + +| 层级 | 输入 | 输出 | 当前状态 | +| --- | --- | --- | --- | +| 官方 seed manifest | `TableCatalog.bytes`、`BundlePackingInfo.bytes`、`MediaCatalog.bytes` | 完整下载 URL、相对路径、hash 校验边界 | 已用于下载计划，仍需沉淀更多结构化字段 | +| Addressables catalog | `catalog_*.zip` 内 JSON/bin catalog、`catalog_*.hash` | asset path、provider、dependencies、size、CRC、bundle name | JSON/compact 当前样本已覆盖，仍需更多真实结构变体 | +| UnityFS container | `.bundle`、zip 内 bundle | header、block、directory、解压文件、基础摘要 | 已支持基础解包、LZ4/LZMA、边界校验 | +| Serialized file | UnityFS directory 文件 | header、type table、TypeTree node、object table、TextAsset bytes | 已支持基础表结构和 TextAsset bytes | +| Unity 对象字段 | TextAsset、MonoBehaviour、ScriptableObject | 可翻译文本单元、上下文、资源定位 | TextAsset bytes 已有入口，字段级反序列化未完成 | +| Patch 发布前解析 | 已翻译文本、中间格式、原版资源 | 可验证 patch manifest、汉化 release 目录 | 未完成 | + +--- + +## 3. 当前已落地能力 + +`crates/bat-assetbundle` 已经承担解析核心： + +1. `UnityFsParser` 解析 UnityFS header、block info、directory。 +2. 支持 LZ4/LZMA block info 和数据 block 解压。 +3. 支持 block info at end 和官方样本中出现的 block data alignment。 +4. 能从 UnityFS directory 提取文件 bytes。 +5. `serialized` 模块能读取 Unity serialized file header、type table、TypeTree node 元数据、object table。 +6. 能提取 TextAsset 的 name 和原始 bytes。 +7. `ResourceImportService` 能把 AssetBundle 摘要、TextAsset/Table/Media 分类写入导入报告。 +8. 官方同步后 `OfficialParseCacheService` 能从 `official-download-manifest.json` 遍历所有资源，解析直接 bundle 和 zip 内条目，非候选资源记录为 unsupported。 + +当前还不能宣称完整： + +1. MonoBehaviour/ScriptableObject 的 TypeTree 字段级反序列化未完成。 +2. Unity managed reference、PPtr、array/map、string alignment 等复杂字段未完整覆盖。 +3. Addressables bin/compact catalog 结构变体仍需真实样本驱动补齐。 +4. 解析结果尚未自动进入用户级 CAS + ResourceRepository 查询流程。 +5. 不能重写 AssetBundle，也不能生成可发布汉化 patch。 + +--- + +## 4. 补全顺序 + +### P0：解析缓存和样本闭环 + +目标：让官方同步后的解析结果可复用、可诊断、可回归。 + +交付： + +1. `official-parse-cache.json` 记录 manifest entry、zip entry、解析状态、Unity 版本、文件数、TextAsset 数、错误摘要和缓存复用状态。 +2. 解析直接 `.bundle` / `.unity3d` 和 zip 内全部文件条目，不能只假设 `FullPatch_*.zip`。 +3. 非候选资源记录为 unsupported，不影响官方同步发布。 +4. 缺失、损坏或无法解析的 bundle 记录 failed，但不回滚已经完成校验的官方原版 release。 +5. 用合成 fixture、隔离真实样本和回归 fixture 覆盖缓存复用、zip 内条目、非候选资源、解析失败。 + +验收： + +1. 第二次运行同一 release 时能看到解析缓存复用计数。 +2. 修改任意 manifest entry 的 size/BLAKE3 后，只重新解析对应资源。 +3. 解析缓存不会写入 `localized-output`。 + +### P1：Addressables catalog 完整化 + +目标：把“能列出资源”推进到“能稳定定位 bundle、依赖、校验字段和资源类型”。 + +交付： + +1. 覆盖 JSON catalog、compact JSON、可能的二进制 catalog 入口。 +2. 解析 provider id、internal id、primary key、dependency key、resource type、bundle name、hash、size、CRC。 +3. 明确 `catalog_*.hash` 只作为 Addressables remote catalog marker，不套用 seed `.hash` 的 xxHash32 规则。 +4. 将 Windows/Android catalog 样本拆成可复现 fixture，不把大文件纳入 Git。 +5. 对未知结构返回明确错误或保真 raw metadata，不静默丢字段。 + +验收： + +1. 当前目标版本 Windows/Android catalog 样本集合解析通过。 +2. 解析结果能反查 bundle 文件和依赖链。 +3. size/CRC/hash 字段能参与本地文件验证或至少进入诊断报告。 + +### P2：Unity Serialized 字段级解析 + +目标：把 Unity object table 推进到可提取文本字段。 + +交付： + +1. TypeTree schema 内部表示稳定化：node path、type、name、size、flags、array 信息。 +2. 实现基础字段 reader：bool、integer、float、string、bytes、array、map、PPtr、managed reference 占位。 +3. 支持 TextAsset script/name 的结构化读取，而不只保留 raw bytes。 +4. 支持 MonoBehaviour 和 ScriptableObject 的 TypeTree 字段遍历。 +5. 对缺 TypeTree 或 stripped 类型返回可诊断错误，保留 raw object bytes 作为后备。 + +验收： + +1. 合成 fixture 覆盖标量、数组、嵌套结构、string alignment。 +2. 隔离真实样本能输出稳定 JSON field tree。 +3. 解析错误包含 file path、object path id、class id、字段路径和偏移。 + +### P3：文本提取中间层 + +目标：为日语汉化提供稳定、可回写定位的文本单元。 + +交付： + +1. 定义 `TextUnit`：source text、resource path、archive entry、object path id、field path、语言、版本、上下文。 +2. TextAsset 支持 JSON/CSV/TSV/plain text 的可配置探测。 +3. MonoBehaviour/ScriptableObject 按字段路径和类型策略提取字符串。 +4. 保留重复文本和上下文，不在解析阶段做会丢定位的合并。 +5. 导出 JSONL 作为第一稳定格式，CSV/XLIFF 可后置。 + +验收： + +1. 提取不会修改官方资源。 +2. 每条文本能追溯回原 bundle、serialized file、path id 和字段路径。 +3. 同一文本在不同上下文中保持可区分。 + +### P4：CAS/Repository 用户级接入 + +目标：让解析结果进入可查询资源库，而不是只停留在文件系统缓存。 + +交付： + +1. 官方同步完成后可配置触发导入 CAS + ResourceRepository。 +2. ResourceRepository 保存版本、平台、资源类型、bundle path、TextAsset 名称、TextUnit 索引摘要。 +3. 支持 CLI/RPC 查询资源、bundle、TextAsset、解析错误和缓存状态。 +4. schema 迁移版本化，旧库可重复升级。 + +验收： + +1. 可以按版本、路径、类型、hash、TextAsset 名称查询。 +2. 解析缓存和 repository 数据能从同一 manifest fingerprint 追溯。 +3. CAS 对象跨版本复用，不重复存储相同文件。 + +### P5：Patch 发布前置解析 + +目标：让解析结果成为可生成汉化 patch 的输入。 + +交付： + +1. 定义 patch manifest：目标官方版本、输入 TextUnit 版本、输出文件、hash、回滚信息。 +2. 支持 TextAsset raw bytes 替换的最小 patch 路径。 +3. MonoBehaviour/ScriptableObject 字段替换必须依赖 P2 字段级解析结果。 +4. Patch 产物写入 `localized-output/.staging/`，校验通过后发布到 `localized-output/versions/` 并切换 `current`。 +5. 成功后发布状态从 `not_localized` 切到 `localized`。 + +验收： + +1. Patch 失败不影响 `bat-resources/current`。 +2. 汉化 release 保留官方相对目录结构。 +3. `localized` 状态能证明原版和汉化两套资源都已发布。 + +--- + +## 5. 解析器接口原则 + +1. 解析器输入只接受 bytes、逻辑路径和可选上下文，不直接访问下载器状态。 +2. 解析器输出必须可序列化，供 CLI/RPC/API、缓存和测试 golden 使用。 +3. 错误必须带位置：URL 或路径、archive entry、UnityFS directory、object path id、field path、offset。 +4. 未识别结构优先保留 raw metadata，不做低保真猜测。 +5. 解析器不写 `bat-resources` 和 `bat-localized`，写文件由上层缓存、导入或 Patch 发布流程负责。 + +--- + +## 6. Fixture 策略 + +1. 合成 fixture 放入代码仓库，覆盖边界和回归。 +2. 真实小样本可放入仓库前必须确认体积、许可和可复现性。 +3. 大型真实官方资源只允许放在 `/tmp`、隔离测试目录或用户显式提供的远端测试目录，不纳入 Git。 +4. 每个新增 fixture 必须说明覆盖的真实风险：字段变体、压缩模式、越界、hash mismatch、zip 内路径、TypeTree 结构等。 + +--- + +## 7. 近期关闭路径 + +优先顺序： + +1. 完成 Addressables Windows/Android 当前版本 catalog 样本集合，关闭 G-007 当前阶段。 +2. 完成 TypeTree 字段 reader 和 MonoBehaviour/ScriptableObject 遍历，推进 G-005。 +3. 将 `official-parse-cache.json` 摘要接入 CAS/ResourceRepository 用户级导入，推进 G-011。 +4. 定义 TextUnit JSONL 和最小 TextAsset 提取命令，衔接 Milestone 5。 +5. 在 Patch 引擎落地后实现 `localized` 发布状态切换，关闭 G-011D 的核心发布缺口。 diff --git a/docs/architecture/official-resource-backend.md b/docs/architecture/official-resource-backend.md index 3e3f0ea..293d55c 100644 --- a/docs/architecture/official-resource-backend.md +++ b/docs/architecture/official-resource-backend.md @@ -6,6 +6,12 @@ 这个后端只处理 **日服官方资源**，只接受官方 `.jp/.com` 域名下的资源链路。 +**Release 布局、URL→磁盘映射、seed 模板与 bat-api 分发 path 的冻结契约**见： + +- `docs/architecture/resource-release-layout.md` + +--- + 明确排除： - `bluearchive.cafe` @@ -207,8 +213,10 @@ 10. 下载先写入 `/.staging/`；若已有 active release，会先 seed staging 以复用已验证文件；若 version-state 中存在同一版本的失败 staging，则优先复用该 staging 并跳过 active seed，避免旧 active 覆盖已下载的新文件。 11. 下载、manifest、本地 BLAKE3、ZIP 和官方 `.hash` 校验完成后写入新的 snapshot。 12. 将 staging rename 为 `/versions/`，再原子替换 `/current` symlink 指向该 versioned 目录。 +13. 发布完成后刷新 active release 下的 `official-parse-cache.json`；未变化文件按 manifest 的 URL、相对路径、size 和 BLAKE3 复用旧解析结果。 +14. 官方同步报告默认给出 `localized_release_status=not_localized`，表示原版资源已发布、汉化资源未发布；后续 Patch 发布阶段完成后才切换为 `localized`，表示原版和汉化两套资源都已发布。 -该入口不安装、不执行官方启动器，也不读取生产外的本地客户端目录。Rust 正式 binary `bat` 支持单次运行、`--watch` 常驻模式、`--daemon` 后台模式，以及 `status`、`stop`、`restart`、`reload`、`logs`、`refresh`、`verify`、`repair`、`doctor`、`clean-stable` 管理命令。`--daemon` 会在后台状态目录下创建 `bat.sock`，使用 Unix socket JSON-RPC 作为 live control plane；`bat.pid`、`bat-status.json` 和 `bat-daemon.log` 是快照、诊断和兼容 fallback；`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志；`bat-control.lock` 串行化控制命令，并在 stale/corrupt 时由下一次控制命令或 `clean-stable` 恢复。`bat-status.json` 和 `status` 子命令包含最后成功时间、下次检查时间、最后错误摘要、当前阶段和当前下载 URL 进度。PID、status、log 和控制锁文件创建时使用私有权限，读取和写入时不跟随 symlink。`status`、`stop`、`logs`、`reload`、默认形态的 `refresh` 和默认形态的 `repair` 优先走 RPC；`reload` 会唤醒或排队 watch 循环重新自动发现并强制刷新，默认 `repair` 会通过 `resource.repair` 入队本地 manifest 审计+修复任务，`restart` 才负责重启进程或替换启动参数；显式 `--proxy` / `--no-proxy` 会作为启动参数保存并在后台重启时复用。后台 daemon 管理某个资源目录时，前台 `run/watch/refresh/repair` 不允许直接写入同一目录；默认形态 `refresh` 会通过 RPC 触发后台刷新，默认形态 `repair` 会通过 RPC 入队任务。正常情况下默认每 1 小时执行一次检查；每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会中断普通 sleep 并强制执行一次自动刷新，该轮注入 `force=true`。远端和本地一致时静默等待下次检查，不一致时自动下载或 repair。下载、发现或校验失败时不等待完整正常周期，默认 60 秒后重试；如果固定时间强制刷新失败，会保留 pending force 并按失败重试周期继续重试，可用 `--error-retry` 或 `--error-retry-seconds` 调整。默认资源输出目录是 `./bat-resources`，默认后台状态目录是 `/tmp/bat-pid`，二者通过 `--output` 和 `--state-dir` 分别配置。单次运行仍保留为核心幂等路径，systemd service、容器或 Go 进程可以只负责守护该常驻进程；cron/systemd timer 调单次模式只是可选集成方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。 +该入口不安装、不执行官方启动器，也不读取生产外的本地客户端目录。Rust 正式 binary `bat` 支持单次运行、`--watch` 常驻模式、`--daemon` 后台模式，以及 `status`、`stop`、`restart`、`reload`、`logs`、`refresh`、`verify`、`repair`、`doctor`、`clean-stable` 管理命令。`--daemon` 会在后台状态目录下创建 `bat.sock`，使用 Unix socket JSON-RPC 作为 live control plane；`bat.pid`、`bat-status.json` 和 `bat-daemon.log` 是快照、诊断和兼容 fallback；`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志；`bat-control.lock` 串行化控制命令，并在 stale/corrupt 时由下一次控制命令或 `clean-stable` 恢复。`bat-status.json` 和 `status` 子命令包含最后成功时间、下次检查时间、最后错误摘要、当前阶段和当前下载 URL 进度。PID、status、log 和控制锁文件创建时使用私有权限，读取和写入时不跟随 symlink。`status`、`stop`、`logs`、`reload`、默认形态的 `refresh` 和默认形态的 `repair` 优先走 RPC；`reload` 会唤醒或排队 watch 循环重新自动发现并强制刷新，默认 `repair` 会通过 `resource.repair` 入队本地 manifest 审计+修复任务，`restart` 才负责重启进程或替换启动参数；显式 `--proxy` / `--no-proxy` 会作为启动参数保存并在后台重启时复用。后台 daemon 管理某个资源目录时，前台 `run/watch/refresh/repair` 不允许直接写入同一目录；默认形态 `refresh` 会通过 RPC 触发后台刷新，默认形态 `repair` 会通过 RPC 入队任务。正常情况下默认每 1 小时执行一次检查；每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会中断普通 sleep 并强制执行一次自动刷新，该轮注入 `force=true`。远端和本地一致时静默等待下次检查，不一致时自动下载或 repair。下载、发现或校验失败时不等待完整正常周期，默认 60 秒后重试；如果固定时间强制刷新失败，会保留 pending force 并按失败重试周期继续重试，可用 `--error-retry` 或 `--error-retry-seconds` 调整。默认官方原版资源输出目录是 `./bat-resources`，默认汉化产物目录是 `./bat-localized`，默认后台状态目录是 `/tmp/bat-pid`，三者分别通过 `--output`、`--localized-output` 和 `--state-dir` 配置；官方目录和汉化目录不能相同或互相嵌套。单次运行仍保留为核心幂等路径，systemd service、容器或 Go 进程可以只负责守护该常驻进程；cron/systemd timer 调单次模式只是可选集成方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。 对应实现主要在： @@ -304,14 +312,22 @@ JSON-RPC 2.0 服务，是面向上层服务（Go 层）的**主要跨语言边 ### 7.2 Go 层职责边界 -- Go 层负责：BlueArchive 客户端请求处理、HTTP API、鉴权、内容分发， - 以及通过 `internal/backendrpc` 作为 RPC client 调用本机 daemon - （连接 `bat.sock`，每行一个 JSON-RPC 请求/响应）。当前 Go 产品入口 - 尚未完成，`cmd/bat` 仍是试验骨架。 -- Rust daemon 负责：官方资源自动拉取与校验、catalog 更新检查、 +- Go 层负责：资源内容分发（`cmd/bat-api`）、HTTP API 进程配置、以及通过 + `internal/backendrpc` 作为 RPC client 调用本机 daemon（连接 `bat.sock`， + 每行一个 JSON-RPC 请求/响应）。`cmd/bat` 仍是试验骨架，不是产品级用户 CLI。 +- **`bat-api`（资源分发，issue #19）**： + - 只读提供 Rust `bat` 已发布 release 中的资源字节（官方 CDN host/path 形态）。 + - 版本/清单发现优先走 RPC：先 `daemon.status`，再 `daemon.doctor`，再 + `catalog.status` / `resource.manifest`（可用 `--socket` 指定 socket 文件）。 + - 支持 `.env` / 环境变量配置监听端口、public base URL、RPC socket，并预留 + database/redis 键供后续 API 持久化；**不**负责资源自动拉取。 + - 可选改写 server-info 中的 `AddressablesCatalogUrlRoot` 指向自身；不伪装 + 完整游戏业务 API，launcher 全链非本服务关闭条件。 +- Rust `bat` / daemon 负责：官方资源自动发现与拉取、校验、catalog 更新检查、 版本状态与发布、任务队列/日志/错误/进度管理等长期状态型工作。 -- Go 层**不**直接嵌入 Rust FFI，不直接读写 daemon 的状态文件与资源 - 目录内部结构；跨语言交互只经 RPC 契约。 +- Go 层**不**直接嵌入 Rust FFI，不直接读写 daemon 的状态文件；跨语言控制面 + 只经 RPC 契约。文件字节从 RPC 给出的 `resource_root`（或显式 + `--resource-root`）读取，与 daemon 同机或共享文件系统部署。 ### 7.3 FFI 的定位（降级说明） diff --git a/docs/architecture/resource-release-layout.md b/docs/architecture/resource-release-layout.md new file mode 100644 index 0000000..06f8b08 --- /dev/null +++ b/docs/architecture/resource-release-layout.md @@ -0,0 +1,311 @@ +# 官方资源 Release 布局与资源侧契约 + +- **更新时间**：2026-07-24 +- **用途**：冻结日服官方资源在本地发布根上的布局、URL 映射、seed 规则，以及 `bat-api` 分发 path 的 1:1 对应关系。 +- **范围**：资源发现 / 清单 / 落盘 / 只读分发（**不是**完整游戏业务 API）。 +- **权威代码**： + - URL / 平台 / seed：`adapters/src/official/yostar_jp.rs` + - inventory 抽取：`adapters/src/official/inventory.rs` + - 落盘与 manifest：`infrastructure/src/official_download.rs`（`destination_for_url`） + - 发布布局：`infrastructure/src/official_update.rs` + - 分发：`cmd/bat-api` + `internal/api`（见 `docs/reports/GO_STATUS.md`） + +--- + +## 1. 产品边界（资源侧） + +| 角色 | 组件 | 职责 | +|---|---|---| +| 同步 / 运维（近乎全自动） | Rust `bat` | auto-discover、拉取、校验、发布、watch/daemon、RPC 后端 | +| 资源只读分发 | Go `bat-api` | 按官方 CDN path 提供已发布字节；经 `bat.sock` 发现版本 | +| 试验 CLI | Go `cmd/bat` → `bin/bat-go` | 非产品；禁止与 Rust `bat` 重名 | + +**禁止**：把已安装客户端目录或 `/home/wanye/D/BlueArchive` 当作生产输入；真实全量样本优先服务器 release 或 `/tmp` 隔离目录。 + +--- + +## 2. 发布根布局（L1） + +```text +/ # 官方原版资源发布根（--output / BAT_OUTPUT） + current -> versions/ # 原子 symlink，生产读侧 + versions// # 已发布 versioned release（= resource_root） + official-download-manifest.json + official-parse-cache.json # 校验后派生解析缓存，不是汉化产物 + official-sync-snapshot.json # 常在 active root / current 下 + prod-clientpatch.bluearchiveyostar.com/ + / + TableBundles/ + TableCatalog.bytes + TableCatalog.hash + # e.g. ExcelDB.db, Excel.zip + Windows_PatchPack/ + BundlePackingInfo.bytes + BundlePackingInfo.hash + catalog_StandaloneWindows64.zip + catalog_StandaloneWindows64.hash + FullPatch_NNN.zip + Android_PatchPack/ + BundlePackingInfo.bytes + BundlePackingInfo.hash + catalog_Android.zip + catalog_Android.hash + FullPatch_NNN.zip + MediaResources-Windows/ + Catalog/MediaCatalog.bytes + Catalog/MediaCatalog.hash + GameData/... + Prologue/... + MediaResources/ # Android + Catalog/MediaCatalog.bytes + Catalog/MediaCatalog.hash + ... + yostar-serverinfo.bluearchiveyostar.com/ # 若曾下载 server-info + .json + .staging// # 未发布写侧（失败可复用） + official-version-state.json # 发布根级版本状态 + official-bootstrap-cache.json # auto-discover 缓存 + +/ # 汉化产物发布根（--localized-output / BAT_LOCALIZED_OUTPUT） + current -> versions/ # 已汉化后才切换；未汉化状态不发布 + versions// # 与官方相对路径一致的汉化资源 + localized-version-state.json # 预留：后续 Patch 发布阶段维护，官方同步阶段不写入 +``` + +官方资源发布和汉化发布是两个独立状态： + +- `not_localized`：官方原版资源已经完成下载、校验和发布，汉化资源尚未发布；这是官方同步完成后的默认状态。 +- `localized`：同一官方版本的原版资源和汉化资源都已发布，生产侧可以同时提供两套资源。 + +### 2.1 读侧 vs 写侧 + +| 阶段 | 根目录 | +|---|---| +| 下载写入 | `/.staging/` | +| 发布完成 | rename 到 `versions/`，再切换 `current` | +| 生产读取 / bat-api | `current` 解析后的 versioned 目录，或 RPC 给出的 `version.resource_root` | + +--- + +## 3. URL → 磁盘映射（核心不变量） + +实现：`OfficialResourcePullService::destination_for_url`。 + +```text +https://{host}/{path...} → /{host}/{path...} +``` + +规则： + +1. 仅 `https://` +2. host 必须是官方 JP 资源 host（见下节） +3. path 分段不得为 `.` / `..` +4. **禁止** query / fragment（否则直接拒绝，避免同路径覆盖） +5. 分段经 sanitize 后 join；结果必须在 `resource_root` 内 + +### 3.1 官方 host + +| Host | 用途 | +|---|---| +| `prod-clientpatch.bluearchiveyostar.com` | Addressables / Table / Media / PatchPack 内容 | +| `yostar-serverinfo.bluearchiveyostar.com` | server-info JSON | + +（launcher 包 CDN 属于启动器链，**不是**默认资源 release 主体。） + +### 3.2 bat-api 对外 path（1:1） + +```text +GET {public-base-url}/prod-clientpatch.bluearchiveyostar.com//... + ≡ 磁盘 /prod-clientpatch.bluearchiveyostar.com//... +``` + +默认仅服务 **download manifest 索引内且 Present + size 匹配** 的文件。 + +--- + +## 4. `official-download-manifest.json` + +| 字段 | 类型 | 说明 | +|---|---|---| +| `version` | u32 | 当前为 `1` | +| `entries` | map URL → entry | 按完整官方 URL 为键（有序 BTreeMap） | + +每条 entry： + +| 字段 | 说明 | +|---|---| +| `url` | 官方 https URL | +| `destination` | 相对 resource_root 的路径（`host/path...`） | +| `bytes` | 文件大小 | +| `blake3` | 本地 BLAKE3 hex | + +**权威清单**：拉取闭环写入的 manifest；`bat-api` / RPC `resource.manifest` 以此为应有集合，再以磁盘校验 Present。 + +--- + +## 5. 发现与 seed URL 规则（L2） + +常量根： + +- server-info：`https://yostar-serverinfo.bluearchiveyostar.com` +- client-patch：`https://prod-clientpatch.bluearchiveyostar.com` + +`AddressablesCatalogUrlRoot` 形如： + +```text +https://prod-clientpatch.bluearchiveyostar.com/ +``` + +默认平台：`Windows` + `Android`。 + +### 5.1 平台目录名 + +| 平台 | Patch 目录 | Media 目录 | Addressables catalog zip | +|---|---|---|---| +| Windows | `Windows_PatchPack` | `MediaResources-Windows` | `catalog_StandaloneWindows64.zip` | +| Android | `Android_PatchPack` | `MediaResources` | `catalog_Android.zip` | + +### 5.2 Seed 端点模板 + +共享（非平台）： + +```text +{CLIENT_PATCH}/{token}/TableBundles/TableCatalog.bytes +{CLIENT_PATCH}/{token}/TableBundles/TableCatalog.hash +``` + +每平台： + +```text +{CLIENT_PATCH}/{token}/{PatchDir}/BundlePackingInfo.bytes +{CLIENT_PATCH}/{token}/{PatchDir}/BundlePackingInfo.hash +{CLIENT_PATCH}/{token}/{PatchDir}/{catalog_zip} +{CLIENT_PATCH}/{token}/{PatchDir}/{catalog_base}.hash +{CLIENT_PATCH}/{token}/{MediaDir}/Catalog/MediaCatalog.bytes +{CLIENT_PATCH}/{token}/{MediaDir}/Catalog/MediaCatalog.hash +``` + +### 5.3 Content URL 模板 + +| 类型 | 模板 | +|---|---| +| Table 文件 | `{CLIENT_PATCH}/{token}/TableBundles/{Name}` | +| Patch pack | `{CLIENT_PATCH}/{token}/{PatchDir}/{FullPatch_NNN.zip}` | +| Media 文件 | `{CLIENT_PATCH}/{token}/{MediaDir}/{relative_path}` | + +`relative_path` 示例：`GameData/Audio/VOC_JP/JP_Airi.zip`、`Prologue/Scenario/Event/10000_Title_Sound.ogg`。 + +### 5.4 校验分层 + +| 对象 | 算法 / 规则 | +|---|---| +| seed `.bytes` + `.hash` | 官方 `.hash` 为 **xxHash32(seed=0)** 的十进制文本；强校验 | +| 一般已下载文件 | 本地 manifest **size + BLAKE3** | +| `.zip` | 另加 ZIP central/local 结构校验 | +| `catalog_*.hash` | Addressables/SBP **Hash128 文本标记**，**不是** seed 的 xxHash32 规则 | + +--- + +## 6. Inventory 抽取规则（L3，当前实现） + +实现：`adapters/src/official/inventory.rs`（**可打印串启发式**，非完整 schema 反序列化）。 + +| Catalog | 抽取逻辑 | 风险 | +|---|---|---| +| `BundlePackingInfo.bytes` | 可打印串中扩展名为 `zip` 且匹配 `FullPatch_NNN.zip`（总长 17，中间 3 位数字） | 漏抽非 FullPatch 包名（当前有意只 FullPatch） | +| `TableCatalog.bytes` | 可打印串中 `.db`/`.zip` 文件名；**出现次数 ≥ 2** 才收录 | 依赖「双份列表」启发式；形态变化会漏/多 | +| `MediaCatalog.bytes` | 可打印串中相对路径，扩展名 zip/mp4/png/jpg/jpeg/ogg/wav | 路径须 `is_plausible_relative_path` | + +**R2 待真机核对**：用服务器全量 seed 字节跑抽取，与 manifest 中 content URL 集合 diff；有未解释差异再改 inventory + fixture。 + +仓库内已有：`adapters/tests/fixtures`、`infrastructure/tests/fixtures/official_regression`；**不能替代**全量 release 实勘。 + +--- + +## 7. 客户端资源请求假设（R3，服务 bat-api） + +| 面 | 假设（当前工程） | bat-api 行为 | +|---|---|---| +| client-patch 内容 | GET 官方 path；无业务鉴权头（资源 CDN） | `GET/HEAD /prod-clientpatch.../...` 原样字节 | +| server-info | GET JSON；字段 PascalCase（`ConnectionGroups` 等） | 可选加载并**只改** `AddressablesCatalogUrlRoot` 指向 `{public-base}/prod-clientpatch.../{token}` | +| seed `.hash` | 纯文本十进制（可含空白） | 原样分发 | +| Range / 断点 | 官方客户端下载器用 Range；bat 用 curl `.part` | **本轮 bat-api 可不实现 Range**；记入后续 | +| 业务 ApiUrl/Gateway | 游戏协议 | **不改写、不仿造** | + +Addressables 改写后客户端拼接： + +```text +{rewritten_root}/TableBundles/TableCatalog.bytes +≡ {public-base}/prod-clientpatch.../{token}/TableBundles/TableCatalog.bytes +``` + +与磁盘映射一致。 + +--- + +## 8. RPC 与分发发现顺序 + +`bat-api`（及任何 Go 服务层）发现当前 release： + +1. `daemon.status` +2. `daemon.doctor` +3. `catalog.status`（`version.resource_root`、`addressables_root`、app/bundle） +4. `resource.manifest` 分页（url / destination / bytes / blake3） +5. 在 `resource_root` 上 Lstat 校验 Present / size + +**不读** `bat-status.json` / `bat-tasks.json` 作为常规路径。 + +配置：`--socket` / `BAT_API_SOCKET`；应急 `--resource-root`。见 `cmd/bat-api/.env.example`。 + +--- + +## 9. issue #2 / #3 样本索引（R4/R5 预置） + +在服务器 release 上优先采集到 `/tmp` 隔离目录（**不入库大文件**）： + +| 用途 | 建议路径模式 | +|---|---| +| Addressables（#2） | `{PatchDir}/catalog_*.zip` 解压后的 JSON/bin + 旁路 `.hash` | +| UnityFS（#3） | `FullPatch_*.zip` 内抽样 `.bundle`，或已解包 bundle | +| seed 加固（R2） | 各平台 `TableCatalog` / `BundlePackingInfo` / `MediaCatalog` 的 `.bytes`+`.hash` | + +字段目标（#2，已有 `m_Crc` 部分）：继续扩大 hash/size/CRC/依赖等可校验字段覆盖。 +结构目标（#3）：header / block / directory / metadata / object table 引擎级解析。 + +--- + +## 10. 服务器实勘清单（R1，等 SSH） + +连接信息到位后只读执行： + +1. `readlink current` → version id +2. 顶层是否仅有官方 host 目录 + manifest/snapshot +3. manifest 条目数 vs 磁盘抽样 size +4. RPC 四步（status → doctor → catalog.status → manifest 首页） +5. 将结论写入 `docs/reports/resource-server-survey-YYYYMMDD.md`（无凭据） + +所需： + +```text +SSH: user@host -p PORT +资源目录: .../official +bat.sock 或 state-dir: ... +``` + +--- + +## 11. 相关文档 + +- `docs/reports/GO_STATUS.md` — Go 边界与进度 +- `docs/architecture/official-resource-backend.md` — 拉取后端总览 +- `docs/reference/rpc-backend-api.md` — RPC 契约 +- `docs/guides/official-resource-test-pull.md` — 用户向运行说明 +- `docs/reports/CURRENT_GAPS.md` — G-009 / #2 / #3 + +--- + +## 12. 变更纪律 + +1. 改 URL 模板或落盘规则 → **必须**同步本文 + 相关单测。 +2. 改 inventory 启发式 → 说明覆盖的真实风险并补 fixture。 +3. 真机实勘若发现与本文冲突 → **以真机为准** 修代码与本文，禁止静默分叉。 diff --git a/docs/guides/deployment.md b/docs/guides/deployment.md index 2acf9ec..99fe229 100644 --- a/docs/guides/deployment.md +++ b/docs/guides/deployment.md @@ -227,10 +227,11 @@ systemctl status bluearchive-toolkit-official-sync.service journalctl -u bluearchive-toolkit-official-sync.service -f sudo -u bat /opt/bluearchive-toolkit/bin/bat doctor \ --output /var/lib/bluearchive-toolkit/official \ + --localized-output /var/lib/bluearchive-toolkit/localized \ --state-dir /run/bluearchive-toolkit ``` -`--watch` 是 Rust 内部持久检查模式，正常情况下默认每 1 小时执行一次检查，并且每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会强制执行一次自动刷新。固定时间刷新会中断普通 interval 的 sleep，该轮注入 `force=true`；如果失败，会按失败重试周期继续重试。远端和本地一致时默认静默；有远端变化或本地文件损坏时自动下载或 repair，并输出人类可读摘要。下载、发现或校验失败时默认 60 秒后重试，可显式加 `BAT_ERROR_RETRY=60s` 或调整 service `ExecStart`。默认平台是 `Windows,Android`，无需显式传 `--platforms`；需要覆盖时用 systemd drop-in 重写 `ExecStart`。默认资源目录是 `./bat-resources`，生产 service 显式使用 `/var/lib/bluearchive-toolkit/official`。生产读取方应读取 `/var/lib/bluearchive-toolkit/official/current`；同步中的文件只会进入 `.staging/`，校验完成后才发布为 `versions/` 并切换 `current`。 +`--watch` 是 Rust 内部持久检查模式，正常情况下默认每 1 小时执行一次检查，并且每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 会强制执行一次自动刷新。固定时间刷新会中断普通 interval 的 sleep，该轮注入 `force=true`；如果失败，会按失败重试周期继续重试。远端和本地一致时默认静默；有远端变化或本地文件损坏时自动下载或 repair，并输出人类可读摘要。下载、发现或校验失败时默认 60 秒后重试，可显式加 `BAT_ERROR_RETRY=60s` 或调整 service `ExecStart`。默认平台是 `Windows,Android`，无需显式传 `--platforms`；需要覆盖时用 systemd drop-in 重写 `ExecStart`。默认官方原版资源目录是 `./bat-resources`，默认汉化产物目录是 `./bat-localized`；生产 service 显式使用 `/var/lib/bluearchive-toolkit/official` 和 `/var/lib/bluearchive-toolkit/localized`，两者不能相同或互相嵌套。生产读取方应读取 `/var/lib/bluearchive-toolkit/official/current`；同步中的原版文件只会进入 `.staging/`，校验完成后才发布为 `versions/` 并切换 `current`。官方同步报告 `localized_release_status=not_localized` 表示汉化资源尚未发布；后续 Patch 发布才切换 `/var/lib/bluearchive-toolkit/localized/current`。 ### 可选模式：CLI 自托管 `--daemon` @@ -240,6 +241,7 @@ sudo -u bat /opt/bluearchive-toolkit/bin/bat doctor \ sudo -u bat /opt/bluearchive-toolkit/bin/bat \ --auto-discover \ --output /var/lib/bluearchive-toolkit/official \ + --localized-output /var/lib/bluearchive-toolkit/localized \ --state-dir /var/lib/bluearchive-toolkit/daemon-state \ --daemon @@ -262,9 +264,11 @@ systemd 模式： - 当前可读 release：`/var/lib/bluearchive-toolkit/official/current` - 资源状态：`/var/lib/bluearchive-toolkit/official/current/official-sync-snapshot.json` - 下载 manifest：`/var/lib/bluearchive-toolkit/official/current/official-download-manifest.json` +- 解析缓存：`/var/lib/bluearchive-toolkit/official/current/official-parse-cache.json` - 历史 release：`/var/lib/bluearchive-toolkit/official/versions/` - 同步 staging：`/var/lib/bluearchive-toolkit/official/.staging/` - 资源写锁：`/var/lib/bluearchive-toolkit/official/.official-sync.lock` +- 汉化 release（Patch 发布后）：`/var/lib/bluearchive-toolkit/localized/current` - 运行期目录：`/run/bluearchive-toolkit/` standalone `--daemon` 模式： @@ -279,10 +283,10 @@ standalone `--daemon` 模式： ### 生产维护命令 ```bash -sudo -u bat /opt/bluearchive-toolkit/bin/bat refresh --output /var/lib/bluearchive-toolkit/official -sudo -u bat /opt/bluearchive-toolkit/bin/bat refresh --force --output /var/lib/bluearchive-toolkit/official -sudo -u bat /opt/bluearchive-toolkit/bin/bat verify --output /var/lib/bluearchive-toolkit/official -sudo -u bat /opt/bluearchive-toolkit/bin/bat repair --output /var/lib/bluearchive-toolkit/official +sudo -u bat /opt/bluearchive-toolkit/bin/bat refresh --output /var/lib/bluearchive-toolkit/official --localized-output /var/lib/bluearchive-toolkit/localized +sudo -u bat /opt/bluearchive-toolkit/bin/bat refresh --force --output /var/lib/bluearchive-toolkit/official --localized-output /var/lib/bluearchive-toolkit/localized +sudo -u bat /opt/bluearchive-toolkit/bin/bat verify --output /var/lib/bluearchive-toolkit/official --localized-output /var/lib/bluearchive-toolkit/localized +sudo -u bat /opt/bluearchive-toolkit/bin/bat repair --output /var/lib/bluearchive-toolkit/official --localized-output /var/lib/bluearchive-toolkit/localized sudo -u bat /opt/bluearchive-toolkit/bin/bat doctor --output /var/lib/bluearchive-toolkit/official --state-dir /run/bluearchive-toolkit sudo -u bat /opt/bluearchive-toolkit/bin/bat clean-stable --output /var/lib/bluearchive-toolkit/official --state-dir /run/bluearchive-toolkit ``` diff --git a/docs/guides/development.md b/docs/guides/development.md index 799176d..5ba1130 100644 --- a/docs/guides/development.md +++ b/docs/guides/development.md @@ -141,11 +141,18 @@ git push origin feature/your-feature-name cargo fmt --check cargo test --workspace cargo clippy --workspace --all-targets -- -D warnings -go test ./... -go vet ./... +make test-go-api +make build-go-api +go vet ./internal/api/... ./internal/backendrpc/... ./cmd/bat-api/... ``` -Go 产品入口尚未完成，但仓库已有 `internal/backendrpc` Rust daemon RPC client、`cmd/bat` 试验骨架与 `internal/ffi` 兼容包装。提交前应运行 `go test ./...` 和 `go vet ./...`；`internal/backendrpc` 使用 fake transport 覆盖 JSON-RPC envelope、错误和 typed helper，不能把这些测试误认为 Go 产品级 CLI/API 已完成。 +Go 边界与进度以 `docs/reports/GO_STATUS.md` 为准： + +- **同步/运维命令行** = Rust `bat`（近乎全自动） +- **资源分发服务** = `cmd/bat-api`（`make build-go-api`） +- **默认 Go 门禁** = `make test-go-api`（无 FFI） +- 试验 CLI 产物为 `bin/bat-go`（`make build-go-cli`），**禁止**与 Rust `bat` 重名 +- 修改 FFI 时再跑 `make test-go-ffi` ### 常用聚焦命令 @@ -160,7 +167,7 @@ cargo clippy -p bat-core -p bat-adapters -p bat-infrastructure --all-targets -- 官方资源同步、下载、daemon、status、verify 或 repair 相关改动必须至少覆盖 `bat-infrastructure` 和 `bat` 二进制测试。 -`bat-ffi` 只是可选无状态 C ABI 兼容层。修改 FFI 导出、JSON schema、错误返回或 `internal/ffi` CGO 包装时必须运行 `cargo test -p bat-ffi -- --nocapture`；未来 Go 产品入口和生产同步默认应通过 `internal/backendrpc` 调用 daemon RPC，或在 one-shot/fallback 场景使用 Rust `bat --json` 进程边界。 +`bat-ffi` 只是可选无状态 C ABI 兼容层。修改 FFI 时必须运行 `cargo test -p bat-ffi -- --nocapture`。Go 服务层默认经 `internal/backendrpc` 调 daemon；同步任务由 Rust `bat` 执行，不由 Go 试验 CLI 承担。 ### 集成测试 @@ -186,7 +193,7 @@ cargo run -p bat-infrastructure --bin bat -- \ --dry-run ``` -开发环境真实下载默认写入 `./bat-resources`；如果要覆盖，必须使用 `/tmp` 或其他隔离目录，不要写入现有资源目录。 +开发环境真实官方资源下载默认写入 `./bat-resources`；汉化产物默认写入独立的 `./bat-localized`。如果要覆盖，官方原版资源使用 `--output` / `BAT_OUTPUT`，汉化产物使用 `--localized-output` / `BAT_LOCALIZED_OUTPUT`。两者都必须使用 `/tmp` 或其他隔离目录，不要写入现有资源目录，也不要把汉化输出覆盖到官方原版资源目录。 生产或 CI 环境不得依赖安装官方启动器。需要启动器信息时，只能分析启动器资源、官方 manifest 或公开更新数据，并将解析结果固化为可验证流程。 diff --git a/docs/guides/official-resource-test-pull.md b/docs/guides/official-resource-test-pull.md index 44d9d6a..a6d5e11 100644 --- a/docs/guides/official-resource-test-pull.md +++ b/docs/guides/official-resource-test-pull.md @@ -45,7 +45,7 @@ target/release/bat \ --watch ``` -默认资源输出目录是 `./bat-resources`，默认后台状态目录是 `/tmp/bat-pid`。资源输出目录是发布根目录：非 dry-run 同步先写 `/.staging/`，校验完成后移动到 `/versions/`，再原子切换 `/current` symlink；生产读取方应读取 `current`。后台状态目录会保存 `bat.sock`、`bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl` 和短生命周期的 `bat-control.lock`；其中 `bat.sock` 是 live daemon 的 Unix socket JSON-RPC 控制通道，`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志，`bat-status.json` 保存最后成功时间、下次检查时间、最后错误摘要和当前下载进度，`bat-control.lock` 串行化 `status/stop/restart/reload/logs/refresh/repair` 等控制命令。生产资源输出目录必须是独立目录；需要覆盖时用 `--output <资源目录>`，不要使用已有游戏客户端目录、官方启动器安装目录、人工维护资源目录，或开发机上的 `/home/wanye/D/BlueArchive`。 +默认官方原版资源输出目录是 `./bat-resources`，默认汉化产物目录是 `./bat-localized`，默认后台状态目录是 `/tmp/bat-pid`。官方资源输出目录是发布根目录：非 dry-run 同步先写 `/.staging/`，校验完成后移动到 `/versions/`，再原子切换 `/current` symlink；生产读取方应读取 `current`。后台状态目录会保存 `bat.sock`、`bat.pid`、`bat-status.json`、`bat-daemon.log`、`bat-events.jsonl` 和短生命周期的 `bat-control.lock`；其中 `bat.sock` 是 live daemon 的 Unix socket JSON-RPC 控制通道，`bat-events.jsonl` 是带轮转的结构化 JSONL 事件日志，`bat-status.json` 保存最后成功时间、下次检查时间、最后错误摘要和当前下载进度，`bat-control.lock` 串行化 `status/stop/restart/reload/logs/refresh/repair` 等控制命令。生产官方资源目录和汉化产物目录都必须是独立目录；需要覆盖官方目录时用 `--output <资源目录>`，需要覆盖汉化目录时用 `--localized-output <目录>` 或 `BAT_LOCALIZED_OUTPUT`，不要使用已有游戏客户端目录、官方启动器安装目录、人工维护资源目录，或开发机上的 `/home/wanye/D/BlueArchive`。 同步流程会拒绝危险输出目录、路径逃逸和现有 symlink 路径组件；下载目标、`.part`、manifest、snapshot、PID、status、log 和控制锁文件不会跟随 symlink，daemon 状态类文件默认以 `0600` 权限创建。 @@ -210,6 +210,8 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - 真实更新会输出 `downloaded_count`、`resumed_count`、`skipped_count`、`transferred_bytes`、`official_seed_hash_verified_count`。 - 校验报告分层输出 `official_seed_hash_verified_count`、`local_manifest_verified_count`、`addressables_marker_checked_count`、`unverified_marker_count`。 - 下载阶段复用同一套本地清单、ZIP 结构校验和 `.part` 续传逻辑；没有清单或校验不匹配的文件会重新下载。 +- 校验和发布完成后会刷新 `/current/official-parse-cache.json`。解析缓存从 `official-download-manifest.json` 的全部条目出发，处理直接 UnityFS bundle 和 zip 内 UnityFS 条目；catalog、hash、媒体等非 UnityFS 文件记录为不支持，不视为同步失败。本地 URL、相对路径、size 和 BLAKE3 未变化时复用缓存并跳过重复解析。 +- 官方同步报告中的 `localized_release_status=not_localized` 表示原版资源已发布、汉化资源未发布，这是当前官方同步阶段的正常完成状态；后续 Patch 发布完成后才应切换为 `localized`，表示原版和汉化两套资源都已发布。 资源同步状态文件默认分布如下： @@ -217,6 +219,7 @@ cargo run -p bat-infrastructure --example official_pull_plan -- \ - `/official-bootstrap-cache.json`：`--auto-discover` 的 `GameMainConfig` 解析缓存。launcher metadata 未变时复用缓存；metadata 变化时才通过官方 HTTP 按 manifest 下载必要 `resources.assets` 或旧版 game zip 到临时目录解析。 - `/official-version-state.json`：资源发布根目录的持久版本状态，包含当前已完成版本、正在拉取版本、上一个可用版本和失败版本。 - `/current/official-download-manifest.json`：本地下载强校验清单，记录 URL、相对路径、size 和 BLAKE3。 +- `/current/official-parse-cache.json`：官方资源发布后的派生解析缓存，记录 bundle/zip 条目解析摘要和缓存复用情况；它不是汉化产物。 - `/current/official-download-quarantine.json` 或当前 staging 下同名文件：下载最终失败的 URL 诊断记录，包含失败类型、HTTP 状态、是否可重试、尝试次数和最后错误。 先 dry-run： @@ -251,7 +254,7 @@ cargo run -p bat-infrastructure --bin bat -- \ --watch ``` -后台自动运行使用 `--daemon`。它会启动一个脱离终端的 watch 子进程，资源默认写入 `./bat-resources`，后台控制和状态默认写入 `/tmp/bat-pid`： +后台自动运行使用 `--daemon`。它会启动一个脱离终端的 watch 子进程，官方原版资源默认写入 `./bat-resources`，汉化产物默认写入 `./bat-localized`，后台控制和状态默认写入 `/tmp/bat-pid`： ```bash cargo run -p bat-infrastructure --bin bat -- \ @@ -310,7 +313,7 @@ cargo run -p bat-infrastructure --bin bat -- \ 默认平台是 `Windows,Android`，无需显式传 `--platforms`；只有要覆盖默认平台时才传。`--interval` 是正常检查周期，默认 `1h`；watch/daemon 模式还会在每天北京时间（UTC+8）`03:00`、`16:00`、`18:00` 强制执行一次自动刷新，该轮会注入 `force=true`，并且会中断普通 interval 的 sleep。`--error-retry` 是下载、发现或校验失败后的重试周期，默认 `60s`，也可以用 `--error-retry-seconds 60`。CLI 默认启动时向 stderr 打印 `BlueArchiveToolkit` ASCII banner，并把阶段进度日志写到 stderr，包括自动发现、proxy、server-info、marker、catalog、audit、download、snapshot 和 publish 阶段；download 阶段会输出已完成计数和单文件开始/完成状态，下载执行保持顺序处理，已完成计数保持单调不倒退，audit 阶段会输出官方 `.hash`、本地 BLAKE3、需修复项和 ZIP 结构校验结果摘要。daemon 还会写 `bat-events.jsonl` 结构化日志并按大小轮转。命令结果默认以人类可读摘要写到 stdout。需要纯机器输出时加 `--json --no-progress`，需要显式开启进度日志则用 `--progress`；只想关闭横幅但保留日志时可加 `--no-banner`。错误时 stderr 输出 JSON error，watch 模式下错误 JSON 的 `next_retry_seconds` 使用失败重试周期；如果未关闭 progress，错误 JSON 前可能已有 banner 和进度日志。普通错误 exit `1`，资源目录锁冲突 exit `75`，`verify` 或 `doctor` 发现问题也返回非 0。 -生产可以直接运行 `--watch`，也可以用 `--daemon` 后台运行，或者用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产资源目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录；上层读取资源时应读取 `--output/current`，不要读取 `.staging` 或 `versions` 中未切换的目录。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一资源目录；live daemon 还会阻止前台写命令直接修改它正在管理的同一目录。 +生产可以直接运行 `--watch`，也可以用 `--daemon` 后台运行，或者用 systemd service、容器或 Go 进程守护它。cron/systemd timer 仍可调用单次模式，但不再是 Rust 自动更新的唯一方式。项目是否热更新、热重载或重启进程，由上层业务集成决定。生产官方资源目录应使用独立输出目录，不要指向现有客户端或人工维护的资源目录；上层读取原版资源时应读取 `--output/current`，不要读取 `.staging` 或 `versions` 中未切换的目录。汉化 Patch/导出应写入 `--localized-output`，并保留官方相对目录结构，不能写回 `--output/current`。发布状态分两档：`not_localized` 只发布原版资源、不发布汉化资源；`localized` 发布原版和汉化两套资源。非 dry-run 每轮会创建 `--output/.official-sync.lock`，防止并发写同一官方资源目录；live daemon 还会阻止前台写命令直接修改它正在管理的同一目录。 需要只做探测时可以加 `--dry-run`。需要关闭本地 audit 或 repair 时可以显式使用 `--no-audit-local` 或 `--no-repair`，但生产同步默认应保持开启。 diff --git a/docs/reports/CURRENT_GAPS.md b/docs/reports/CURRENT_GAPS.md index c927c52..68445be 100644 --- a/docs/reports/CURRENT_GAPS.md +++ b/docs/reports/CURRENT_GAPS.md @@ -1,6 +1,8 @@ # 当前实现缺口清单 -- **更新时间**：2026-07-20 +- **更新时间**：2026-07-24 +- **Go 进度权威**：`GO_STATUS.md` +- **资源布局 / 逆向契约**：`../architecture/resource-release-layout.md` - **用途**：集中跟踪当前代码中的占位实现、设计缺口和下一步验收项。 - **权威计划**：`../../PROJECT_PLAN.md` @@ -109,24 +111,40 @@ ### G-005：AssetBundle 引擎解析器仍未完成 +状态：**部分完成** + 现象： -- `crates/bat-assetbundle/src/parser.rs` 只有 `Parser::name`。 -- `types.rs` 只有 `AssetType::TextAsset`。 -- `adapters/src/unity/unity_2021_3.rs` 已能解析 UnityFS header、block info、directory，并校验 directory `offset+size` 不越界；这属于 adapter 层基础摘要能力，不等于 `bat-assetbundle` 引擎已完成。 -- 仍没有对象表、TypeTree、TextAsset、MonoBehaviour、ScriptableObject 或可扩展提取入口。 +- `crates/bat-assetbundle` 已接管 UnityFS 解析，提供 `UnityFsParser`、`UnityFsBundle`、header、block info、directory、压缩模式、block info at end、LZ4/LZMA block info 解压、数据 block 解压、directory 文件提取和边界诊断。 +- `crates/bat-assetbundle::serialized` 已提供 Unity serialized file header、type table、TypeTree node 元数据、object table 和 `TextAsset` bytes 提取。 +- `adapters/src/unity/unity_2021_3.rs` 已降为 Unity 版本选择薄层，复用 `bat-assetbundle`，不再维护第二套 UnityFS parser。 +- `ResourceImportService` 的 UnityFS 摘要已经能暴露解包文件数、serialized file 数、TextAsset 数量和名称。 +- 仍没有 `MonoBehaviour`、`ScriptableObject` 的 TypeTree 字段级反序列化和可编辑重打包入口。 影响： -- 可以对部分 UnityFS 样本做基础结构校验，但无法完成真实资源对象解析和文本提取。 -- 无法提取 TextAsset 或配置文本。 +- 可以对 UnityFS 容器做结构校验、解包 directory 文件，并提取 serialized file 中的 TextAsset 原始 bytes。 +- 对日语汉化最关键的 TextAsset 索引和 bytes 提取已有基础入口，但还不能直接解析 `MonoBehaviour`/`ScriptableObject` 自定义字段或完成修改后重打包。 -验收： +当前验收证据： - `crates/bat-assetbundle` 能解析结构化测试样本和隔离真实样本。 -- 支持 UnityFS header、blocks、directory、metadata、object table。 +- 支持 UnityFS header、blocks、directory、metadata 摘要、directory 文件提取。 +- 支持 Unity serialized file object table、TypeTree node 元数据和 TextAsset 提取的合成 fixture。 - 错误包含偏移和字段上下文。 +关闭前仍需： + +- 完成 TypeTree 字段 reader：bool、integer、float、string、bytes、array、map、PPtr 和 managed reference 诊断占位。 +- 支持 MonoBehaviour、ScriptableObject 字段级遍历，形成可扩展文本提取入口。 +- 输出可追溯文本定位：bundle path、archive entry、serialized file、path id、class id、field path。 +- 用真实资源 fixture 覆盖对象级解析、TextAsset 提取和字段级文本提取。 +- 将解析结果作为 Patch 输入；真正的重打包、Patch 生成和 `localized` 发布切换归 G-006/G-011D。 + +解析补全路线图： + +- 见 `docs/architecture/assetbundle.md` 的 P2/P3/P5。 + ### G-006：Patch 引擎仍是占位 现象： @@ -153,8 +171,9 @@ - `AddressablesCatalogDriver` 已能解析当前真实形态 JSON catalog fixture/golden。 - 已输出 path、hash、size、resource_type、address、dependencies、metadata，并已提取 `m_Crc` 到 `crc` 字段。 +- compact catalog 解析已补充 hash/size/CRC 的非 0 回归、资源计数 metadata，以及 blob 解码失败时的明确错误；不再在 compact 字段损坏时静默退回低保真 `m_InternalIds`。 - `bat-core` 已提供 `crc32_ieee` 和 `ResourceEntry::verify_downloaded_bytes`，SQLite `ResourceRepository` 已有 `crc` 列迁移。 -- 仍需覆盖更多官方 catalog 结构变体、二进制/压缩字段组合和更明确的失败诊断。 +- 仍需覆盖更多官方 catalog 结构变体、provider/bundle name 持久化字段，以及真实 Windows/Android catalog 样本集合。 影响： @@ -165,54 +184,53 @@ - 能解析项目目标版本的真实 Catalog 样本集合。 - 解析结果包含资源 key、provider、dependency、hash、size、path、CRC。 - 对不支持的 catalog 结构返回明确错误，而不是静默丢字段。 +- 解析结果能反查 bundle 文件、依赖链和本地下载 manifest 条目。 +- Windows/Android 样本集合需要覆盖 JSON、compact JSON 和后续二进制 catalog 入口。 + +解析补全路线图： + +- 见 `docs/architecture/assetbundle.md` 的 P1。 --- ## 3. 应用层缺口 -### G-008：Go CLI 产品入口尚未完成 +### G-008：Go 同步/运维 CLI 产品入口 -状态：**未完成（此前“并入 G-009”只是短期跟踪调整，不代表能力完成）** +状态：**已决策关闭（wontfix）** -现象： +决策（2026-07-24，见 `GO_STATUS.md`）： -- 当前可用的用户同步/运维入口是 Rust `bat` binary。 -- `internal/backendrpc` 已提供 Go 到 Rust daemon 的 typed JSON-RPC client；`cmd/bat` 已存在，但仅有 `doctor`、`manifest inspect`、`sync plan` 试验能力；`doctor` 只输出固定 `ok`，`manifest`/`sync` 依赖可选 CGO/FFI helper。 -- Go 侧尚未实现通过 Rust `bat --json` 或 daemon RPC 包装官方同步命令、稳定 human/json 输出、真实 doctor 检查和端到端测试。 -- 如果项目决策改为“用户 CLI 永久由 Rust `bat` 承担，Go 只做 `bat-api`/服务层”，必须同步更新 `AGENTS.md`、`PROJECT_PLAN.md` 和 issue 跟踪；在完成该决策前，不能把 Go CLI 写成已完成。 +- **正式同步/运维命令行 = Rust `bat`**（近乎全自动：auto-discover + watch/daemon，无需持久手操维护）。 +- **不另做**产品级 Go 同步 CLI，避免与 Rust `bat` 双轨。 +- Go 试验入口 `cmd/bat` 可保留为 experimental，产物必须为 `bin/bat-go`，**禁止**再构建为 `bin/bat`。 +- Go 正式产品入口集中在 **`bat-api` 资源分发服务** + `internal/backendrpc`（G-009）。 -验收： +原验收（真实 doctor / Go sync 包装）**不再作为当前里程碑**。 -- `cmd/bat doctor` 做真实环境诊断，而不是固定字符串。 -- `cmd/bat sync` 能通过 `internal/backendrpc` daemon RPC 或 Rust `bat --json` one-shot/fallback 触发/查询官方同步，不走 FFI 控制下载器或 daemon。 -- human/json 输出、退出码和错误码与 Rust `bat` 契约一致。 -- `go test ./...`、`go vet ./...` 覆盖命令解析、错误输出和至少一个 mocked Rust 边界。 +### G-009：API Server（`bat-api`，资源分发）部分完成 -### G-009：API Server（`bat-api`，仿官方 API）尚未实现 +状态：**资源 CDN MVP 已落地；非完整官方游戏 API** -现象： +目标（对应 issue #19，**按资源面收窄**）： -- `api/` 只有目录结构，无 handler、service、路由。 -- Go 侧尚无对接 daemon RPC / `current/` 发布布局的服务端入口。 +- `cmd/bat-api`：只读分发 Rust `bat` 已发布 release（官方 CDN host/path 形态）。 +- **拉取归属 Rust `bat`**；`bat-api` 不做下载器。 +- 发现经 `bat.sock`：先 `daemon.status`，再 `daemon.doctor`，再 `catalog.status` / `resource.manifest`。 +- `.env` 配置端口 / public base / RPC socket；预留 database/redis。 +- launcher 全链、完整业务 API **非关闭条件**；USERGUIDE bat-api 专章延后。 -目标（对应 issue #19）： +已完成： -- 新建 `cmd/bat-api`：**完全仿照 BlueArchive 官方 API** 的 Go HTTP 服务，把 Rust `bat` 后端发布的 `current/` release 按官方接口形态对外提供，使真实客户端/工具可将其当作官方服务端。 -- 仿真面：资源 CDN 面（TableCatalog/MediaCatalog/BundlePackingInfo、bundle、seed `.hash`）、server-info 面、launcher API 面。 -- 鉴权/签名**完全仿照**官方实现，服务端做验签（对齐 `adapters/src/official/launcher.rs` 的签名逻辑）。 -- 版本/状态经 daemon RPC（`catalog.*` / `resource.manifest`）发现，资源字节从 `current/` 读取；不读写 daemon 状态文件内部。 +- `cmd/bat-api`、`internal/api`、fixture 单测、`make build-go-api` / `test-go-api` +- 进度权威：`docs/reports/GO_STATUS.md` -影响： +验收（剩余）： -- 客户端、工具和第三方集成无服务端入口。 +- 与全量 release / 服务器 daemon 联调（SSH 实勘可后置） +- 文档与 GO_STATUS 持续一致 -验收： - -- Go 单测覆盖路由、签名验签、错误响应形态。 -- 真机 e2e：daemon 发布 fixture release → 启动 `bat-api` → 按官方 URL 与鉴权头请求 server-info / catalog / bundle / launcher 链，断言字节与形态正确、验签生效（缺签/错签被拒）。 -- 统一错误结构与官方响应 envelope 对齐；Makefile 增加 Go 构建/测试目标。 - -排期：P2，排在 issue #17 验收收口以及 issue #2 / #3 的解析能力继续推进之后启动；可与 G-008 的 Go 产品入口边界收敛并行。 +排期：P2 主体可联调；持久化 API 层与 launcher 另议。 ### G-010：Web 管理后台尚未实现 @@ -248,6 +266,11 @@ - schema 和迁移可重复执行。 - 可按版本、类型、hash、路径查询资源。 - 官方同步后的资源可通过 CLI 查询并能追溯到 CAS 对象。 +- `official-parse-cache.json` 的 bundle、zip entry、TextAsset 摘要能进入 ResourceRepository 查询面。 + +解析补全路线图： + +- 见 `docs/architecture/assetbundle.md` 的 P4。 ### G-011A：资源导入链路基础能力不足 @@ -294,6 +317,30 @@ - `cargo test -p bat-infrastructure version_state` - `cargo test -p bat-infrastructure --test official_game_main_config_bootstrap` +### G-011D：汉化发布状态与 Patch 发布流程未完成 + +状态：**新建，未关闭** + +当前已完成： + +- 官方原版资源发布根为 `./bat-resources`，汉化产物发布根为 `./bat-localized`。 +- CLI 支持 `--localized-output` / `BAT_LOCALIZED_OUTPUT`，并拒绝官方目录和汉化目录相同或互相嵌套。 +- 官方同步报告新增 `localized_release_status=not_localized`，明确表示原版资源已发布、汉化资源未发布。 + +仍未完成： + +- Patch 发布阶段尚未生成 `localized-output/versions/`。 +- 尚未维护 `localized-output/current` 原子指针和汉化版本状态文件。 +- 尚未实现 `localized` 状态切换、回滚和汉化产物完整性校验。 +- 尚未实现原版资源与汉化资源双发布后的查询、分发和清理策略。 + +验收： + +- 原版资源同步成功后保持 `not_localized`，不发布半成品汉化资源。 +- Patch 构建和校验成功后，汉化产物按官方相对路径写入 `localized-output/versions/`。 +- 汉化发布必须原子切换 `localized-output/current`，失败时不影响已发布原版资源。 +- `localized` 状态能证明原版和汉化两套资源都可发布，并能被 CLI/RPC/API 查询。 + ### G-011C：真实 fixture 与回归样本不足 状态：**已关闭当前阶段** @@ -470,13 +517,15 @@ ## 6. 当前关闭顺序建议 -1. issue #24：失败 staging 复用已补离线回归；继续核对 issue 口径、状态与后续是否仍有真实场景无法复现的残余。 -2. issue #1：Rust daemon/backend API 边界收口；`resource.repair`、`resource.list`、`daemon.doctor`、`internal/backendrpc` Go client 和稳定 RPC reference 已补齐，剩余确认 `patch.*` / `unityfs.*`（待引擎）、`task.create`（暂不开放通用入口）以及 `daemon.restart` / `daemon.clean-stable`（CLI 生命周期入口）的 issue 验收口径。 -3. issue #17/#20/#21/#22：多线程下载入口已按最新决定移除，下载回归顺序执行并保留指数退避与单调进度上报；daemon 子进程不再透传并发参数，但 GitHub issue 仍 open。 -4. issue #2 / G-007：继续扩大 Addressables 可校验字段和结构变体覆盖。 -5. issue #3 / G-005：把 UnityFS 基础摘要推进到 `bat-assetbundle` 引擎级解析。 -6. G-011：官方同步结果接入 CAS + ResourceRepository 用户级工作流。 -7. G-008 / G-009：收敛 Go 产品入口边界，并实现 `bat-api`（issue #19）。 -8. G-012 / G-006：翻译系统、Patch 引擎。 +1. issue #24：失败 staging 复用回归已补；核对残余场景。 +2. issue #1：RPC 主体已落地；剩余 `patch.*` / `unityfs.*`、设计边界确认。 +3. issue #17 及子 issue：已按 wontfix 关闭多线程下载（顺序下载 + 指数退避）。 +4. **G-008：已决策关闭**（同步 CLI = Rust `bat`；见 `GO_STATUS.md`）。 +5. **G-009 / issue #19**：资源分发 MVP 已编码；优先服务器联调与索引实勘，非「从零实现」。 +6. issue #2 / G-007（P1）：Addressables 可校验字段。 +7. issue #3 / G-005（P1）：UnityFS 容器基础解析已落地；对象级引擎解析继续跟踪 G-005。 +8. G-011：官方同步结果接入 CAS + ResourceRepository 用户级工作流。 +9. G-011D / G-006：汉化发布状态持久化、Patch 发布和回滚。 +10. G-012 / G-006：翻译系统、Patch 引擎。 -这个顺序优先把 Rust `bat` 后端做扎实（解析能力 + 下载性能），再把官方同步结果进入可查询资源库，之后收敛 Go 入口和仿官方 API 服务端，最后推进翻译和补丁。G-018 已固化为可重复 smoke 命令并关闭；G-017 已按“不引入托管 CI”决策关闭。 +Go 进度以 `docs/reports/GO_STATUS.md` 为准。G-018 / G-017 已关闭。 diff --git a/infrastructure/Cargo.toml b/infrastructure/Cargo.toml index 2d2c361..cc2aa64 100644 --- a/infrastructure/Cargo.toml +++ b/infrastructure/Cargo.toml @@ -12,6 +12,7 @@ path = "src/bin/bat_official_sync.rs" [dependencies] bat-core = { path = "../core" } bat-adapters = { path = "../adapters" } +bat-assetbundle = { path = "../crates/bat-assetbundle" } bat-cas-engine = { path = "../crates/bat-cas-engine" } anyhow.workspace = true thiserror.workspace = true diff --git a/infrastructure/src/bin/bat_official_sync.rs b/infrastructure/src/bin/bat_official_sync.rs index 0568c0e..64b3ce1 100644 --- a/infrastructure/src/bin/bat_official_sync.rs +++ b/infrastructure/src/bin/bat_official_sync.rs @@ -52,7 +52,7 @@ const BEIJING_UTC_OFFSET_SECONDS: u64 = 8 * 60 * 60; const DAILY_FORCED_REFRESH_LOCAL_SECONDS: [u64; 3] = [3 * 60 * 60, 16 * 60 * 60, 18 * 60 * 60]; const DAILY_FORCED_REFRESH_LABEL: &str = "UTC+8 03:00, 16:00, 18:00"; const STARTUP_BANNER: &str = r#" -============================================================ +===================================================================================== ____ _ _ _ _ _____ _ _ _ _ | __ )| |_ _ ___ / \ _ __ ___| |__ (_)_ _____|_ _|__ ___ | | | _(_) |_ | _ \| | | | |/ _ \/ _ \ | '__/ __| '_ \| \ \ / / _ \ | |/ _ \ / _ \| | |/ / | __| @@ -61,7 +61,7 @@ const STARTUP_BANNER: &str = r#" BlueArchiveToolkit Official Resource Sync -============================================================ +===================================================================================== "#; fn main() { @@ -3104,6 +3104,7 @@ fn sync_command_rpc_method(options: &CliOptions, command_name: &str) -> Option<& && options.config.launcher_version == defaults.launcher_version && options.config.platforms.is_none() && options.config.output_root == defaults.output_root + && options.config.localized_output_root == defaults.localized_output_root && options.config.snapshot_path.is_none() && options.config.curl_command == defaults.curl_command && options.config.curl_proxy == defaults.curl_proxy @@ -3122,6 +3123,7 @@ fn sync_command_rpc_method(options: &CliOptions, command_name: &str) -> Option<& } } +#[cfg(test)] fn refresh_should_use_daemon_rpc(options: &CliOptions, command_name: &str) -> bool { sync_command_rpc_method(options, command_name) == Some(RPC_METHOD_REFRESH) } @@ -3456,7 +3458,14 @@ impl HumanReport for OfficialUpdateReport { print_field("本地审计", format_bool(self.audit_local)); print_field("自动修复", format_bool(self.repair)); print_field("dry-run", format_bool(self.dry_run)); - print_path_field("输出根目录", &self.output_root); + print_path_field("官方资源目录", &self.output_root); + print_path_field("汉化输出目录", &self.localized_output_root); + print_field("汉化发布状态", self.localized_release_status.as_str()); + print_path_field("汉化 current", &self.localized_current_path); + print_optional_path_field( + "汉化 published", + self.localized_published_version_path.as_ref(), + ); print_path_field("active release", &self.active_resource_root); print_path_field("current", &self.current_path); print_path_field("version state", &self.version_state_path); @@ -3464,6 +3473,7 @@ impl HumanReport for OfficialUpdateReport { print_optional_path_field("published", self.published_version_path.as_ref()); print_path_field("snapshot", &self.snapshot_path); print_path_field("manifest", &self.download_manifest); + print_optional_path_field("解析缓存", self.parse_cache_path.as_ref()); print_optional_path_field("写入 snapshot", self.snapshot_written.as_ref()); print_optional_path_field("bootstrap cache", self.bootstrap_cache_path.as_ref()); print_optional_field("bootstrap 命中", self.bootstrap_cache_hit.map(format_bool)); @@ -3478,6 +3488,14 @@ impl HumanReport for OfficialUpdateReport { print_field("需修复", self.local_manifest_repair_needed_count); print_field("官方 hash 校验", self.official_seed_hash_verified_count); print_verification_summary(&self.verification_summary); + if let Some(summary) = self.parse_summary.as_ref() { + print_field("解析缓存条目", summary.cache_entry_count); + print_field("解析成功 bundle", summary.parsed_bundle_count); + print_field("解析复用", summary.skipped_unchanged_count); + print_field("解析不支持", summary.unsupported_count); + print_field("解析失败", summary.failed_count); + print_field("TextAsset", summary.text_asset_count); + } print_field("catalog marker", self.addressables_marker_checked_count); print_list("变更 endpoint", &self.changed_endpoint_urls, 8); print_list("计划 URL", &self.download_urls, 8); @@ -3868,11 +3886,21 @@ fn build_doctor_report( let mut checks = vec![ path_check("state_dir", state_dir, "后台状态目录可用"), path_check("output_root", &config.output_root, "资源输出目录可用"), + path_check( + "localized_output_root", + &config.localized_output_root, + "汉化输出目录可用", + ), safety_check( "output_root_safety", validate_output_root(&config.output_root), "资源输出目录安全边界通过", ), + safety_check( + "localized_output_root_safety", + validate_output_root(&config.localized_output_root), + "汉化输出目录安全边界通过", + ), safety_check( "state_dir_safety", validate_runtime_state_dir(state_dir), @@ -4524,6 +4552,8 @@ fn daemon_child_args(options: &CliOptions) -> Vec { } args.push("--output".to_string()); args.push(config.output_root.to_string_lossy().to_string()); + args.push("--localized-output".to_string()); + args.push(config.localized_output_root.to_string_lossy().to_string()); if let Some(snapshot_path) = config.snapshot_path.as_ref() { args.push("--snapshot".to_string()); args.push(snapshot_path.to_string_lossy().to_string()); @@ -4937,6 +4967,7 @@ fn stage_to_static(stage: &str) -> &'static str { "download" => "download", "snapshot" => "snapshot", "publish" => "publish", + "parse" => "parse", "finish" => "finish", "watch" => "watch", "daemon" => "daemon", @@ -4968,6 +4999,7 @@ fn localized_stage(stage: &str) -> &str { "dry-run" => "试运行", "download" => "下载", "publish" => "发布", + "parse" => "解析", "resource" => "资源", "finish" => "完成", "watch" => "常驻", @@ -5003,8 +5035,10 @@ const ENV_TEMPLATE: &str = r#"# BlueArchive Toolkit 配置文件（bat 首次启 # 设 BAT_SKIP_ENV_FILE=1 可让 bat 完全忽略本文件。 # ---- 基本配置 ---- -# 资源发布根目录（默认 ./bat-resources，相对当前工作目录） +# 官方原版资源发布根目录（默认 ./bat-resources，相对当前工作目录） BAT_OUTPUT=./bat-resources +# 汉化产物输出根目录（默认 ./bat-localized，与官方原版资源分离） +BAT_LOCALIZED_OUTPUT=./bat-localized # 自动发现 app-version / connection-group / server-info（无参启动建议保持 1） BAT_AUTO_DISCOVER=1 # 后台状态目录（bat.sock / 日志 / 任务历史等；默认 /tmp/bat-pid） @@ -5175,6 +5209,9 @@ fn apply_bat_env_overrides( if let Some(v) = value("BAT_OUTPUT") { options.config.output_root = PathBuf::from(v); } + if let Some(v) = value("BAT_LOCALIZED_OUTPUT") { + options.config.localized_output_root = PathBuf::from(v); + } if let Some(v) = value("BAT_STATE_DIR") { options.state_dir = PathBuf::from(v); } @@ -5343,6 +5380,11 @@ fn parse_args_with_env( options.config.output_root = PathBuf::from(next_option_value(&mut args, &flag)?); options.output_explicit = true; } + "--localized-output" => { + options.config.localized_output_root = + PathBuf::from(next_option_value(&mut args, &flag)?); + options.output_explicit = true; + } "--state-dir" | "--pid-dir" => { options.state_dir = PathBuf::from(next_option_value(&mut args, &flag)?); } @@ -5707,7 +5749,10 @@ fn print_usage(binary: &str) { eprintln!("Sync:"); eprintln!(" --platforms Platforms, e.g. Windows,Android"); eprintln!( - " --output Resource publish root (default: ./bat-resources)" + " --output Official resource publish root (default: ./bat-resources)" + ); + eprintln!( + " --localized-output Localized output root (default: ./bat-localized)" ); eprintln!(" --snapshot Override snapshot path (default: /current/official-sync-snapshot.json)"); eprintln!(" --curl curl executable (default: curl)"); @@ -5739,7 +5784,10 @@ fn print_usage(binary: &str) { eprintln!(); eprintln!("Defaults:"); eprintln!(" platforms: Windows,Android"); - eprintln!(" resource output: ./bat-resources (current -> versions/, .staging/)"); + eprintln!( + " official resource output: ./bat-resources (current -> versions/, .staging/)" + ); + eprintln!(" localized output: ./bat-localized (separate patch/export target)"); eprintln!(" daemon state: /tmp/bat-pid (bat.sock, bat.pid, bat-status.json, bat-daemon.log, bat-events.jsonl)"); eprintln!(" forced refresh: {DAILY_FORCED_REFRESH_LABEL}"); } @@ -5820,6 +5868,7 @@ mod tests { &["bat"], &[ ("BAT_OUTPUT", "/srv/bat"), + ("BAT_LOCALIZED_OUTPUT", "/srv/bat-localized"), ("BAT_AUTO_DISCOVER", "1"), ("BAT_STATE_DIR", "/srv/state"), ("BAT_INTERVAL_SECONDS", "120"), @@ -5827,6 +5876,10 @@ mod tests { ) .unwrap(); assert_eq!(options.config.output_root, PathBuf::from("/srv/bat")); + assert_eq!( + options.config.localized_output_root, + PathBuf::from("/srv/bat-localized") + ); assert!(options.config.auto_discover); assert_eq!(options.state_dir, PathBuf::from("/srv/state")); assert_eq!(options.interval, Duration::from_secs(120)); @@ -5946,6 +5999,7 @@ mod tests { keys.push(key); } assert!(keys.contains(&"BAT_OUTPUT".to_string())); + assert!(keys.contains(&"BAT_LOCALIZED_OUTPUT".to_string())); assert!(keys.contains(&"BAT_AUTO_DISCOVER".to_string())); } @@ -6118,6 +6172,10 @@ mod tests { assert!(options.banner); assert_eq!(options.output_format, OutputFormat::Human); assert_eq!(config.output_root, PathBuf::from("./bat-resources")); + assert_eq!( + config.localized_output_root, + PathBuf::from("./bat-localized") + ); assert_eq!(options.state_dir, PathBuf::from(DEFAULT_DAEMON_STATE_DIR)); } @@ -6210,6 +6268,8 @@ mod tests { "--daemon", "--output", "/tmp/daemon-output", + "--localized-output", + "/tmp/daemon-localized", "--interval", "30m", ]) @@ -6222,6 +6282,10 @@ mod tests { options.config.output_root, PathBuf::from("/tmp/daemon-output") ); + assert_eq!( + options.config.localized_output_root, + PathBuf::from("/tmp/daemon-localized") + ); assert_eq!(options.state_dir, PathBuf::from(DEFAULT_DAEMON_STATE_DIR)); assert_eq!(options.interval, Duration::from_secs(30 * 60)); assert!(options.quiet_up_to_date); @@ -6246,11 +6310,19 @@ mod tests { assert!(!status.progress); assert!(!status.banner); assert_eq!(status.config.output_root, PathBuf::from("./bat-resources")); + assert_eq!( + status.config.localized_output_root, + PathBuf::from("./bat-localized") + ); assert_eq!(status.state_dir, PathBuf::from(DEFAULT_DAEMON_STATE_DIR)); let stop = parse(&["bat", "stop", "--state-dir", "/tmp/custom-bat-pid"]).unwrap(); assert_eq!(stop.command, CliCommand::Stop); assert_eq!(stop.config.output_root, PathBuf::from("./bat-resources")); + assert_eq!( + stop.config.localized_output_root, + PathBuf::from("./bat-localized") + ); assert_eq!(stop.state_dir, PathBuf::from("/tmp/custom-bat-pid")); } @@ -6312,6 +6384,8 @@ mod tests { "Windows,Android", "--output", "/tmp/daemon-output", + "--localized-output", + "/tmp/daemon-localized", "--state-dir", "/tmp/daemon-state", "--curl", @@ -6336,6 +6410,9 @@ mod tests { assert!(args .windows(2) .any(|pair| pair == ["--output", "/tmp/daemon-output"])); + assert!(args + .windows(2) + .any(|pair| pair == ["--localized-output", "/tmp/daemon-localized"])); assert!(args .windows(2) .any(|pair| pair == ["--state-dir", "/tmp/daemon-state"])); @@ -6579,8 +6656,10 @@ mod tests { let temp = tempfile::TempDir::new().unwrap(); let state_dir = temp.path().join("state"); let output_root = temp.path().join("output"); - let mut base_config = OfficialUpdateConfig::default(); - base_config.output_root = output_root; + let base_config = OfficialUpdateConfig { + output_root, + ..Default::default() + }; let (queue, _rx) = mpsc::channel::(); let context = DaemonTaskContext { registry: TaskRegistry::new(), @@ -6662,11 +6741,13 @@ mod tests { let control = new_daemon_control(); // 保留 rx 让 send 成功（不启动 worker，任务停留在 queued）。 let (queue, rx) = mpsc::channel::(); - let mut base_config = OfficialUpdateConfig::default(); - base_config.force = true; - base_config.dry_run = true; - base_config.audit_local = false; - base_config.repair = false; + let base_config = OfficialUpdateConfig { + force: true, + dry_run: true, + audit_local: false, + repair: false, + ..Default::default() + }; let context = DaemonTaskContext { registry: TaskRegistry::new(), queue, diff --git a/infrastructure/src/import.rs b/infrastructure/src/import.rs index 84d5935..b3e12f4 100644 --- a/infrastructure/src/import.rs +++ b/infrastructure/src/import.rs @@ -87,6 +87,16 @@ pub struct UnityFsImportSummary { pub directory_count: usize, /// UnityFS directory 路径。 pub directories: Vec, + /// UnityFS directory 解出的文件数量。 + pub file_count: usize, + /// 成功解析出的 Unity serialized file 数量。 + pub serialized_file_count: usize, + /// 成功解析出的 TextAsset 数量。 + pub text_asset_count: usize, + /// TextAsset 名称列表。 + pub text_assets: Vec, + /// 非致命 serialized-file 解析诊断数量。 + pub serialized_parse_error_count: usize, } /// Manifest 导入报告。 @@ -266,6 +276,15 @@ impl<'a> ResourceImportService<'a> { .into_iter() .map(|directory| directory.path) .collect(), + file_count: parsed.files.len(), + serialized_file_count: parsed.serialized_files.len(), + text_asset_count: parsed.text_assets.len(), + text_assets: parsed + .text_assets + .into_iter() + .map(|asset| asset.name) + .collect(), + serialized_parse_error_count: parsed.serialized_parse_errors.len(), }) } } @@ -375,6 +394,26 @@ mod tests { data.extend_from_slice(&value.to_be_bytes()); } + fn push_i16_le(data: &mut Vec, value: i16) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u32_le(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i32_le(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_i64_le(data: &mut Vec, value: i64) { + data.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u64_le(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_le_bytes()); + } + fn align(data: &mut Vec, alignment: usize) { let remainder = data.len() % alignment; if remainder != 0 { @@ -414,6 +453,83 @@ mod tests { data } + fn synthetic_text_asset_unityfs_bundle() -> Vec { + let serialized_file = synthetic_serialized_text_asset(); + let mut blocks_info = Vec::new(); + blocks_info.extend_from_slice(&[1; 16]); + push_i32(&mut blocks_info, 1); + push_u32(&mut blocks_info, serialized_file.len() as u32); + push_u32(&mut blocks_info, serialized_file.len() as u32); + push_u16(&mut blocks_info, 0); + push_i32(&mut blocks_info, 1); + push_u64(&mut blocks_info, 0); + push_u64(&mut blocks_info, serialized_file.len() as u64); + push_u32(&mut blocks_info, 0); + push_c_string(&mut blocks_info, "CAB-scenario"); + + let mut data = Vec::new(); + push_c_string(&mut data, "UnityFS"); + push_u32(&mut data, 8); + push_c_string(&mut data, "5.x.x"); + push_c_string(&mut data, "2021.3.56f2"); + push_u64(&mut data, 0); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, 0); + align(&mut data, 16); + data.extend_from_slice(&blocks_info); + data.extend_from_slice(&serialized_file); + + let total_size = data.len() as u64; + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); + data + } + + fn synthetic_serialized_text_asset() -> Vec { + let mut object_data = Vec::new(); + push_u32_le(&mut object_data, 8); + object_data.extend_from_slice(b"Scenario"); + align(&mut object_data, 4); + push_u32_le(&mut object_data, 15); + object_data.extend_from_slice("こんにちは".as_bytes()); + + let mut metadata = Vec::new(); + metadata.extend_from_slice(b"2021.3.56f2\0"); + push_i32_le(&mut metadata, 19); + metadata.push(0); + push_i32_le(&mut metadata, 1); + push_i32_le(&mut metadata, 49); + metadata.push(0); + push_i16_le(&mut metadata, 0); + metadata.extend_from_slice(&[0; 16]); + push_i32_le(&mut metadata, 1); + align(&mut metadata, 4); + push_i64_le(&mut metadata, 1); + push_u64_le(&mut metadata, 0); + push_u32_le(&mut metadata, object_data.len() as u32); + push_i32_le(&mut metadata, 0); + + let header_len = 48usize; + let data_offset = header_len + metadata.len(); + let file_size = data_offset + object_data.len(); + + let mut file = Vec::new(); + push_u32(&mut file, metadata.len() as u32); + push_u32(&mut file, file_size as u32); + push_u32(&mut file, 22); + push_u32(&mut file, 0); + file.push(0); + file.extend_from_slice(&[0, 0, 0]); + push_u32(&mut file, metadata.len() as u32); + push_u64(&mut file, file_size as u64); + push_u64(&mut file, data_offset as u64); + push_u64(&mut file, 0); + file.extend_from_slice(&metadata); + file.extend_from_slice(&object_data); + file + } + fn synthetic_manifest() -> GenericManifest { GenericManifest { format: ManifestFormat::AddressablesCatalog, @@ -594,6 +710,11 @@ mod tests { assert_eq!(unityfs.block_count, 1); assert_eq!(unityfs.directory_count, 1); assert_eq!(unityfs.directories, vec!["SYNTHETIC-CAB".to_string()]); + assert_eq!(unityfs.file_count, 1); + assert_eq!(unityfs.serialized_file_count, 0); + assert_eq!(unityfs.text_asset_count, 0); + assert!(unityfs.text_assets.is_empty()); + assert_eq!(unityfs.serialized_parse_error_count, 0); assert_eq!( report.imported[1].category, ResourceImportCategory::TextAsset @@ -645,6 +766,41 @@ mod tests { ); } + #[tokio::test] + async fn import_summary_reports_text_assets_inside_assetbundle() { + let temp_dir = TempDir::new().unwrap(); + let cas = FileSystemCasRepository::new(temp_dir.path().join("cas")); + let resources = InMemoryResourceRepository::new(); + let service = ResourceImportService::new(&cas, &resources); + let manifest = manifest_with(vec![ResourceEntry { + path: "synthetic/scenario.bundle".to_string(), + hash: "synthetic-scenario-hash".to_string(), + size: 1, + resource_type: ResourceType::AssetBundle, + address: None, + dependencies: Vec::new(), + crc: None, + }]); + + let report = service + .import_manifest_bundles( + &manifest, + &[BundleSource::new( + "scenario.bundle", + synthetic_text_asset_unityfs_bundle(), + )], + ) + .await + .unwrap(); + + let unityfs = report.imported[0].unityfs.as_ref().unwrap(); + assert_eq!(unityfs.file_count, 1); + assert_eq!(unityfs.serialized_file_count, 1); + assert_eq!(unityfs.text_asset_count, 1); + assert_eq!(unityfs.text_assets, vec!["Scenario".to_string()]); + assert_eq!(unityfs.serialized_parse_error_count, 0); + } + #[tokio::test] async fn returns_error_when_bundle_data_is_missing() { let temp_dir = TempDir::new().unwrap(); diff --git a/infrastructure/src/lib.rs b/infrastructure/src/lib.rs index dbf5e57..78c4bda 100644 --- a/infrastructure/src/lib.rs +++ b/infrastructure/src/lib.rs @@ -16,6 +16,7 @@ pub mod import; pub mod official_download; pub mod official_game_main_config; pub mod official_launcher; +pub mod official_parse; pub mod official_pull; pub mod official_sync; pub mod official_update; @@ -45,6 +46,12 @@ pub use official_launcher::{ OfficialLauncherBootstrapService, YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest, }; +pub use official_parse::{ + read_parse_cache_at, write_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, + OfficialParseCacheService, OfficialParseConfig, OfficialParseReport, + OfficialParseSourceFingerprint, OfficialParseSourceKind, OfficialParseStatus, + OfficialParseSummary, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_PARSE_CACHE_VERSION, +}; pub use official_pull::{ build_official_pull_plan, build_official_pull_plan_for_platform_inventory, build_official_pull_plan_for_platforms, build_official_pull_plan_from_platform_inventory, @@ -58,11 +65,11 @@ pub use official_update::{ cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging, read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot, write_version_state, ExtendedSnapshotDelta, GameMainConfigSnapshot, LauncherMetadataSnapshot, - OfficialBootstrapCache, OfficialEndpointMarkerRole, OfficialEndpointMarkerSnapshot, - OfficialFailedVersionRecord, OfficialServerInfoSource, OfficialUpdateConfig, - OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService, OfficialUpdateSnapshot, - OfficialUpdateStatus, OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState, - ResolvedBootstrap, + LocalizedReleaseStatus, OfficialBootstrapCache, OfficialEndpointMarkerRole, + OfficialEndpointMarkerSnapshot, OfficialFailedVersionRecord, OfficialServerInfoSource, + OfficialUpdateConfig, OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService, + OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary, + OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap, }; pub use path_security::{ ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute, diff --git a/infrastructure/src/official_parse.rs b/infrastructure/src/official_parse.rs new file mode 100644 index 0000000..b67bc5e --- /dev/null +++ b/infrastructure/src/official_parse.rs @@ -0,0 +1,950 @@ +//! Official resource post-sync parse cache. +//! +//! The parser runs after official download verification has completed. It reads +//! the immutable published resource tree, writes a small derived cache next to +//! `official-download-manifest.json`, and leaves localized output generation to +//! later patch/export stages. + +use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry}; +use crate::path_security::{ + ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, + STATE_FILE_MODE, +}; +use bat_assetbundle::{Parser, UnityFsParser}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Parse-cache file name stored under a published official resource root. +pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json"; +/// Current parse-cache schema version. +pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1; + +/// Configuration for one official resource parse-cache refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OfficialParseConfig { + /// Published official resource root containing `official-download-manifest.json`. + pub resource_root: PathBuf, + /// `unzip` executable used to inspect zip archives without extracting them. + pub unzip_command: PathBuf, +} + +impl OfficialParseConfig { + /// Creates parse-cache configuration for a published official resource root. + pub fn new(resource_root: impl Into, unzip_command: impl Into) -> Self { + Self { + resource_root: resource_root.into(), + unzip_command: unzip_command.into(), + } + } + + /// Returns the parse-cache path for this resource root. + pub fn cache_path(&self) -> PathBuf { + self.resource_root.join(OFFICIAL_PARSE_CACHE_FILE) + } +} + +/// Structured report returned by a parse-cache refresh. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialParseReport { + /// Official resource root that was inspected. + pub resource_root: PathBuf, + /// Parse-cache path written by the refresh. + pub cache_path: PathBuf, + /// Aggregate parse-cache summary. + pub summary: OfficialParseSummary, +} + +/// Aggregate counters for a parse-cache refresh. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialParseSummary { + /// Number of entries in `official-download-manifest.json`. + pub manifest_entry_count: usize, + /// Number of entries stored in `official-parse-cache.json`. + pub cache_entry_count: usize, + /// Number of direct bundle files or zip inner files inspected as parser candidates. + pub candidate_file_count: usize, + /// Number of zip inner file entries inspected. + pub zip_entry_count: usize, + /// Number of cached entries reused because source URL/path/size/BLAKE3 did not change. + pub skipped_unchanged_count: usize, + /// Number of bundles parsed successfully. + pub parsed_bundle_count: usize, + /// Number of files intentionally skipped because they are not UnityFS bundles. + pub unsupported_count: usize, + /// Number of files or archives that failed parser/cache inspection. + pub failed_count: usize, + /// Total TextAsset objects found in parsed Unity serialized files. + pub text_asset_count: usize, +} + +/// Persistent parse cache for one official resource root. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialParseCache { + /// Cache schema version. + #[serde(default = "default_parse_cache_version")] + pub version: u32, + /// Cache generation time as Unix seconds. + pub generated_unix_seconds: u64, + /// Aggregate summary for this cache. + pub summary: OfficialParseSummary, + /// Cache entries keyed by source URL and optional archive entry. + #[serde(default)] + pub entries: BTreeMap, +} + +/// One parsed or skipped official resource file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialParseCacheEntry { + /// Stable cache key. + pub key: String, + /// Official URL from the download manifest. + pub source_url: String, + /// Relative destination path under the official resource root. + pub destination: String, + /// Inner archive path when the source is a zip file. + pub archive_entry: Option, + /// Source classification used by the parser. + pub source_kind: OfficialParseSourceKind, + /// Fingerprint derived from the download manifest. + pub fingerprint: OfficialParseSourceFingerprint, + /// Parse status for this cache entry. + pub status: OfficialParseStatus, + /// Whether this entry was reused from the previous parse cache in this run. + #[serde(default)] + pub reused_from_previous_cache: bool, + /// Unity editor version when a UnityFS bundle was parsed. + pub unity_version: Option, + /// Number of files extracted from the UnityFS directory table. + pub file_count: usize, + /// Number of Unity serialized files parsed from extracted files. + pub serialized_file_count: usize, + /// Number of TextAsset objects found. + pub text_asset_count: usize, + /// TextAsset names found in stable order. + pub text_assets: Vec, + /// Non-fatal serialized-file parse diagnostic count. + pub serialized_parse_error_count: usize, + /// Human-readable error or skip reason. + pub error: Option, +} + +/// Source kind for a parse-cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OfficialParseSourceKind { + /// A direct official resource file that should be a UnityFS bundle. + DirectBundle, + /// A file entry streamed from an official zip archive. + ZipEntry, + /// A manifest entry that is not expected to contain a UnityFS bundle. + Unsupported, +} + +/// Stable fingerprint copied from the official download manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OfficialParseSourceFingerprint { + /// Official URL from the download manifest. + pub source_url: String, + /// Relative destination path under the official resource root. + pub destination: String, + /// File byte count from the verified download manifest. + pub bytes: u64, + /// BLAKE3 digest from the verified download manifest. + pub blake3: String, +} + +/// Parse status for one official resource cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OfficialParseStatus { + /// UnityFS parsing succeeded. + Parsed, + /// The source was inspected and is not currently parsed as UnityFS. + SkippedUnsupported, + /// The source was expected to be parseable but failed inspection. + Failed, +} + +/// Service that refreshes official parse caches. +#[derive(Debug, Default, Clone, Copy)] +pub struct OfficialParseCacheService; + +impl OfficialParseCacheService { + /// Creates a parse-cache service. + pub fn new() -> Self { + Self + } + + /// Refreshes the parse cache for the configured official resource root. + pub fn run(&self, config: &OfficialParseConfig) -> Result { + let manifest = read_download_manifest_at(&config.resource_root)?.ok_or_else(|| { + format!( + "缺少官方下载 manifest，无法更新解析缓存：{}", + config.resource_root.display() + ) + })?; + let previous_cache = read_parse_cache_at(&config.resource_root)?; + let mut summary = OfficialParseSummary { + manifest_entry_count: manifest.entries.len(), + ..OfficialParseSummary::default() + }; + let mut entries = BTreeMap::new(); + + for manifest_entry in manifest.entries.values() { + let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref()); + for entry in produced { + summary.record_entry(&entry); + entries.insert(entry.key.clone(), entry); + } + } + + summary.cache_entry_count = entries.len(); + let cache = OfficialParseCache { + version: OFFICIAL_PARSE_CACHE_VERSION, + generated_unix_seconds: unix_seconds_now(), + summary: summary.clone(), + entries, + }; + write_parse_cache_at(&config.resource_root, &cache)?; + + Ok(OfficialParseReport { + resource_root: config.resource_root.clone(), + cache_path: config.cache_path(), + summary, + }) + } +} + +impl OfficialParseSummary { + fn record_entry(&mut self, entry: &OfficialParseCacheEntry) { + match entry.source_kind { + OfficialParseSourceKind::DirectBundle => { + self.candidate_file_count += 1; + } + OfficialParseSourceKind::ZipEntry => { + self.candidate_file_count += 1; + self.zip_entry_count += 1; + } + OfficialParseSourceKind::Unsupported => {} + } + match entry.status { + OfficialParseStatus::Parsed => { + self.parsed_bundle_count += 1; + self.text_asset_count += entry.text_asset_count; + } + OfficialParseStatus::SkippedUnsupported => { + self.unsupported_count += 1; + } + OfficialParseStatus::Failed => { + self.failed_count += 1; + } + } + if entry.reused_from_previous_cache { + self.skipped_unchanged_count += 1; + } + } +} + +/// Reads the parse cache under a published official resource root. +/// +/// Missing, corrupt, or unsupported-version caches return `Ok(None)` so a new +/// cache can be regenerated without blocking official resource publication. +pub fn read_parse_cache_at(resource_root: &Path) -> Result, String> { + let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE); + let Some(bytes) = read_file_no_symlink(&path, "官方解析缓存")? else { + return Ok(None); + }; + let Ok(cache) = serde_json::from_slice::(&bytes) else { + return Ok(None); + }; + if cache.version != OFFICIAL_PARSE_CACHE_VERSION { + return Ok(None); + } + Ok(Some(cache)) +} + +/// Writes the parse cache under a published official resource root. +pub fn write_parse_cache_at( + resource_root: &Path, + cache: &OfficialParseCache, +) -> Result<(), String> { + let path = resource_root.join(OFFICIAL_PARSE_CACHE_FILE); + ensure_path_within_root(resource_root, &path)?; + ensure_safe_file_target(resource_root, &path, "官方解析缓存")?; + let bytes = serde_json::to_vec_pretty(cache) + .map_err(|error| format!("序列化官方解析缓存失败：{error}"))?; + write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存") +} + +fn process_manifest_entry( + config: &OfficialParseConfig, + manifest_entry: &OfficialDownloadManifestEntry, + previous_cache: Option<&OfficialParseCache>, +) -> Vec { + let fingerprint = fingerprint_for(manifest_entry); + if looks_like_zip_source(manifest_entry) { + return process_zip_entry(config, manifest_entry, previous_cache, fingerprint); + } + + if looks_like_direct_bundle_source(manifest_entry) { + let key = direct_key(&manifest_entry.url); + if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) { + cached.reused_from_previous_cache = true; + return vec![cached]; + } + return vec![parse_direct_bundle( + config, + manifest_entry, + key, + fingerprint, + )]; + } + + let key = unsupported_key(&manifest_entry.url); + if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) { + cached.reused_from_previous_cache = true; + return vec![cached]; + } + vec![unsupported_entry( + manifest_entry, + None, + OfficialParseSourceKind::Unsupported, + fingerprint, + key, + "非 UnityFS 候选资源", + )] +} + +fn process_zip_entry( + config: &OfficialParseConfig, + manifest_entry: &OfficialDownloadManifestEntry, + previous_cache: Option<&OfficialParseCache>, + fingerprint: OfficialParseSourceFingerprint, +) -> Vec { + let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint); + if !cached_entries.is_empty() { + return cached_entries + .into_iter() + .map(|mut entry| { + entry.reused_from_previous_cache = true; + entry + }) + .collect(); + } + + let archive_path = match resource_path_for(&config.resource_root, manifest_entry) { + Ok(path) => path, + Err(error) => { + return vec![failed_entry( + manifest_entry, + None, + OfficialParseSourceKind::ZipEntry, + fingerprint, + zip_list_key(&manifest_entry.url), + error, + )] + } + }; + let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) { + Ok(entries) => entries, + Err(error) => { + return vec![failed_entry( + manifest_entry, + None, + OfficialParseSourceKind::ZipEntry, + fingerprint, + zip_list_key(&manifest_entry.url), + error, + )] + } + }; + + let mut produced = Vec::new(); + for archive_entry in archive_entries { + let key = zip_key(&manifest_entry.url, &archive_entry); + let bytes = match read_zip_entry_bytes(&config.unzip_command, &archive_path, &archive_entry) + { + Ok(bytes) => bytes, + Err(error) => { + produced.push(failed_entry( + manifest_entry, + Some(archive_entry), + OfficialParseSourceKind::ZipEntry, + fingerprint.clone(), + key, + error, + )); + continue; + } + }; + produced.push(parse_zip_inner_file( + manifest_entry, + archive_entry, + fingerprint.clone(), + key, + &bytes, + )); + } + + if produced.is_empty() { + produced.push(unsupported_entry( + manifest_entry, + None, + OfficialParseSourceKind::Unsupported, + fingerprint, + zip_list_key(&manifest_entry.url), + "ZIP 内没有可检查文件条目", + )); + } + produced +} + +fn parse_direct_bundle( + config: &OfficialParseConfig, + manifest_entry: &OfficialDownloadManifestEntry, + key: String, + fingerprint: OfficialParseSourceFingerprint, +) -> OfficialParseCacheEntry { + let path = match resource_path_for(&config.resource_root, manifest_entry) { + Ok(path) => path, + Err(error) => { + return failed_entry( + manifest_entry, + None, + OfficialParseSourceKind::DirectBundle, + fingerprint, + key, + error, + ) + } + }; + let bytes = match read_resource_file(&path) { + Ok(bytes) => bytes, + Err(error) => { + return failed_entry( + manifest_entry, + None, + OfficialParseSourceKind::DirectBundle, + fingerprint, + key, + error, + ) + } + }; + if !UnityFsParser::has_unityfs_signature(&bytes) { + return unsupported_entry( + manifest_entry, + None, + OfficialParseSourceKind::DirectBundle, + fingerprint, + key, + "文件不是 UnityFS bundle", + ); + } + parsed_bundle_entry( + manifest_entry, + None, + OfficialParseSourceKind::DirectBundle, + fingerprint, + key, + &bytes, + ) +} + +fn parse_zip_inner_file( + manifest_entry: &OfficialDownloadManifestEntry, + archive_entry: String, + fingerprint: OfficialParseSourceFingerprint, + key: String, + bytes: &[u8], +) -> OfficialParseCacheEntry { + if !UnityFsParser::has_unityfs_signature(bytes) { + return unsupported_entry( + manifest_entry, + Some(archive_entry), + OfficialParseSourceKind::ZipEntry, + fingerprint, + key, + "ZIP 条目不是 UnityFS bundle", + ); + } + parsed_bundle_entry( + manifest_entry, + Some(archive_entry), + OfficialParseSourceKind::ZipEntry, + fingerprint, + key, + bytes, + ) +} + +fn parsed_bundle_entry( + manifest_entry: &OfficialDownloadManifestEntry, + archive_entry: Option, + source_kind: OfficialParseSourceKind, + fingerprint: OfficialParseSourceFingerprint, + key: String, + bytes: &[u8], +) -> OfficialParseCacheEntry { + let parser = UnityFsParser::new(); + match parser.parse(bytes) { + Ok(parsed) => OfficialParseCacheEntry { + key, + source_url: manifest_entry.url.clone(), + destination: manifest_entry.destination.clone(), + archive_entry, + source_kind, + fingerprint, + status: OfficialParseStatus::Parsed, + reused_from_previous_cache: false, + unity_version: Some(parsed.unity_version), + file_count: parsed.files.len(), + serialized_file_count: parsed.serialized_files.len(), + text_asset_count: parsed.text_assets.len(), + text_assets: parsed + .text_assets + .iter() + .map(|asset| asset.name.clone()) + .collect(), + serialized_parse_error_count: parsed.serialized_parse_errors.len(), + error: None, + }, + Err(error) => failed_entry( + manifest_entry, + archive_entry, + source_kind, + fingerprint, + key, + error.to_string(), + ), + } +} + +fn failed_entry( + manifest_entry: &OfficialDownloadManifestEntry, + archive_entry: Option, + source_kind: OfficialParseSourceKind, + fingerprint: OfficialParseSourceFingerprint, + key: String, + error: impl Into, +) -> OfficialParseCacheEntry { + status_entry( + manifest_entry, + archive_entry, + source_kind, + fingerprint, + key, + OfficialParseStatus::Failed, + error.into(), + ) +} + +fn unsupported_entry( + manifest_entry: &OfficialDownloadManifestEntry, + archive_entry: Option, + source_kind: OfficialParseSourceKind, + fingerprint: OfficialParseSourceFingerprint, + key: String, + reason: impl Into, +) -> OfficialParseCacheEntry { + status_entry( + manifest_entry, + archive_entry, + source_kind, + fingerprint, + key, + OfficialParseStatus::SkippedUnsupported, + reason.into(), + ) +} + +fn status_entry( + manifest_entry: &OfficialDownloadManifestEntry, + archive_entry: Option, + source_kind: OfficialParseSourceKind, + fingerprint: OfficialParseSourceFingerprint, + key: String, + status: OfficialParseStatus, + reason: String, +) -> OfficialParseCacheEntry { + OfficialParseCacheEntry { + key, + source_url: manifest_entry.url.clone(), + destination: manifest_entry.destination.clone(), + archive_entry, + source_kind, + fingerprint, + status, + reused_from_previous_cache: false, + unity_version: None, + file_count: 0, + serialized_file_count: 0, + text_asset_count: 0, + text_assets: Vec::new(), + serialized_parse_error_count: 0, + error: Some(reason), + } +} + +fn read_resource_file(path: &Path) -> Result, String> { + read_file_no_symlink(path, "官方解析输入")? + .ok_or_else(|| format!("官方解析输入不存在：{}", path.display())) +} + +fn resource_path_for( + resource_root: &Path, + manifest_entry: &OfficialDownloadManifestEntry, +) -> Result { + let path = resource_root.join(Path::new(&manifest_entry.destination)); + ensure_path_within_root(resource_root, &path)?; + ensure_safe_file_target(resource_root, &path, "官方解析输入")?; + Ok(path) +} + +fn list_zip_entries(unzip_command: &Path, archive_path: &Path) -> Result, String> { + let output = Command::new(unzip_command) + .arg("-Z1") + .arg(archive_path) + .output() + .map_err(|error| { + format!( + "启动 unzip 列出 ZIP 条目失败 {}：{error}", + archive_path.display() + ) + })?; + if !output.status.success() { + return Err(format!( + "列出 ZIP 条目失败 {}：{}", + archive_path.display(), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| line.trim_end_matches('\r').trim().to_string()) + .filter(|line| !line.is_empty() && !line.ends_with('/')) + .collect()) +} + +fn read_zip_entry_bytes( + unzip_command: &Path, + archive_path: &Path, + archive_entry: &str, +) -> Result, String> { + let output = Command::new(unzip_command) + .arg("-p") + .arg(archive_path) + .arg(archive_entry) + .output() + .map_err(|error| { + format!( + "启动 unzip 读取 ZIP 条目失败 {}!{}：{error}", + archive_path.display(), + archive_entry + ) + })?; + if !output.status.success() { + return Err(format!( + "读取 ZIP 条目失败 {}!{}：{}", + archive_path.display(), + archive_entry, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(output.stdout) +} + +fn reusable_entry( + previous_cache: Option<&OfficialParseCache>, + key: &str, + fingerprint: &OfficialParseSourceFingerprint, +) -> Option { + previous_cache + .and_then(|cache| cache.entries.get(key)) + .filter(|entry| &entry.fingerprint == fingerprint) + .cloned() +} + +fn reusable_archive_entries( + previous_cache: Option<&OfficialParseCache>, + manifest_entry: &OfficialDownloadManifestEntry, + fingerprint: &OfficialParseSourceFingerprint, +) -> Vec { + previous_cache + .into_iter() + .flat_map(|cache| cache.entries.values()) + .filter(|entry| { + entry.source_url == manifest_entry.url + && entry.destination == manifest_entry.destination + && &entry.fingerprint == fingerprint + }) + .cloned() + .collect() +} + +fn fingerprint_for( + manifest_entry: &OfficialDownloadManifestEntry, +) -> OfficialParseSourceFingerprint { + OfficialParseSourceFingerprint { + source_url: manifest_entry.url.clone(), + destination: manifest_entry.destination.clone(), + bytes: manifest_entry.bytes, + blake3: manifest_entry.blake3.clone(), + } +} + +fn looks_like_zip_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool { + has_case_insensitive_suffix(&manifest_entry.url, ".zip") + || has_case_insensitive_suffix(&manifest_entry.destination, ".zip") +} + +fn looks_like_direct_bundle_source(manifest_entry: &OfficialDownloadManifestEntry) -> bool { + [".bundle", ".unity3d"].iter().any(|suffix| { + has_case_insensitive_suffix(&manifest_entry.url, suffix) + || has_case_insensitive_suffix(&manifest_entry.destination, suffix) + }) +} + +fn has_case_insensitive_suffix(value: &str, suffix: &str) -> bool { + value + .rsplit(['/', '\\']) + .next() + .is_some_and(|name| name.to_ascii_lowercase().ends_with(suffix)) +} + +fn direct_key(url: &str) -> String { + format!("direct:{url}") +} + +fn zip_key(url: &str, archive_entry: &str) -> String { + format!("zip:{url}!{archive_entry}") +} + +fn zip_list_key(url: &str) -> String { + format!("zip-list:{url}") +} + +fn unsupported_key(url: &str) -> String { + format!("unsupported:{url}") +} + +fn default_parse_cache_version() -> u32 { + OFFICIAL_PARSE_CACHE_VERSION +} + +fn unix_seconds_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + fn push_c_string(data: &mut Vec, value: &str) { + data.extend_from_slice(value.as_bytes()); + data.push(0); + } + + fn push_u16(data: &mut Vec, value: u16) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u32(data: &mut Vec, value: u32) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_i32(data: &mut Vec, value: i32) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u64(data: &mut Vec, value: u64) { + data.extend_from_slice(&value.to_be_bytes()); + } + + fn align(data: &mut Vec, alignment: usize) { + let remainder = data.len() % alignment; + if remainder != 0 { + data.resize(data.len() + alignment - remainder, 0); + } + } + + fn synthetic_unityfs_bundle(directory_path: &str, payload: &[u8]) -> Vec { + let mut blocks_info = Vec::new(); + blocks_info.extend_from_slice(&[0xAB; 16]); + push_i32(&mut blocks_info, 1); + push_u32(&mut blocks_info, payload.len() as u32); + push_u32(&mut blocks_info, payload.len() as u32); + push_u16(&mut blocks_info, 0); + push_i32(&mut blocks_info, 1); + push_u64(&mut blocks_info, 0); + push_u64(&mut blocks_info, payload.len() as u64); + push_u32(&mut blocks_info, 0); + push_c_string(&mut blocks_info, directory_path); + + let mut data = Vec::new(); + push_c_string(&mut data, "UnityFS"); + push_u32(&mut data, 8); + push_c_string(&mut data, "5.x.x"); + push_c_string(&mut data, "2021.3.56f2"); + push_u64(&mut data, 0); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, blocks_info.len() as u32); + push_u32(&mut data, 0); + align(&mut data, 16); + data.extend_from_slice(&blocks_info); + data.extend_from_slice(payload); + + let total_size = data.len() as u64; + let total_size_offset = b"UnityFS\0".len() + 4 + b"5.x.x\0".len() + b"2021.3.56f2\0".len(); + data[total_size_offset..total_size_offset + 8].copy_from_slice(&total_size.to_be_bytes()); + data + } + + fn write_download_manifest( + root: &Path, + entries: &[(&str, &str, &[u8])], + ) -> Vec { + let mut manifest_entries = BTreeMap::new(); + let mut written = Vec::new(); + for (url, destination, bytes) in entries { + let path = root.join(destination); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, bytes).unwrap(); + let entry = OfficialDownloadManifestEntry { + url: (*url).to_string(), + destination: (*destination).to_string(), + bytes: bytes.len() as u64, + blake3: blake3::hash(bytes).to_hex().to_string(), + }; + manifest_entries.insert((*url).to_string(), entry.clone()); + written.push(entry); + } + let manifest = serde_json::json!({ + "version": 1, + "entries": manifest_entries, + }); + fs::write( + root.join("official-download-manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + written + } + + #[test] + fn parses_direct_bundle_and_reuses_unchanged_cache() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path(); + let bundle = synthetic_unityfs_bundle("CAB-fixture", b"data"); + write_download_manifest( + root, + &[( + "https://prod-clientpatch.bluearchiveyostar.com/r93/Bundle/test.bundle", + "Bundle/test.bundle", + &bundle, + )], + ); + let config = OfficialParseConfig::new(root, "unzip"); + + let first = OfficialParseCacheService::new().run(&config).unwrap(); + assert_eq!(first.summary.manifest_entry_count, 1); + assert_eq!(first.summary.parsed_bundle_count, 1); + assert_eq!(first.summary.skipped_unchanged_count, 0); + assert!(first.cache_path.exists()); + + let second = OfficialParseCacheService::new().run(&config).unwrap(); + assert_eq!(second.summary.parsed_bundle_count, 1); + assert_eq!(second.summary.skipped_unchanged_count, 1); + let cache = read_parse_cache_at(root).unwrap().unwrap(); + let entry = cache.entries.values().next().unwrap(); + assert_eq!(entry.status, OfficialParseStatus::Parsed); + assert_eq!(entry.file_count, 1); + } + + #[test] + fn records_non_candidate_resources_as_unsupported() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path(); + write_download_manifest( + root, + &[( + "https://prod-clientpatch.bluearchiveyostar.com/r93/TableBundles/TableCatalog.bytes", + "TableBundles/TableCatalog.bytes", + b"catalog", + )], + ); + let config = OfficialParseConfig::new(root, "unzip"); + + let report = OfficialParseCacheService::new().run(&config).unwrap(); + + assert_eq!(report.summary.parsed_bundle_count, 0); + assert_eq!(report.summary.unsupported_count, 1); + assert_eq!(report.summary.failed_count, 0); + } + + #[cfg(unix)] + #[test] + fn parses_zip_entries_without_extracting_archive_tree() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path(); + let bundle = synthetic_unityfs_bundle("CAB-zipped", b"zipdata"); + let bundle_path = temp.path().join("inner.bundle"); + fs::write(&bundle_path, &bundle).unwrap(); + let archive_bytes = b"zip-placeholder"; + write_download_manifest( + root, + &[( + "https://prod-clientpatch.bluearchiveyostar.com/r93/Windows_PatchPack/FullPatch_000.zip", + "Windows_PatchPack/FullPatch_000.zip", + archive_bytes, + )], + ); + let unzip_script = temp.path().join("fake-unzip"); + fs::write( + &unzip_script, + format!( + r#"#!/usr/bin/env bash +set -euo pipefail +if [[ "${{1:-}}" == "-Z1" ]]; then + printf '%s\n' 'assets/scenario.bundle' 'assets/readme.txt' + exit 0 +fi +if [[ "${{1:-}}" == "-p" && "${{3:-}}" == "assets/scenario.bundle" ]]; then + cat '{}' + exit 0 +fi +if [[ "${{1:-}}" == "-p" ]]; then + printf 'plain text' + exit 0 +fi +exit 2 +"#, + bundle_path.display() + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&unzip_script).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&unzip_script, permissions).unwrap(); + let config = OfficialParseConfig::new(root, unzip_script); + + let report = OfficialParseCacheService::new().run(&config).unwrap(); + + assert_eq!(report.summary.manifest_entry_count, 1); + assert_eq!(report.summary.zip_entry_count, 2); + assert_eq!(report.summary.parsed_bundle_count, 1); + assert_eq!(report.summary.unsupported_count, 1); + assert_eq!(report.summary.failed_count, 0); + assert!(!root.join("assets").exists()); + } +} diff --git a/infrastructure/src/official_update.rs b/infrastructure/src/official_update.rs index e8b6c8e..8116250 100644 --- a/infrastructure/src/official_update.rs +++ b/infrastructure/src/official_update.rs @@ -7,7 +7,7 @@ use crate::curl_transfer::{resolve_curl_proxy, CurlProxyConfig}; use crate::path_security::{ - ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, + ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, lexical_absolute, read_file_no_symlink, validate_output_root, write_file_atomic, STATE_FILE_MODE, }; use crate::{ @@ -18,6 +18,7 @@ use crate::{ OfficialResourcePullService, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest, }; +use crate::{OfficialParseCacheService, OfficialParseConfig, OfficialParseSummary}; use bat_adapters::official::game_main_config::YostarJpGameMainConfig; use bat_adapters::official::inventory::{ YostarJpPlatformCatalogInventory, YostarJpPlatformDownloadInventory, @@ -78,6 +79,8 @@ pub struct OfficialUpdateConfig { pub platforms: Option>, /// Output root for resources and state files. pub output_root: PathBuf, + /// Output root reserved for localized resources generated from official data. + pub localized_output_root: PathBuf, /// Optional explicit sync snapshot path. pub snapshot_path: Option, /// Curl command used by the current infrastructure downloader. @@ -108,6 +111,7 @@ impl Default for OfficialUpdateConfig { auto_discover: false, platforms: None, output_root: PathBuf::from("./bat-resources"), + localized_output_root: PathBuf::from("./bat-localized"), snapshot_path: None, curl_command: PathBuf::from("curl"), curl_proxy: CurlProxyConfig::default(), @@ -168,6 +172,26 @@ impl OfficialUpdateStatus { } } +/// Publication state for localized resources associated with an official release. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LocalizedReleaseStatus { + /// Official resources are published, but localized resources are not. + NotLocalized, + /// Official resources and localized resources are both published. + Localized, +} + +impl LocalizedReleaseStatus { + /// Returns a stable string label for CLI and JSON callers. + pub fn as_str(self) -> &'static str { + match self { + Self::NotLocalized => "not_localized", + Self::Localized => "localized", + } + } +} + /// Snapshot of the official JP update state observed at a point in time. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialUpdateSnapshot { @@ -477,6 +501,14 @@ pub struct OfficialUpdateReport { pub platforms: Vec, /// Managed publish root containing `current`, `versions`, and staging. pub output_root: PathBuf, + /// Separate root reserved for localized resources. + pub localized_output_root: PathBuf, + /// Localized publication state for this official resource version. + pub localized_release_status: LocalizedReleaseStatus, + /// Future localized `current` pointer path under `localized_output_root`. + pub localized_current_path: PathBuf, + /// Versioned localized release directory when localized resources are published. + pub localized_published_version_path: Option, /// Active resource root used for local audit before this run. pub active_resource_root: PathBuf, /// Atomic `current` pointer path. @@ -543,6 +575,10 @@ pub struct OfficialUpdateReport { pub verification_summary: OfficialVerificationSummary, /// Download manifest path. pub download_manifest: PathBuf, + /// Parse-cache path written or refreshed after successful verification. + pub parse_cache_path: Option, + /// Post-sync parse-cache summary. + pub parse_summary: Option, /// Snapshot path written after success. pub snapshot_written: Option, } @@ -1175,6 +1211,10 @@ impl OfficialUpdateService { addressables_root: current_snapshot.addressables_root.clone(), platforms: platforms.to_vec(), output_root: config.output_root.clone(), + localized_output_root: config.localized_output_root.clone(), + localized_release_status: LocalizedReleaseStatus::NotLocalized, + localized_current_path: config.localized_output_root.join(OFFICIAL_CURRENT_LINK), + localized_published_version_path: None, active_resource_root: active_resource_root.clone(), current_path: publish_layout.current_path.clone(), version_state_path: version_state_path.clone(), @@ -1214,6 +1254,8 @@ impl OfficialUpdateService { local_zip_structure_verified_count, ), download_manifest: fetcher.download_manifest_path(), + parse_cache_path: None, + parse_summary: None, snapshot_written: None, }; @@ -1242,6 +1284,13 @@ impl OfficialUpdateService { &active_resource_root, &snapshot_path, )?; + run_post_sync_parse_cache( + config, + &active_resource_root, + &mut report, + &mut progress, + &mut should_cancel, + )?; } progress(OfficialUpdateProgress::new( "finish", @@ -1454,6 +1503,13 @@ impl OfficialUpdateService { format!("资源已发布，但写入版本状态失败（下轮可重试）：{error}"), )); } + run_post_sync_parse_cache( + config, + &published_version_path, + &mut report, + &mut progress, + &mut should_cancel, + )?; // 清理未被最新版本状态引用的孤儿 staging 目录（GC 失败仅告警，不影响发布结果）。 match read_version_state(&version_state_path) { @@ -1487,6 +1543,46 @@ impl OfficialUpdateService { } } +fn run_post_sync_parse_cache( + config: &OfficialUpdateConfig, + resource_root: &Path, + report: &mut OfficialUpdateReport, + progress: &mut dyn FnMut(OfficialUpdateProgress), + should_cancel: &mut dyn FnMut() -> bool, +) -> anyhow::Result<()> { + check_shutdown_requested(should_cancel)?; + progress(OfficialUpdateProgress::new( + "parse", + format!("更新官方资源解析缓存 {}", resource_root.display()), + )); + let parse_config = OfficialParseConfig::new(resource_root, &config.unzip_command); + match OfficialParseCacheService::new().run(&parse_config) { + Ok(parse_report) => { + progress(OfficialUpdateProgress::new( + "parse", + format!( + "解析缓存完成：条目={} 已解析={} 复用={} 不支持={} 失败={} TextAsset={}", + parse_report.summary.cache_entry_count, + parse_report.summary.parsed_bundle_count, + parse_report.summary.skipped_unchanged_count, + parse_report.summary.unsupported_count, + parse_report.summary.failed_count, + parse_report.summary.text_asset_count + ), + )); + report.parse_cache_path = Some(parse_report.cache_path); + report.parse_summary = Some(parse_report.summary); + } + Err(error) => { + progress(OfficialUpdateProgress::new( + "parse", + format!("解析缓存更新失败（不影响已校验官方资源）：{error}"), + )); + } + } + Ok(()) +} + fn build_pull_plan( server_info: &YostarJpServerInfo, connection_group: &str, @@ -1751,7 +1847,10 @@ pub fn diff_extended_snapshot( fn validate_update_paths(config: &OfficialUpdateConfig) -> Result<(), String> { validate_output_root(&config.output_root)?; + validate_output_root(&config.localized_output_root)?; + validate_separate_output_roots(&config.output_root, &config.localized_output_root)?; ensure_safe_directory_path(&config.output_root, "资源输出目录")?; + ensure_safe_directory_path(&config.localized_output_root, "汉化输出目录")?; if let Some(snapshot_path) = config.snapshot_path.as_ref() { ensure_path_within_root(&config.output_root, snapshot_path)?; ensure_safe_file_target(&config.output_root, snapshot_path, "官方更新快照")?; @@ -1770,6 +1869,28 @@ fn validate_update_paths(config: &OfficialUpdateConfig) -> Result<(), String> { Ok(()) } +fn validate_separate_output_roots( + output_root: &Path, + localized_output_root: &Path, +) -> Result<(), String> { + let official = lexical_absolute(output_root)?; + let localized = lexical_absolute(localized_output_root)?; + if official == localized { + return Err(format!( + "官方资源目录和汉化输出目录不能相同：{}", + official.display() + )); + } + if localized.starts_with(&official) || official.starts_with(&localized) { + return Err(format!( + "官方资源目录和汉化输出目录不能互相嵌套：官方={} 汉化={}", + official.display(), + localized.display() + )); + } + Ok(()) +} + fn snapshot_path_for(config: &OfficialUpdateConfig, resource_root: &Path) -> PathBuf { config .snapshot_path @@ -3412,6 +3533,35 @@ mod tests { assert!(error.to_string().contains("危险路径")); } + #[test] + fn update_rejects_same_official_and_localized_output_root() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path().join("resources"); + let config = OfficialUpdateConfig { + output_root: root.clone(), + localized_output_root: root, + dry_run: true, + ..OfficialUpdateConfig::default() + }; + + let error = OfficialUpdateService::new().run(&config).unwrap_err(); + assert!(error.to_string().contains("不能相同")); + } + + #[test] + fn update_rejects_nested_official_and_localized_output_roots() { + let temp = tempfile::TempDir::new().unwrap(); + let config = OfficialUpdateConfig { + output_root: temp.path().join("resources"), + localized_output_root: temp.path().join("resources/localized"), + dry_run: true, + ..OfficialUpdateConfig::default() + }; + + let error = OfficialUpdateService::new().run(&config).unwrap_err(); + assert!(error.to_string().contains("不能互相嵌套")); + } + #[test] fn update_rejects_snapshot_path_escape() { let temp = tempfile::TempDir::new().unwrap();