feat(resource): 增加历史 release 与 CAS 复用
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s

Closes #47
This commit is contained in:
2026-09-02 01:26:53 +08:00
parent fdd4075e7e
commit bd1e1a06f2
14 changed files with 1235 additions and 59 deletions
+18 -4
View File
@@ -9,8 +9,8 @@ use bat_infrastructure::{
apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file,
apply_unityfs_text_asset_patch_file, changed_endpoint_urls,
completed_worker_translation_workbench, diff_extended_snapshot, export_translation_workbench,
gc_orphan_staging, get_translation_entry, lexical_absolute, localized_patch_operations,
open_append_file, read_download_manifest_at, read_file_no_symlink,
gc_orphan_staging_with_cas_root, get_translation_entry, lexical_absolute,
localized_patch_operations, open_append_file, read_download_manifest_at, read_file_no_symlink,
read_localized_patch_manifest_at, read_localized_version_state, read_parse_cache_at,
read_snapshot, read_textunit_index_at, read_translation_workbench, read_version_state,
redact_proxy_url, repack_bundle, resolve_curl_proxy, set_translation, unset_translation,
@@ -5103,8 +5103,18 @@ impl HumanReport for OfficialUpdateReport {
print_optional_field("资源数", self.resource_count);
print_field("已下载", self.downloaded_count);
print_field("已续传", self.resumed_count);
print_field("已跳过", self.skipped_count);
print_field("当前 manifest 复用", self.skipped_count);
print_field("历史 release 复用", self.release_reused_count);
print_field("CAS 复用", self.cas_reused_count);
print_field("复用量", format_bytes(self.reused_bytes));
print_field("传输量", format_bytes(self.transferred_bytes));
print_field("复用诊断", self.reuse_warnings.len());
for warning in &self.reuse_warnings {
println!(
" - 复用回退 [{}] {} {}",
warning.source, warning.url, warning.message
);
}
print_field("最终大小", format_bytes(self.final_bytes));
print_field("本地校验通过", self.local_manifest_verified_count);
print_field("需修复", self.local_manifest_repair_needed_count);
@@ -6172,7 +6182,11 @@ fn run_clean_stable_command(options: &CliOptions) -> anyhow::Result<()> {
// 后台已停止,清理未被版本状态引用的孤儿 staging 目录。
if let Some(state) = read_version_state(&options.config.version_state_path())? {
removed_paths.extend(gc_orphan_staging(&options.config.output_root, &state)?);
removed_paths.extend(gc_orphan_staging_with_cas_root(
&options.config.output_root,
&state,
&options.config.effective_import_cas_root(),
)?);
}
let report = CleanStableReport {
+12 -9
View File
@@ -70,13 +70,15 @@ pub use official_changes::{
OFFICIAL_RESOURCE_CHANGES_VERSION,
};
pub use official_download::{
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
read_cas_reuse_reference_manifest_at, read_download_manifest_at, release_cas_reuse_references,
DownloadError, OfficialCasReuseReferenceManifest, OfficialDownloadManifest,
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
OfficialLocalManifestAuditReport, OfficialLocalManifestAuditStatus,
OfficialLocalVerificationReport, OfficialResourceHashAlgorithm,
OfficialResourceHashVerification, OfficialResourcePullItem, OfficialResourcePullProgress,
OfficialResourcePullProgressKind, OfficialResourcePullReport, OfficialResourcePullService,
OfficialResourcePullStatus, OfficialResourceVerification,
OfficialResourcePullStatus, OfficialResourceReuseWarning, OfficialResourceVerification,
OFFICIAL_CAS_REUSE_REFERENCES_FILE,
};
pub use official_game_main_config::OfficialGameMainConfigBootstrapService;
pub use official_launcher::{
@@ -116,13 +118,14 @@ pub use official_textunit_queue::{
};
pub use official_update::{
cached_game_main_config_for_metadata, diff_extended_snapshot, gc_orphan_staging,
read_bootstrap_cache, read_snapshot, read_version_state, write_bootstrap_cache, write_snapshot,
write_version_state, ExtendedSnapshotDelta, GameMainConfigSnapshot, LauncherMetadataSnapshot,
LocalizedReleaseStatus, OfficialBootstrapCache, OfficialEndpointMarkerRole,
OfficialEndpointMarkerSnapshot, OfficialFailedVersionRecord, OfficialServerInfoSource,
OfficialUpdateConfig, OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService,
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
OfficialVersionRecord, OfficialVersionState, ResolvedBootstrap,
gc_orphan_staging_with_cas_root, read_bootstrap_cache, read_snapshot, read_version_state,
write_bootstrap_cache, write_snapshot, write_version_state, ExtendedSnapshotDelta,
GameMainConfigSnapshot, LauncherMetadataSnapshot, LocalizedReleaseStatus,
OfficialBootstrapCache, OfficialEndpointMarkerRole, OfficialEndpointMarkerSnapshot,
OfficialFailedVersionRecord, OfficialServerInfoSource, OfficialUpdateConfig,
OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService, OfficialUpdateSnapshot,
OfficialUpdateStatus, OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState,
ResolvedBootstrap,
};
pub use patch_ops::{
apply_patch_file, apply_unityfs_field_patch_file, apply_unityfs_string_field_patch_file,
File diff suppressed because it is too large Load Diff
+95 -7
View File
@@ -34,12 +34,12 @@ use crate::translation_tasks::{
};
use crate::{
build_official_pull_plan_for_platform_inventory, build_official_sync_plan,
changed_endpoint_urls, default_official_platforms, DownloadError,
changed_endpoint_urls, default_official_platforms, release_cas_reuse_references, DownloadError,
OfficialGameMainConfigBootstrapService, OfficialLauncherBootstrapService,
OfficialResourceHashVerification, OfficialResourcePullPlan, OfficialResourcePullProgress,
OfficialResourcePullProgressKind, OfficialResourcePullService, OfficialResourceVerification,
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl,
YostarJpLauncherRemoteManifest,
OfficialResourcePullProgressKind, OfficialResourcePullService, OfficialResourceReuseWarning,
OfficialResourceVerification, YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig,
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
};
use crate::{
read_parse_cache_at, OfficialParseCacheService, OfficialParseConfig, OfficialParseSummary,
@@ -827,6 +827,18 @@ pub struct OfficialUpdateReport {
pub resumed_count: usize,
/// Number of resources skipped using the local manifest.
pub skipped_count: usize,
/// Number of resources reused from immutable historical releases.
#[serde(default)]
pub release_reused_count: usize,
/// Number of resources reused from CAS.
#[serde(default)]
pub cas_reused_count: usize,
/// Bytes materialized without network transfer.
#[serde(default)]
pub reused_bytes: u64,
/// Diagnostics recorded while invalid reuse candidates fell back.
#[serde(default)]
pub reuse_warnings: Vec<OfficialResourceReuseWarning>,
/// Final local byte count in the pull report.
pub final_bytes: u64,
/// Bytes transferred in this run.
@@ -1120,6 +1132,36 @@ impl OfficialPublishLayout {
path_exists_no_follow(&self.root.join(OFFICIAL_DOWNLOAD_MANIFEST_FILE))
}
fn release_reuse_roots(&self, active_root: &Path) -> Result<Vec<PathBuf>, String> {
let mut roots = Vec::new();
if path_exists_no_follow(active_root)? {
roots.push(active_root.to_path_buf());
}
if !path_exists_no_follow(&self.versions_dir)? {
return Ok(roots);
}
ensure_safe_directory_path(&self.versions_dir, "官方资源历史 release 根目录")?;
let mut versions = fs::read_dir(&self.versions_dir)
.map_err(|error| {
format!(
"读取官方资源历史 release 根目录失败 {}{error}",
self.versions_dir.display()
)
})?
.filter_map(|entry| entry.ok().map(|entry| entry.path()))
.filter(|path| {
fs::symlink_metadata(path)
.map(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink())
.unwrap_or(false)
})
.collect::<Vec<_>>();
versions.sort();
versions.reverse();
roots.extend(versions);
roots.dedup();
Ok(roots)
}
fn publish(&self, plan: &OfficialPublishPlan) -> Result<PathBuf, String> {
ensure_safe_directory_path(&self.versions_dir, "官方资源 versions 根目录")?;
fs::create_dir_all(&self.versions_dir).map_err(|error| {
@@ -1663,6 +1705,10 @@ impl OfficialUpdateService {
downloaded_count: 0,
resumed_count: 0,
skipped_count: 0,
release_reused_count: 0,
cas_reused_count: 0,
reused_bytes: 0,
reuse_warnings: Vec::new(),
final_bytes: 0,
transferred_bytes: 0,
official_seed_hash_verified_count: 0,
@@ -1861,6 +1907,12 @@ impl OfficialUpdateService {
&config.curl_command,
)
.with_proxy_config(config.curl_proxy.clone())
.with_release_reuse_roots(
publish_layout
.release_reuse_roots(&active_resource_root)
.map_err(anyhow::Error::msg)?,
)
.with_cas_reuse_root(config.effective_import_cas_root())
.with_max_concurrency(config.download_concurrency);
let pruned_stale_resource_count = staging_fetcher
.prune_stale_manifest_entries(&pull_plan)
@@ -1898,13 +1950,24 @@ impl OfficialUpdateService {
progress(OfficialUpdateProgress::new(
"download",
format!(
"下载阶段完成:已下载={} 已续传={} 复用={} 本轮传输字节={}",
"下载阶段完成:已下载={} 已续传={} 当前 manifest 复用={} 历史 release 复用={} CAS 复用={} 本轮传输字节={}",
pull_report.downloaded_count(),
pull_report.resumed_count(),
pull_report.skipped_count(),
pull_report.release_reused_count(),
pull_report.cas_reused_count(),
pull_report.transferred_bytes()
),
));
if !pull_report.reuse_warnings.is_empty() {
progress(OfficialUpdateProgress::new(
"download",
format!(
"复用候选诊断:{} 项候选未通过校验,已按顺序回退",
pull_report.reuse_warnings.len()
),
));
}
progress(OfficialUpdateProgress::new(
"audit",
"执行最终本地 manifest 审计",
@@ -1996,6 +2059,10 @@ impl OfficialUpdateService {
report.downloaded_count = pull_report.downloaded_count();
report.resumed_count = pull_report.resumed_count();
report.skipped_count = pull_report.skipped_count();
report.release_reused_count = pull_report.release_reused_count();
report.cas_reused_count = pull_report.cas_reused_count();
report.reused_bytes = pull_report.reused_bytes();
report.reuse_warnings = pull_report.reuse_warnings.clone();
report.final_bytes = pull_report.total_bytes();
report.transferred_bytes = pull_report.transferred_bytes();
report.official_seed_hash_verified_count = pull_report.official_hash_verified_count();
@@ -2050,7 +2117,11 @@ impl OfficialUpdateService {
// 清理未被最新版本状态引用的孤儿 staging 目录(GC 失败仅告警,不影响发布结果)。
match read_version_state(&version_state_path) {
Ok(Some(state)) => match gc_orphan_staging(&config.output_root, &state) {
Ok(Some(state)) => match gc_orphan_staging_with_cas_root(
&config.output_root,
&state,
&config.effective_import_cas_root(),
) {
Ok(removed) if !removed.is_empty() => progress(OfficialUpdateProgress::new(
"publish",
format!("已清理 {} 个孤儿 staging 目录", removed.len()),
@@ -2556,6 +2627,10 @@ fn waiting_for_official_resources_report(
downloaded_count: 0,
resumed_count: 0,
skipped_count: 0,
release_reused_count: 0,
cas_reused_count: 0,
reused_bytes: 0,
reuse_warnings: Vec::new(),
final_bytes: 0,
transferred_bytes: 0,
official_seed_hash_verified_count: 0,
@@ -2874,6 +2949,8 @@ fn verification_progress_message(summary: &OfficialVerificationSummary) -> Strin
fn localized_pull_status(status: crate::OfficialResourcePullStatus) -> &'static str {
match status {
crate::OfficialResourcePullStatus::SkippedExisting => "已复用",
crate::OfficialResourcePullStatus::ReleaseReused => "已复用历史 release",
crate::OfficialResourcePullStatus::CasReused => "已复用 CAS",
crate::OfficialResourcePullStatus::Resumed => "已续传",
crate::OfficialResourcePullStatus::Downloaded => "已下载",
}
@@ -3700,12 +3777,22 @@ pub fn gc_orphan_staging(
output_root: &Path,
state: &OfficialVersionState,
) -> anyhow::Result<Vec<PathBuf>> {
gc_orphan_staging_dirs(&output_root.join(OFFICIAL_STAGING_DIR), state)
gc_orphan_staging_with_cas_root(output_root, state, &output_root.join(".cas"))
}
/// Cleans orphan staging directories and releases their recorded CAS refs.
pub fn gc_orphan_staging_with_cas_root(
output_root: &Path,
state: &OfficialVersionState,
cas_root: &Path,
) -> anyhow::Result<Vec<PathBuf>> {
gc_orphan_staging_dirs(&output_root.join(OFFICIAL_STAGING_DIR), state, cas_root)
}
fn gc_orphan_staging_dirs(
staging_dir: &Path,
state: &OfficialVersionState,
cas_root: &Path,
) -> anyhow::Result<Vec<PathBuf>> {
let read_dir = match fs::read_dir(staging_dir) {
Ok(read_dir) => read_dir,
@@ -3740,6 +3827,7 @@ fn gc_orphan_staging_dirs(
if referenced.contains(&name) {
continue;
}
release_cas_reuse_references(&path, cas_root).map_err(anyhow::Error::msg)?;
fs::remove_dir_all(&path).map_err(|error| {
anyhow::anyhow!("清理孤儿 staging 目录失败 {}{error}", path.display())
})?;