//! Official JP resource download execution. use crate::curl_transfer::{run_curl_with_retry_with_proxy, CurlProxyConfig, CurlRetryError}; use crate::downloader::{ DownloadScheduler, DownloaderBackend, DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY, MIN_DOWNLOAD_CONCURRENCY, }; use crate::official_pull::OfficialResourcePullPlan; use crate::path_security::{ ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, read_file_no_symlink, validate_output_root, write_file_atomic, STATE_FILE_MODE, }; use crate::zip_validation::{ path_has_zip_extension, url_or_path_has_zip_extension, validate_zip_structure, }; use bat_adapters::official::yostar_jp::YostarJpResourceEndpointKind; use bat_adapters::official::{ destination_under_root, DownloadUrlMapper, OfficialResourceBackend, SidecarHashStrategy, XxHash32DecimalSeedZero, YostarJpBackend, }; use bat_core::repositories::CasRepository; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs::{self, File}; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; /// 官方资源下载错误：携带统一错误码，便于 CLI/RPC 归类。 /// /// 下载链路内部大量使用 `Result<_, String>`；这些经 `From` 归入 /// `internal`，只有 curl 下载失败等有明确来源的错误会带上准确的网络域码。 #[derive(Debug)] pub struct DownloadError { code: bat_core::ErrorCode, message: String, } impl DownloadError { pub(crate) fn new(code: bat_core::ErrorCode, message: impl Into) -> Self { Self { code, message: message.into(), } } /// 该错误的统一错误码。 pub fn code(&self) -> bat_core::ErrorCode { self.code } } impl From for DownloadError { fn from(message: String) -> Self { Self::new(bat_core::ErrorCode::INTERNAL, message) } } impl std::fmt::Display for DownloadError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str(&self.message) } } impl std::error::Error for DownloadError {} const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json"; const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json"; /// Independent publication fact for the official distribution manifest. pub const OFFICIAL_DISTRIBUTION_PUBLICATION_FILE: &str = "official-distribution-publication.json"; /// Current official distribution verification result. pub const OFFICIAL_DISTRIBUTION_ATTESTATION_FILE: &str = "official-distribution-attestation.json"; /// Persisted attestation schema version. pub const OFFICIAL_DISTRIBUTION_ATTESTATION_VERSION: u32 = 1; /// Attestations older than this are no longer allowed to authorize current CDN. pub const OFFICIAL_DISTRIBUTION_ATTESTATION_MAX_AGE_SECONDS: u64 = 900; /// 记录一个已发布官方 release 获取的 CAS 引用。 pub const OFFICIAL_CAS_REUSE_REFERENCES_FILE: &str = "official-cas-reuse-references.json"; const OFFICIAL_CAS_REUSE_REFERENCES_VERSION: u32 = 1; const DOWNLOAD_MANIFEST_VERSION: u32 = 1; const DOWNLOAD_QUARANTINE_VERSION: u32 = 1; const DEFAULT_RETRY_ATTEMPTS: usize = 3; const CAS_OWNER_SCOPE_FILE: &str = ".cas-owner-scope"; const CAS_OWNER_SCOPE_VERSION: u32 = 1; static CAS_OWNERSHIP_SEQUENCE: AtomicU64 = AtomicU64::new(1); #[derive(Debug, Clone, Serialize, Deserialize)] struct CasOwnerScopeState { version: u32, scope_id: String, #[serde(default)] next_generation: BTreeMap, #[serde(default)] completed_legacy_release_ids: BTreeSet, #[serde(default)] legacy_basename_compatibility: BTreeSet, } fn ownership_scope_root(release_root: &Path) -> PathBuf { let Some(parent) = release_root.parent() else { return release_root.to_path_buf(); }; match parent.file_name().and_then(|name| name.to_str()) { Some("versions" | ".staging") => parent.parent().unwrap_or(parent).to_path_buf(), _ => release_root.to_path_buf(), } } fn load_owner_scope_state(release_root: &Path) -> Result { let scope_root = ownership_scope_root(release_root); ensure_safe_directory_path(&scope_root, "CAS ownership scope 根目录")?; fs::create_dir_all(&scope_root) .map_err(|error| format!("创建 CAS ownership scope 根目录失败：{error}"))?; let path = scope_root.join(CAS_OWNER_SCOPE_FILE); let Some(bytes) = read_file_no_symlink(&path, "CAS ownership scope")? else { let sequence = CAS_OWNERSHIP_SEQUENCE.fetch_add(1, Ordering::Relaxed); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or_default(); let material = format!("{}:{}:{}", scope_root.display(), std::process::id(), now); let state = CasOwnerScopeState { version: CAS_OWNER_SCOPE_VERSION, scope_id: format!( "cas-scope-{}", blake3::hash(format!("{material}:{sequence}").as_bytes()).to_hex() ), next_generation: BTreeMap::new(), completed_legacy_release_ids: BTreeSet::new(), legacy_basename_compatibility: BTreeSet::new(), }; write_owner_scope_state(&scope_root, &state)?; return Ok(state); }; let state: CasOwnerScopeState = serde_json::from_slice(&bytes) .map_err(|error| format!("解析 CAS ownership scope 失败 {}：{error}", path.display()))?; if state.version != CAS_OWNER_SCOPE_VERSION || state.scope_id.is_empty() { return Err(format!("不支持的 CAS ownership scope：{}", path.display())); } Ok(state) } fn write_owner_scope_state(release_root: &Path, state: &CasOwnerScopeState) -> Result<(), String> { let scope_root = ownership_scope_root(release_root); let path = scope_root.join(CAS_OWNER_SCOPE_FILE); ensure_path_within_root(&scope_root, &path)?; ensure_safe_file_target(&scope_root, &path, "CAS ownership scope")?; let bytes = serde_json::to_vec_pretty(state) .map_err(|error| format!("序列化 CAS ownership scope 失败：{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "CAS ownership scope") } fn new_cas_ownership_id(output_root: &Path) -> Result { let scope = load_owner_scope_state(output_root)?; let sequence = CAS_OWNERSHIP_SEQUENCE.fetch_add(1, Ordering::Relaxed); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or_default(); let material = format!( "{}:{}:{}:{}", scope.scope_id, std::process::id(), now, sequence, ); Ok(format!( "cas-owner-{}", blake3::hash(material.as_bytes()).to_hex() )) } /// Returns the deterministic identity of an official distribution mapping. /// /// The framing is deliberately independent from JSON serialization and map /// iteration details. Destination is the primary sort key; the remaining /// fields make duplicate destinations deterministic as well. pub fn official_distribution_mapping_identity(manifest: &OfficialDownloadManifest) -> String { let mut entries = manifest.entries.values().collect::>(); entries.sort_by(|left, right| { left.destination .cmp(&right.destination) .then_with(|| left.url.cmp(&right.url)) .then_with(|| left.bytes.cmp(&right.bytes)) .then_with(|| left.blake3.cmp(&right.blake3)) }); let mut hasher = blake3::Hasher::new(); hasher.update(b"official-distribution-mapping-v1"); hasher.update(&(entries.len() as u64).to_be_bytes()); for entry in entries { update_identity_string(&mut hasher, &entry.destination); update_identity_string(&mut hasher, &entry.url); hasher.update(&entry.bytes.to_be_bytes()); update_identity_string(&mut hasher, &entry.blake3); } format!("odm-v1-{}", hasher.finalize().to_hex()) } pub(crate) fn official_distribution_destination_index( manifest: &OfficialDownloadManifest, ) -> Result, String> { let mut index = BTreeMap::new(); for entry in manifest.entries.values() { if index .insert(entry.destination.clone(), entry.url.clone()) .is_some() { return Err(format!( "official distribution manifest 存在重复 destination：{}", entry.destination )); } } Ok(index) } fn update_identity_string(hasher: &mut blake3::Hasher, value: &str) { hasher.update(&(value.len() as u64).to_be_bytes()); hasher.update(value.as_bytes()); } /// Outcome for one official resource pull item. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OfficialResourcePullStatus { /// The destination already existed and passed local manifest validation. SkippedExisting, /// The resource was materialized from a previously published release. ReleaseReused, /// The resource was materialized from a CAS object. CasReused, /// A partial `.part` file was resumed. Resumed, /// The resource was downloaded from scratch. Downloaded, } impl OfficialResourcePullStatus { /// Returns a stable label for reports and progress logs. pub fn as_str(self) -> &'static str { match self { Self::SkippedExisting => "skipped_existing", Self::ReleaseReused => "release_reused", Self::CasReused => "cas_reused", Self::Resumed => "resumed", Self::Downloaded => "downloaded", } } } /// Progress event kind emitted while executing an official pull plan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialResourcePullProgressKind { /// A URL is about to be checked or reused/downloaded. Started, /// A URL finished as skipped, reused, resumed, or downloaded. Finished, /// An official sidecar hash pair passed verification. Verification, /// A URL failed after retry classification and was quarantined for this run. Failed, } impl OfficialResourcePullProgressKind { /// Returns a stable label for progress logs. pub fn as_str(self) -> &'static str { match self { Self::Started => "started", Self::Finished => "finished", Self::Verification => "verification", Self::Failed => "failed", } } } /// Local validation results for one completed resource. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceVerification { /// Expected byte size from the local manifest, when reusing a file. pub expected_bytes: Option, /// Actual byte size observed after the transfer. pub actual_bytes: u64, /// Expected BLAKE3 from the local manifest, when reusing a file. pub expected_blake3: Option, /// Actual BLAKE3 computed from the completed file. pub actual_blake3: String, /// Whether ZIP structure validation was required for this resource. pub zip_checked: bool, /// Whether the required ZIP structure validation passed. pub zip_structure_verified: bool, } /// Progress for one URL in an official pull plan. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialResourcePullProgress { /// Progress event kind. pub kind: OfficialResourcePullProgressKind, /// Monotonic completed item count for the whole pull. /// /// `Started` events report the currently completed count before the URL /// finishes; `Finished` events report the count after completion. This is /// intentionally not the URL's plan position, so status percentages stay /// monotonic even if execution order or skip/resume mix changes. pub index: usize, /// Total URL count in the pull plan. pub total: usize, /// Official URL currently being processed. pub url: String, /// Finished pull status, when `kind` is `Finished`. pub status: Option, /// Final local file size, when `kind` is `Finished`. pub bytes: Option, /// Bytes transferred during this run, when `kind` is `Finished`. pub transferred_bytes: Option, /// Local size/BLAKE3/ZIP validation for a completed URL. pub verification: Option, /// Official sidecar hash verification completed after this URL. pub official_hash: Option, /// Stable failure kind label, when `kind` is `Failed`. pub failure_kind: Option, /// HTTP status parsed from curl stderr, when available. pub failure_http_status: Option, /// Whether retry policy considered the final failure retryable. pub failure_retryable: Option, /// Number of curl attempts that were executed before this failure. pub failure_attempts: Option, /// Whether the URL was written to the local quarantine manifest. pub quarantined: bool, } impl OfficialResourcePullProgress { fn started(index: usize, total: usize, url: String) -> Self { Self { kind: OfficialResourcePullProgressKind::Started, index, total, url, status: None, bytes: None, transferred_bytes: None, verification: None, official_hash: None, failure_kind: None, failure_http_status: None, failure_retryable: None, failure_attempts: None, quarantined: false, } } fn finished( index: usize, total: usize, url: String, status: OfficialResourcePullStatus, bytes: u64, transferred_bytes: u64, verification: OfficialResourceVerification, ) -> Self { Self { kind: OfficialResourcePullProgressKind::Finished, index, total, url, status: Some(status), bytes: Some(bytes), transferred_bytes: Some(transferred_bytes), verification: Some(verification), official_hash: None, failure_kind: None, failure_http_status: None, failure_retryable: None, failure_attempts: None, quarantined: false, } } fn verification( index: usize, total: usize, url: String, official_hash: OfficialResourceHashVerification, ) -> Self { Self { kind: OfficialResourcePullProgressKind::Verification, index, total, url, status: None, bytes: None, transferred_bytes: None, verification: None, official_hash: Some(official_hash), failure_kind: None, failure_http_status: None, failure_retryable: None, failure_attempts: None, quarantined: false, } } fn failed(index: usize, total: usize, url: String, error: &PullOneError) -> Self { Self { kind: OfficialResourcePullProgressKind::Failed, index, total, url, status: None, bytes: None, transferred_bytes: None, verification: None, official_hash: None, failure_kind: error.failure_kind().map(ToOwned::to_owned), failure_http_status: error.http_status(), failure_retryable: error.retryable(), failure_attempts: error.attempts(), quarantined: true, } } } /// Official hash algorithm used by a verified resource sidecar. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum OfficialResourceHashAlgorithm { /// Decimal text form of `xxHash32` with seed `0`. #[serde(rename = "xxhash32_decimal")] XxHash32Decimal, } impl OfficialResourceHashAlgorithm { /// Returns a stable algorithm label for reports and logs. pub fn as_str(self) -> &'static str { match self { Self::XxHash32Decimal => "xxhash32_decimal", } } } /// Successful official hash verification for one downloaded resource. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceHashVerification { /// Downloaded data URL that was verified. pub data_url: String, /// Official `.hash` URL used as the verification source. pub hash_url: String, /// Official hash algorithm. pub algorithm: OfficialResourceHashAlgorithm, /// Expected hash value read from the official `.hash` file. pub expected: String, /// Actual hash value computed from the downloaded data file. pub actual: String, } /// One downloaded official resource. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourcePullItem { /// Original URL. pub url: String, /// Local destination path under the output root. pub destination: PathBuf, /// Final local file size. pub bytes: u64, /// Bytes transferred during this pull execution. pub transferred_bytes: u64, /// Pull outcome. pub status: OfficialResourcePullStatus, /// CAS object referenced when this item was reused from CAS. #[serde(default)] pub cas_object_id: Option, } /// Download report for one executed pull plan. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourcePullReport { /// Downloaded resources in pull-plan order, independent of completion order. pub items: Vec, /// Official hash sidecars successfully verified after download. pub verified_hashes: Vec, /// Non-fatal reuse diagnostics that fell back to the next source or network. pub reuse_warnings: Vec, } /// Non-fatal diagnostic emitted when a historical release or CAS candidate /// could not be reused and the pull fell back to another source. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceReuseWarning { /// Current official URL being processed. pub url: String, /// Reuse source class, such as `historical_release` or `cas`. pub source: String, /// Human-readable diagnostic message. pub message: String, } /// Versioned CAS references acquired while materializing one official release. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialCasReuseReferenceManifest { /// 引用清单版本。 pub version: u32, /// Persistent ownership identity for this release generation. /// /// `None` is an explicit legacy manifest marker. Legacy cleanup keeps the /// historical basename key and never invents a new identity while loading. #[serde(default)] pub ownership_id: Option, /// 每个 CAS 引用一项。允许重复，因为每个拉取项分别拥有一个引用。 pub object_ids: Vec, } impl Default for OfficialCasReuseReferenceManifest { fn default() -> Self { Self { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: Vec::new(), } } } /// Reads a release-local CAS reference manifest. pub fn read_cas_reuse_reference_manifest_at( release_root: &Path, ) -> Result, String> { let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE); let Some(bytes) = read_file_no_symlink(&path, "官方 release CAS 引用清单")? else { return Ok(None); }; let manifest: OfficialCasReuseReferenceManifest = serde_json::from_slice(&bytes).map_err(|error| { format!( "解析官方 release CAS 引用清单失败 {}：{error}", path.display() ) })?; if manifest.version != OFFICIAL_CAS_REUSE_REFERENCES_VERSION { return Err(format!( "不支持的官方 release CAS 引用清单版本 {}，文件 {}", manifest.version, path.display() )); } Ok(Some(manifest)) } fn cas_has_ownership(cas_root: &Path, ownership_id: &str) -> Result { let cas_root = cas_root.to_path_buf(); let ownership_id = ownership_id.to_string(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|error| format!("创建 CAS ownership 查询 runtime 失败：{error}"))?; runtime.block_on(async move { let cas = crate::FileSystemCasRepository::new(cas_root); cas.has_release_ownership(&ownership_id) .await .map_err(|error| format!("查询 CAS ownership ledger 失败：{error}")) }) } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct OfficialDistributionPublicationAnchor { pub(crate) version: u32, pub(crate) official_release_id: String, pub(crate) mapping_identity: String, pub(crate) manifest_identity: String, pub(crate) entry_count: u64, } const OFFICIAL_DISTRIBUTION_PUBLICATION_VERSION: u32 = 1; /// Rust-owned lightweight proof that the current official publication is safe /// for the read-only distribution path. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialDistributionAttestation { /// Attestation schema version. pub version: u32, /// Distribution channel; currently always `official`. pub channel: String, /// Stable official release ID. pub official_release_id: String, /// Published version root this result describes. pub resource_root: PathBuf, /// Identity of the publication anchor and its manifest generation. pub publication_identity: String, /// Identity of the complete destination mapping. pub mapping_identity: String, /// BLAKE3 identity of the manifest bytes. pub manifest_identity: String, /// Number of entries in the bound manifest. pub entry_count: u64, /// `verified`, `stale`, `invalid`, or `unavailable`. pub integrity_status: String, /// Human-readable stable state label. pub status: String, /// Namespaced status code consumed by RPC clients. pub status_code: String, /// Whether this attestation currently authorizes distribution. pub ready: bool, /// Monotonic verification generation for this published root. pub verification_generation: u64, /// Time of the last successful full local verification. #[serde(default, skip_serializing_if = "Option::is_none")] pub verified_at: Option, /// Freshness window used by the lightweight RPC reader. pub max_age_seconds: u64, /// Diagnostics retained with the result. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub diagnostics: Vec, } pub(crate) fn official_distribution_publication_identity( anchor: &OfficialDistributionPublicationAnchor, ) -> String { format!( "odp-v1-{}-{}", anchor.mapping_identity, anchor.manifest_identity ) } fn official_distribution_attestation_status_code(integrity_status: &str) -> &'static str { match integrity_status { "verified" => "distribution.ready", "stale" => "distribution.attestation_stale", "invalid" => "distribution.attestation_invalid", _ => "distribution.attestation_unavailable", } } fn official_distribution_attestation_status(integrity_status: &str) -> &'static str { match integrity_status { "verified" => "ready", "stale" => "stale", "invalid" => "invalid", _ => "unavailable", } } pub(crate) fn read_official_distribution_attestation_at( release_root: &Path, ) -> Result, String> { let path = release_root.join(OFFICIAL_DISTRIBUTION_ATTESTATION_FILE); let Some(bytes) = read_file_no_symlink(&path, "官方 distribution attestation")? else { return Ok(None); }; let attestation: OfficialDistributionAttestation = serde_json::from_slice(&bytes) .map_err(|error| format!("解析官方 distribution attestation 失败：{error}"))?; if attestation.version != OFFICIAL_DISTRIBUTION_ATTESTATION_VERSION { return Err(format!( "不支持的官方 distribution attestation 版本：{}", attestation.version )); } Ok(Some(attestation)) } /// Records a verification result for one already published official root. /// /// The publication anchor is reused as the immutable generation identity. /// The caller chooses `verified` only after the existing full local audit has /// passed; this function itself never turns a partial audit into a healthy /// result. pub(crate) fn write_official_distribution_attestation_at( release_root: &Path, official_release_id: &str, integrity_status: &str, diagnostics: Vec, ) -> Result { if !matches!( integrity_status, "verified" | "stale" | "invalid" | "unavailable" ) { return Err(format!( "不支持的官方 distribution attestation 状态：{integrity_status}" )); } ensure_safe_directory_path(release_root, "官方 distribution attestation 根目录")?; let anchor = verify_official_distribution_publication_at(release_root, official_release_id)? .ok_or_else(|| { format!( "官方 distribution attestation 缺少 publication anchor：{}", release_root.display() ) })?; let previous_generation = read_official_distribution_attestation_at(release_root)? .map(|previous| previous.verification_generation) .unwrap_or(0); let verified_at = (integrity_status == "verified").then_some(unix_seconds_now()); let attestation = OfficialDistributionAttestation { version: OFFICIAL_DISTRIBUTION_ATTESTATION_VERSION, channel: "official".to_string(), official_release_id: official_release_id.to_string(), resource_root: release_root.to_path_buf(), publication_identity: official_distribution_publication_identity(&anchor), mapping_identity: anchor.mapping_identity, manifest_identity: anchor.manifest_identity, entry_count: anchor.entry_count, integrity_status: integrity_status.to_string(), status: official_distribution_attestation_status(integrity_status).to_string(), status_code: official_distribution_attestation_status_code(integrity_status).to_string(), ready: integrity_status == "verified", verification_generation: previous_generation.saturating_add(1), verified_at, max_age_seconds: OFFICIAL_DISTRIBUTION_ATTESTATION_MAX_AGE_SECONDS, diagnostics, }; let path = release_root.join(OFFICIAL_DISTRIBUTION_ATTESTATION_FILE); ensure_safe_file_target(release_root, &path, "官方 distribution attestation")?; let bytes = serde_json::to_vec_pretty(&attestation) .map_err(|error| format!("序列化官方 distribution attestation 失败：{error}"))?; write_file_atomic( &path, &bytes, STATE_FILE_MODE, "官方 distribution attestation", )?; Ok(attestation) } /// Writes the independent publication anchor after the complete official /// release verification has succeeded. /// /// The caller must invoke this only for a fully audited staging tree immediately /// before publishing it. The function repeats the manifest/file checks so the /// anchor cannot be created over an incomplete tree. pub(crate) fn write_official_distribution_publication_anchor_at( release_root: &Path, official_release_id: &str, ) -> Result<(), String> { if official_release_id.is_empty() { return Err("官方 distribution publication 缺少 release ID".to_string()); } ensure_safe_directory_path(release_root, "官方 distribution publication 根目录")?; let manifest_path = release_root.join(DOWNLOAD_MANIFEST_FILE); let manifest_bytes = read_file_no_symlink(&manifest_path, "官方下载 manifest")? .ok_or_else(|| format!("缺少官方下载 manifest：{}", manifest_path.display()))?; let manifest: OfficialDownloadManifest = serde_json::from_slice(&manifest_bytes).map_err(|error| { format!( "解析官方下载 manifest 失败 {}：{error}", manifest_path.display() ) })?; if manifest.version != DOWNLOAD_MANIFEST_VERSION { return Err(format!( "不支持的官方下载 manifest 版本：{}", manifest.version )); } let mapping_identity = official_distribution_mapping_identity(&manifest); let expected_index = official_distribution_destination_index(&manifest)?; if manifest.distribution_mapping_identity.as_deref() != Some(mapping_identity.as_str()) || manifest.destination_index != expected_index { return Err(format!( "官方下载 manifest 未完成 distribution mapping 验证：{}", manifest_path.display() )); } for entry in manifest.entries.values() { let path = release_root.join(&entry.destination); ensure_path_within_root(release_root, &path)?; ensure_safe_file_target(release_root, &path, "官方 distribution 文件")?; let bytes = fs::read(&path).map_err(|error| { format!("读取官方 distribution 文件失败 {}：{error}", path.display()) })?; let actual = blake3::hash(&bytes).to_hex().to_string(); if bytes.len() as u64 != entry.bytes || actual != entry.blake3 { return Err(format!( "官方 distribution 文件完整性失败：{}", entry.destination )); } if url_or_path_has_zip_extension(&entry.url) || path_has_zip_extension(&path) { validate_zip_structure(&path).map_err(|error| { format!( "官方 distribution ZIP 结构校验失败 {}：{error}", path.display() ) })?; } } let anchor = OfficialDistributionPublicationAnchor { version: OFFICIAL_DISTRIBUTION_PUBLICATION_VERSION, official_release_id: official_release_id.to_string(), mapping_identity, manifest_identity: blake3::hash(&manifest_bytes).to_hex().to_string(), entry_count: manifest.entries.len() as u64, }; let anchor_path = release_root.join(OFFICIAL_DISTRIBUTION_PUBLICATION_FILE); ensure_safe_file_target(release_root, &anchor_path, "官方 distribution publication")?; let anchor_bytes = serde_json::to_vec_pretty(&anchor) .map_err(|error| format!("序列化官方 distribution publication 失败：{error}"))?; write_file_atomic( &anchor_path, &anchor_bytes, STATE_FILE_MODE, "官方 distribution publication", ) } /// Verifies a published official distribution anchor without hashing resource /// files or canonicalizing the complete mapping. pub(crate) fn verify_official_distribution_publication_at( release_root: &Path, expected_release_id: &str, ) -> Result, String> { ensure_safe_directory_path(release_root, "官方 distribution publication 根目录")?; let anchor_path = release_root.join(OFFICIAL_DISTRIBUTION_PUBLICATION_FILE); let Some(anchor_bytes) = read_file_no_symlink(&anchor_path, "官方 distribution publication")? else { return Ok(None); }; let anchor: OfficialDistributionPublicationAnchor = serde_json::from_slice(&anchor_bytes) .map_err(|error| format!("解析官方 distribution publication 失败：{error}"))?; if anchor.version != OFFICIAL_DISTRIBUTION_PUBLICATION_VERSION || anchor.official_release_id != expected_release_id || anchor.mapping_identity.is_empty() || anchor.manifest_identity.is_empty() { return Err(format!( "官方 distribution publication anchor 不一致：{}", anchor_path.display() )); } let manifest_path = release_root.join(DOWNLOAD_MANIFEST_FILE); let Some(manifest_bytes) = read_file_no_symlink(&manifest_path, "官方下载 manifest")? else { return Err(format!( "官方 distribution publication 缺少 manifest：{}", manifest_path.display() )); }; let manifest: OfficialDownloadManifest = serde_json::from_slice(&manifest_bytes).map_err(|error| { format!( "解析官方下载 manifest 失败 {}：{error}", manifest_path.display() ) })?; if manifest.version != DOWNLOAD_MANIFEST_VERSION { return Err(format!( "不支持的官方下载 manifest 版本：{}", manifest.version )); } if manifest.distribution_mapping_identity.as_deref() != Some(anchor.mapping_identity.as_str()) || blake3::hash(&manifest_bytes).to_hex().to_string() != anchor.manifest_identity || manifest.entries.len() as u64 != anchor.entry_count || manifest.destination_index.len() as u64 != anchor.entry_count { return Err(format!( "官方 distribution publication 与 manifest 不一致：{}", release_root.display() )); } Ok(Some(anchor)) } fn legacy_source_mapping_identity(release_root: &Path) -> Result { let Some(manifest) = read_download_manifest_at(release_root)? else { return Ok("missing-official-manifest".to_string()); }; Ok(manifest .distribution_mapping_identity .clone() .unwrap_or_else(|| official_distribution_mapping_identity(&manifest))) } fn resolve_legacy_ownership( release_root: &Path, cas_root: &Path, ) -> Result<(String, bool), String> { let release_id = release_root .file_name() .and_then(|name| name.to_str()) .filter(|name| !name.is_empty() && *name != "." && *name != "..") .ok_or_else(|| { format!( "无法从 release 路径确定 CAS ownership ID：{}", release_root.display() ) })? .to_string(); let mut scope = load_owner_scope_state(release_root)?; if scope.completed_legacy_release_ids.contains(&release_id) { let source_identity = legacy_source_mapping_identity(release_root)?; let generation_key = format!("{release_id}\0{source_identity}"); let generation = scope.next_generation.entry(generation_key).or_insert(0); let current_generation = *generation; *generation = generation.saturating_add(1); write_owner_scope_state(release_root, &scope)?; let owner_material = format!( "cas-legacy-owner-v1:{}:{}:{}:{}", scope.scope_id, release_id, source_identity, current_generation ); return Ok(( format!( "cas-legacy-owner-v1-{}", blake3::hash(owner_material.as_bytes()).to_hex() ), false, )); } if scope.legacy_basename_compatibility.contains(&release_id) { return Ok((release_id, true)); } if cas_has_ownership(cas_root, &release_id)? { return Err(format!( "ambiguous_legacy_ownership: CAS basename ledger {} 无法证明属于 output root {}；请先显式记录 compatibility marker", release_id, ownership_scope_root(release_root).display() )); } let source_identity = legacy_source_mapping_identity(release_root)?; let generation_key = format!("{release_id}\0{source_identity}"); let generation = scope.next_generation.entry(generation_key).or_insert(0); let current_generation = *generation; *generation = generation.saturating_add(1); write_owner_scope_state(release_root, &scope)?; let owner_material = format!( "cas-legacy-owner-v1:{}:{}:{}:{}", scope.scope_id, release_id, source_identity, current_generation ); Ok(( format!( "cas-legacy-owner-v1-{}", blake3::hash(owner_material.as_bytes()).to_hex() ), false, )) } fn mark_legacy_ownership_completed(release_root: &Path, release_id: &str) -> Result<(), String> { let mut scope = load_owner_scope_state(release_root)?; scope .completed_legacy_release_ids .insert(release_id.to_string()); write_owner_scope_state(release_root, &scope) } /// Decrements and removes CAS references recorded for a release. /// /// Each decrement is committed together with a durable `(ownership, ordinal)` /// record in CAS metadata. The release-local manifest remains a resumable /// progress cursor, so a crash before its rewrite cannot decrement the same /// ownership twice. Legacy manifests without `ownership_id` use the historical /// release-basename key only when an existing ledger requires compatibility; /// otherwise cleanup persists a generation-aware identity before decrementing. pub fn release_cas_reuse_references(release_root: &Path, cas_root: &Path) -> Result { let Some(mut manifest) = read_cas_reuse_reference_manifest_at(release_root)? else { return Ok(0); }; let objects_root = cas_root.join("objects"); let metadata_path = cas_root.join("metadata.sqlite"); require_existing_directory(cas_root, "CAS 根目录")?; require_existing_directory(&objects_root, "CAS 对象目录")?; require_existing_file(cas_root, &metadata_path, "CAS 元数据库")?; let (ownership_id, legacy_basename_compatibility) = match manifest.ownership_id.clone() { Some(ownership_id) => (ownership_id, false), None => resolve_legacy_ownership(release_root, cas_root)?, }; if manifest.ownership_id.is_none() && !legacy_basename_compatibility { manifest.ownership_id = Some(ownership_id.clone()); write_cas_reuse_reference_manifest(release_root, &manifest)?; } let legacy_release_id = release_root .file_name() .and_then(|name| name.to_str()) .unwrap_or_default() .to_string(); let mut released = 0usize; while let Some(object_id) = manifest.object_ids.pop() { let ordinal = manifest.object_ids.len() as u64; let cas_root = cas_root.to_path_buf(); let object_id_for_runtime = object_id.clone(); let ownership_id_for_runtime = ownership_id.clone(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|error| format!("创建 CAS 引用清理 runtime 失败：{error}"))?; let did_release = runtime.block_on(async move { let cas = crate::FileSystemCasRepository::new(cas_root); cas.release_reference_once(&ownership_id_for_runtime, ordinal, &object_id_for_runtime) .await .map_err(|error| format!("减少 CAS release 引用失败 object={object_id}：{error}")) })?; write_cas_reuse_reference_manifest(release_root, &manifest)?; if did_release { released += 1; } } let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE); if legacy_basename_compatibility { mark_legacy_ownership_completed(release_root, &legacy_release_id)?; } match fs::symlink_metadata(&path) { Ok(metadata) if metadata.file_type().is_symlink() => { return Err(format!( "官方 release CAS 引用清单不能是 symlink：{}", path.display() )) } Ok(_) => fs::remove_file(&path) .map_err(|error| format!("删除空官方 release CAS 引用清单失败：{error}"))?, Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { return Err(format!( "检查官方 release CAS 引用清单失败 {}：{error}", path.display() )) } } Ok(released) } fn write_cas_reuse_reference_manifest( release_root: &Path, manifest: &OfficialCasReuseReferenceManifest, ) -> Result<(), String> { let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE); ensure_path_within_root(release_root, &path)?; ensure_safe_file_target(release_root, &path, "官方 release CAS 引用清单")?; let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| format!("序列化官方 release CAS 引用清单失败：{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 release CAS 引用清单") } impl OfficialResourcePullReport { /// Returns the total downloaded byte count. pub fn total_bytes(&self) -> u64 { self.items.iter().map(|item| item.bytes).sum() } /// Returns bytes transferred by this pull execution. pub fn transferred_bytes(&self) -> u64 { self.items.iter().map(|item| item.transferred_bytes).sum() } /// Returns the number of resources reused from the local output tree. pub fn skipped_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::SkippedExisting) .count() } /// Returns the number of resources reused from historical releases. pub fn release_reused_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::ReleaseReused) .count() } /// Returns the number of resources reused from CAS. pub fn cas_reused_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::CasReused) .count() } /// Returns the byte count satisfied without network transfer. pub fn reused_bytes(&self) -> u64 { self.items .iter() .filter(|item| { matches!( item.status, OfficialResourcePullStatus::SkippedExisting | OfficialResourcePullStatus::ReleaseReused | OfficialResourcePullStatus::CasReused ) }) .map(|item| item.bytes) .sum() } /// Returns the number of resources resumed from `.part` files. pub fn resumed_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::Resumed) .count() } /// Returns the number of resources downloaded from scratch. pub fn downloaded_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::Downloaded) .count() } /// Returns the number of official hash sidecars verified after download. pub fn official_hash_verified_count(&self) -> usize { self.verified_hashes.len() } } /// Local download-manifest audit status for one expected official URL. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialLocalManifestAuditStatus { /// The local file exists and matches the recorded manifest size and BLAKE3. Verified, /// No manifest entry exists for this URL. MissingManifestEntry, /// The manifest entry URL does not match the expected URL. UrlMismatch, /// The manifest destination does not match the URL-derived destination. DestinationMismatch, /// The manifest entry exists but the local file is missing. MissingFile, /// The local file size differs from the manifest entry. SizeMismatch, /// The local file BLAKE3 differs from the manifest entry. Blake3Mismatch, /// The local file has a `.zip` URL or destination but its ZIP structure is /// invalid. ZipStructureInvalid, } impl OfficialLocalManifestAuditStatus { /// Returns true when this status proves the local file can be reused. pub fn is_verified(self) -> bool { self == Self::Verified } /// Returns a stable status label for CLI and JSON reports. pub fn as_str(self) -> &'static str { match self { Self::Verified => "verified", Self::MissingManifestEntry => "missing_manifest_entry", Self::UrlMismatch => "url_mismatch", Self::DestinationMismatch => "destination_mismatch", Self::MissingFile => "missing_file", Self::SizeMismatch => "size_mismatch", Self::Blake3Mismatch => "blake3_mismatch", Self::ZipStructureInvalid => "zip_structure_invalid", } } } /// Local download-manifest audit result for one expected official URL. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialLocalManifestAuditItem { /// Expected official URL from the pull plan. pub url: String, /// URL-derived local destination. pub destination: PathBuf, /// Audit status. pub status: OfficialLocalManifestAuditStatus, /// Manifest-recorded destination, when an entry exists. pub manifest_destination: Option, /// Manifest-recorded byte size, when an entry exists. pub expected_bytes: Option, /// Actual local byte size, when the destination exists. pub actual_bytes: Option, /// Manifest-recorded BLAKE3, when an entry exists. pub expected_blake3: Option, /// Actual local BLAKE3, when it could be computed. pub actual_blake3: Option, /// ZIP structure validation error, when the URL or destination is a ZIP /// and the local file did not pass ZIP structure validation. pub zip_error: Option, /// Whether this item required and passed ZIP structure validation. pub zip_structure_verified: bool, } /// Local download-manifest audit result for a pull plan. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialLocalManifestAuditReport { /// Per-URL audit items in pull-plan order. pub items: Vec, } impl OfficialLocalManifestAuditReport { /// Returns true when every expected URL was locally verified. pub fn is_clean(&self) -> bool { self.items.iter().all(|item| item.status.is_verified()) } /// Returns the number of locally verified files. pub fn verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of files that require repair. pub fn repair_needed_count(&self) -> usize { self.items.len().saturating_sub(self.verified_count()) } /// Returns the number of files that passed local manifest path, size, and /// BLAKE3 validation. pub fn manifest_blake3_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of ZIP files that passed structural validation. pub fn zip_structure_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.zip_structure_verified) .count() } } /// Full local verification report for all entries currently recorded in the /// official download manifest. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialLocalVerificationReport { /// Per-entry manifest audit results. pub items: Vec, /// Official seed hash pairs found in the local manifest. pub official_hash_pair_count: usize, /// Official seed hash pairs that passed xxHash32 verification. pub official_hash_verified_count: usize, /// Official seed hash verification failures. pub official_hash_errors: Vec, } impl OfficialLocalVerificationReport { /// Returns true when every local manifest entry and every local official /// seed hash pair passed verification. pub fn is_clean(&self) -> bool { self.items.iter().all(|item| item.status.is_verified()) && self.official_hash_errors.is_empty() && self.official_hash_verified_count == self.official_hash_pair_count } /// Returns the number of manifest entries that passed verification. pub fn verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of manifest entries that need attention. pub fn failure_count(&self) -> usize { self.items.len().saturating_sub(self.verified_count()) } /// Returns the number of local manifest entries that passed path, size, /// and BLAKE3 validation. pub fn manifest_blake3_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of ZIP files that passed structural validation. pub fn zip_structure_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.zip_structure_verified) .count() } } /// Existing local state for an official pull plan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct OfficialLocalResourceState { /// Number of expected URLs that already have download-manifest entries. pub manifest_entry_count: usize, /// Number of expected URL destinations that already exist on disk. pub existing_file_count: usize, } impl OfficialLocalResourceState { /// Returns true when this plan has any reusable or auditable local state. pub fn has_any_resources(self) -> bool { self.manifest_entry_count > 0 || self.existing_file_count > 0 } } /// Executes an official pull plan with the system `curl` command. #[derive(Debug, Clone)] pub struct OfficialResourcePullService { output_root: PathBuf, backend: YostarJpBackend, curl_command: PathBuf, curl_proxy: CurlProxyConfig, retry_attempts: usize, max_concurrency: usize, release_reuse_roots: Vec, cas_reuse_root: Option, cas_reference_tracker: Arc>>, } impl OfficialResourcePullService { /// Creates a pull service that uses `curl` from `PATH`. pub fn new(output_root: impl Into) -> Self { Self::with_curl_command(output_root, "curl") } /// Creates a pull service with an explicit `curl` binary path. pub fn with_curl_command( output_root: impl Into, curl_command: impl Into, ) -> Self { Self { output_root: output_root.into(), backend: YostarJpBackend, curl_command: curl_command.into(), curl_proxy: CurlProxyConfig::default(), retry_attempts: DEFAULT_RETRY_ATTEMPTS, max_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY, release_reuse_roots: Vec::new(), cas_reuse_root: None, cas_reference_tracker: Arc::new(Mutex::new(Vec::new())), } } /// Sets the proxy configuration for every `curl` transfer. pub fn with_proxy_config(mut self, curl_proxy: CurlProxyConfig) -> Self { self.curl_proxy = curl_proxy; self } /// Sets the number of attempts for each `curl` transfer. pub fn with_retry_attempts(mut self, retry_attempts: usize) -> Self { self.retry_attempts = retry_attempts.max(1); self } /// Sets the bounded number of concurrent downloads. /// /// The default is [`DEFAULT_DOWNLOAD_CONCURRENCY`]. Values are kept /// within the supported `1..=256` range; the public update configuration /// validates input before constructing this service. pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self { self.max_concurrency = max_concurrency.clamp(MIN_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY); self } /// Enables best-effort reuse from immutable published release directories. pub fn with_release_reuse_root(mut self, release_root: impl Into) -> Self { self.release_reuse_roots.push(release_root.into()); self } /// Enables best-effort reuse from several immutable release roots. pub fn with_release_reuse_roots(mut self, release_roots: I) -> Self where I: IntoIterator, P: Into, { self.release_reuse_roots .extend(release_roots.into_iter().map(Into::into)); self } /// Enables best-effort reuse from an existing filesystem CAS root. pub fn with_cas_reuse_root(mut self, cas_root: impl Into) -> Self { self.cas_reuse_root = Some(cas_root.into()); self } /// Returns the configured download concurrency. pub fn max_concurrency(&self) -> usize { self.max_concurrency } /// Returns the output root used for downloaded files. pub fn output_root(&self) -> &Path { &self.output_root } /// Returns the local manifest path used to validate completed downloads. pub fn download_manifest_path(&self) -> PathBuf { self.output_root.join(DOWNLOAD_MANIFEST_FILE) } /// Returns the local quarantine path used to diagnose failed downloads. pub fn download_quarantine_path(&self) -> PathBuf { self.output_root.join(DOWNLOAD_QUARANTINE_FILE) } fn track_cas_reference(&self, object_id: &str) { if let Ok(mut references) = self.cas_reference_tracker.lock() { references.push(object_id.to_string()); } } fn untrack_cas_reference(&self, object_id: &str) { if let Ok(mut references) = self.cas_reference_tracker.lock() { if let Some(index) = references.iter().position(|id| id == object_id) { references.remove(index); } } } fn persist_cas_reuse_references( &self, report: &OfficialResourcePullReport, ) -> Result<(), String> { let object_ids = report .items .iter() .filter_map(|item| item.cas_object_id.clone()) .collect::>(); if object_ids.is_empty() { return Ok(()); } let mut manifest = match read_cas_reuse_reference_manifest_at(&self.output_root)? { Some(manifest) => manifest, None => OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: Some(new_cas_ownership_id(&self.output_root)?), object_ids: Vec::new(), }, }; manifest.object_ids.extend(object_ids); write_cas_reuse_reference_manifest(&self.output_root, &manifest) } /// Removes resource files recorded by an older manifest but absent from /// the current pull plan. /// /// This is intentionally limited to destinations owned by the previous /// download manifest. Management files such as snapshots, version state, /// and quarantine records are left untouched. pub fn prune_stale_manifest_entries( &self, plan: &OfficialResourcePullPlan, ) -> Result { self.ensure_output_root_ready()?; let expected_urls = plan.all_urls()?.into_iter().collect::>(); let mut manifest = self.read_download_manifest()?; let stale_entries = manifest .entries .iter() .filter(|(url, _)| !expected_urls.contains(*url)) .map(|(url, entry)| (url.clone(), entry.destination.clone())) .collect::>(); for (url, relative_destination) in &stale_entries { let destination = self.output_root.join(relative_destination); self.remove_managed_destination(&destination, url)?; self.remove_managed_destination(&partial_path_for(&destination), url)?; manifest.entries.remove(url); } if !stale_entries.is_empty() { self.write_download_manifest(&manifest)?; } Ok(stale_entries.len()) } /// Executes the plan and downloads every official URL to disk. pub fn pull( &self, plan: &OfficialResourcePullPlan, ) -> Result { self.pull_with_progress(plan, |_| {}) } /// Executes the plan and calls `progress` for every URL before and after it /// is checked or downloaded. pub fn pull_with_progress( &self, plan: &OfficialResourcePullPlan, mut progress: impl FnMut(OfficialResourcePullProgress), ) -> Result { self.pull_with_progress_and_cancellation(plan, &mut progress, || false) } /// Executes the plan, emits URL progress, and aborts between URLs when /// `should_cancel` returns true. pub fn pull_with_progress_and_cancellation( &self, plan: &OfficialResourcePullPlan, mut progress: impl FnMut(OfficialResourcePullProgress), mut should_cancel: impl FnMut() -> bool, ) -> Result { self.ensure_output_root_ready()?; let cas_reference_guard = CasReferenceRollbackGuard::new( self.cas_reuse_root.clone(), Arc::clone(&self.cas_reference_tracker), ); let official_hash_pairs = official_seed_hash_pairs(plan); let force_refresh_urls = official_hash_refresh_urls(&official_hash_pairs); let mut manifest = self.read_download_manifest()?; let release_reuse_index = self.build_release_reuse_index()?; let urls = plan.all_urls()?; let total = urls.len(); // Phase A：无网络的前置校验（fail-fast）。逐个校验官方性、算目标路径、 // 建目录，并判定该 URL 是「已验证可跳过」还是「需下载」。任何非官方 // URL 在发起任何下载前即拒绝。 if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string().into()); } let mut planned: Vec = Vec::with_capacity(total); let mut planned_destinations = HashSet::with_capacity(total); for url in urls { if !self.backend.is_official_url(&url) { return Err(DownloadError::new( bat_core::ErrorCode::NON_OFFICIAL_URL, format!("拒绝下载非官方 URL：{url}"), )); } let destination = self.destination_for_url(&url)?; if !planned_destinations.insert(destination.clone()) { return Err(DownloadError::new( bat_core::ErrorCode::INVALID_ARGUMENT, format!("官方资源拉取计划包含重复目标：{}", destination.display()), )); } if let Some(parent) = destination.parent() { ensure_safe_directory_path(parent, "下载目标目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载目标目录失败 {}：{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载目标目录")?; } ensure_safe_file_target(&self.output_root, &destination, "下载目标文件")?; let force_refresh = force_refresh_urls.contains(&url); let existing = if force_refresh { None } else { self.validated_existing_file(&url, &destination, &manifest)? }; let reuse_candidate = if existing.is_none() && !force_refresh { self.reuse_candidate_for_url(&url, &destination, &release_reuse_index)? } else { None }; planned.push(PlannedDownload { url, destination, existing, reuse_candidate, }); } if self.max_concurrency > 1 { let report = self.pull_planned_concurrently( planned, manifest, official_hash_pairs, release_reuse_index.warnings, &mut progress, &mut should_cancel, )?; self.persist_cas_reuse_references(&report)?; cas_reference_guard.commit(); return Ok(report); } // Phase B：按 plan 顺序处理每个 URL。下载或复用完成并写入 manifest 后， // 立即尝试校验已经到齐的官方 `.bytes/.hash` pair，避免把文件级问题延后到整轮末尾。 let mut completed_count = 0usize; let mut items = Vec::with_capacity(planned.len()); let mut verified_hashes = Vec::new(); let mut reuse_warnings = release_reuse_index .warnings .into_iter() .map(|message| OfficialResourceReuseWarning { url: String::new(), source: "historical_release_index".to_string(), message, }) .collect::>(); let mut verified_hash_urls = HashSet::::new(); let mut processed_urls = HashSet::::new(); for item in &planned { progress(OfficialResourcePullProgress::started( completed_count, total, item.url.clone(), )); if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string().into()); } let result = if let Some(existing) = &item.existing { self.clear_quarantine_entry(&item.url)?; existing.clone() } else { match self.pull_task(item.clone()) { Ok(mut pull_result) => { reuse_warnings.append(&mut pull_result.reuse_warnings); let verification_result = self .clear_quarantine_entry(&item.url) .and_then(|_| { self.record_download_manifest_entry( &mut manifest, &item.url, &item.destination, ) }) .and_then(|verification| { self.write_download_manifest(&manifest) .map(|_| verification) }); match verification_result { Ok(verification) => { pull_result.verification = verification; } Err(error) => { let error = PullOneError::plain(format!("记录下载 manifest 失败：{error}")); self.record_quarantine_entry(&item.url, &item.destination, &error)?; progress(OfficialResourcePullProgress::failed( completed_count, total, item.url.clone(), &error, )); return Err(DownloadError::new( error.error_code(), format!( "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", item.url, self.download_quarantine_path().display(), error.message ), )); } } pull_result } Err(error) => { self.record_quarantine_entry(&item.url, &item.destination, &error)?; progress(OfficialResourcePullProgress::failed( completed_count, total, item.url.clone(), &error, )); return Err(DownloadError::new( error.error_code(), format!( "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", item.url, self.download_quarantine_path().display(), error.message ), )); } } }; completed_count += 1; progress(OfficialResourcePullProgress::finished( completed_count, total, item.url.clone(), result.status, result.bytes, result.transferred_bytes, result.verification.clone(), )); processed_urls.insert(item.url.clone()); let newly_verified_hashes = self.verify_ready_official_hashes( &official_hash_pairs, &processed_urls, &mut verified_hash_urls, &mut verified_hashes, &mut manifest, )?; for verification in newly_verified_hashes { progress(OfficialResourcePullProgress::verification( completed_count, total, verification.data_url.clone(), verification, )); } items.push(OfficialResourcePullItem { url: item.url.clone(), destination: item.destination.clone(), bytes: result.bytes, transferred_bytes: result.transferred_bytes, status: result.status, cas_object_id: result.cas_object_id.clone(), }); } self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?; let report = OfficialResourcePullReport { items, verified_hashes, reuse_warnings, }; self.persist_cas_reuse_references(&report)?; cas_reference_guard.commit(); Ok(report) } fn pull_planned_concurrently( &self, planned: Vec, mut manifest: OfficialDownloadManifest, official_hash_pairs: Vec, mut reuse_warnings: Vec, progress: &mut impl FnMut(OfficialResourcePullProgress), should_cancel: &mut impl FnMut() -> bool, ) -> Result { let total = planned.len(); if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string().into()); } for item in &planned { progress(OfficialResourcePullProgress::started( 0, total, item.url.clone(), )); } let backend = CurlDownloadBackend { service: self }; let mut completed_count = 0usize; let mut items_by_plan_index: Vec> = (0..total).map(|_| None).collect(); let mut verified_hashes = Vec::new(); let mut reuse_warnings = reuse_warnings .drain(..) .map(|message| OfficialResourceReuseWarning { url: String::new(), source: "historical_release_index".to_string(), message, }) .collect::>(); let mut verified_hash_urls = HashSet::::new(); let mut processed_urls = HashSet::::new(); DownloadScheduler::new(self.max_concurrency).execute_with_observer( &backend, planned.clone(), |plan_index, result| { let item = &planned[plan_index]; if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string().into()); } let needs_manifest = item.existing.is_none(); let verification = match result { Ok(pull_result) => { reuse_warnings.extend(pull_result.reuse_warnings.clone()); let verification_result = self .clear_quarantine_entry(&item.url) .and_then(|_| { if needs_manifest { self.record_download_manifest_entry( &mut manifest, &item.url, &item.destination, ) } else { Ok(pull_result.verification.clone()) } }) .and_then(|verification| { if needs_manifest { self.write_download_manifest(&manifest) .map(|_| verification) } else { Ok(verification) } }); match verification_result { Ok(verification) => verification, Err(error) => { let error = PullOneError::plain(format!( "记录下载 manifest 失败：{error}" )); self.record_quarantine_entry( &item.url, &item.destination, &error, )?; progress(OfficialResourcePullProgress::failed( completed_count, total, item.url.clone(), &error, )); return Err(DownloadError::new( error.error_code(), format!( "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", item.url, self.download_quarantine_path().display(), error.message ), )); } } } Err(error) => { self.record_quarantine_entry(&item.url, &item.destination, error)?; progress(OfficialResourcePullProgress::failed( completed_count, total, item.url.clone(), error, )); return Err(DownloadError::new( error.error_code(), format!( "官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}", item.url, self.download_quarantine_path().display(), error.message ), )); } }; completed_count += 1; progress(OfficialResourcePullProgress::finished( completed_count, total, item.url.clone(), result .as_ref() .expect("successful result handled above") .status, result .as_ref() .expect("successful result handled above") .bytes, result .as_ref() .expect("successful result handled above") .transferred_bytes, verification, )); processed_urls.insert(item.url.clone()); let newly_verified_hashes = self.verify_ready_official_hashes( &official_hash_pairs, &processed_urls, &mut verified_hash_urls, &mut verified_hashes, &mut manifest, )?; for verification in newly_verified_hashes { progress(OfficialResourcePullProgress::verification( completed_count, total, verification.data_url.clone(), verification, )); } let pull_result = result .as_ref() .expect("successful result handled above"); items_by_plan_index[plan_index] = Some(OfficialResourcePullItem { url: item.url.clone(), destination: item.destination.clone(), bytes: pull_result.bytes, transferred_bytes: pull_result.transferred_bytes, status: pull_result.status, cas_object_id: pull_result.cas_object_id.clone(), }); Ok(()) }, )?; self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?; let items = items_by_plan_index .into_iter() .enumerate() .map(|(index, item)| { item.ok_or_else(|| { DownloadError::from(format!( "并发下载结果缺少 plan index={index}，拒绝发布不完整资源" )) }) }) .collect::, _>>()?; Ok(OfficialResourcePullReport { items, verified_hashes, reuse_warnings, }) } /// Audits every URL in a pull plan against the local download manifest. /// /// This performs no network I/O. It checks that each URL has a manifest /// entry, that the entry maps to the URL-derived destination, and that the /// local file still matches the recorded size and BLAKE3 digest. pub fn audit_local_manifest( &self, plan: &OfficialResourcePullPlan, ) -> Result { let manifest = self.read_download_manifest()?; let mut items = Vec::new(); for url in plan.all_urls()? { let destination = self.destination_for_url(&url)?; items.push(self.audit_one(&url, destination, &manifest)?); } Ok(OfficialLocalManifestAuditReport { items }) } /// Verifies every entry currently present in the local download manifest. /// /// This is intentionally network-free. It validates each recorded file /// using the local manifest's size and BLAKE3 digest, then verifies every /// local seed `.bytes`/`.hash` pair that is present using the official /// decimal xxHash32 rule. pub fn verify_local_download_manifest( &self, ) -> Result { let manifest = self.read_download_manifest()?; let urls = manifest.entries.keys().cloned().collect::>(); let mut items = Vec::with_capacity(urls.len()); for url in urls { let destination = self.destination_for_url(&url)?; items.push(self.audit_one(&url, destination, &manifest)?); } let pairs = local_manifest_seed_hash_pairs(&manifest); let mut official_hash_verified_count = 0; let mut official_hash_errors = Vec::new(); for pair in &pairs { match self.verify_official_seed_hash_pair_from_disk(pair) { Ok(_) => official_hash_verified_count += 1, Err(error) => official_hash_errors.push(error), } } Ok(OfficialLocalVerificationReport { items, official_hash_pair_count: pairs.len(), official_hash_verified_count, official_hash_errors, }) } /// Returns whether the current output root already contains local state for /// this plan. This is intentionally cheaper than a full audit and is used /// to choose between audit-then-repair and first-time full pull flows. pub fn local_resource_state( &self, plan: &OfficialResourcePullPlan, ) -> Result { let manifest = self.read_download_manifest()?; let mut manifest_entry_count = 0; let mut existing_file_count = 0; for url in plan.all_urls()? { if manifest.entries.contains_key(&url) { manifest_entry_count += 1; } let destination = self.destination_for_url(&url)?; if file_len_if_exists(&destination)?.is_some() { existing_file_count += 1; } } Ok(OfficialLocalResourceState { manifest_entry_count, existing_file_count, }) } fn build_release_reuse_index(&self) -> Result { let mut index = ReleaseReuseIndex::default(); let mut roots = Vec::new(); for configured_root in &self.release_reuse_roots { let metadata = match fs::symlink_metadata(configured_root) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { index.warnings.push(format!( "读取历史 release 根失败 {}：{error}", configured_root.display() )); continue; } }; if metadata.file_type().is_symlink() { index.warnings.push(format!( "历史 release 根不能是 symlink：{}", configured_root.display() )); continue; } if !metadata.is_dir() { index.warnings.push(format!( "历史 release 根不是目录：{}", configured_root.display() )); continue; } if let Err(error) = ensure_safe_directory_path(configured_root, "历史 release 根目录") { index.warnings.push(error); continue; } let manifest_path = configured_root.join(DOWNLOAD_MANIFEST_FILE); if fs::symlink_metadata(&manifest_path) .map(|metadata| !metadata.file_type().is_symlink() && metadata.is_file()) .unwrap_or(false) { roots.push(configured_root.clone()); continue; } let entries = match fs::read_dir(configured_root) { Ok(entries) => entries, Err(error) => { index.warnings.push(format!( "读取历史 release 目录失败 {}：{error}", configured_root.display() )); continue; } }; let mut child_roots = entries .filter_map(|entry| entry.ok().map(|entry| entry.path())) .filter(|path| { fs::symlink_metadata(path) .map(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink()) .unwrap_or(false) }) .collect::>(); child_roots.sort(); child_roots.reverse(); roots.extend(child_roots); } for root in roots { let manifest = match read_download_manifest_at(&root) { Ok(Some(manifest)) => manifest, Ok(None) => continue, Err(error) => { index.warnings.push(format!( "忽略无效历史 release manifest {}：{error}", root.display() )); continue; } }; for entry in manifest.entries.values() { if !is_valid_blake3_hex(&entry.blake3) { index.warnings.push(format!( "忽略没有可靠 BLAKE3 的历史条目：release={} destination={}", root.display(), entry.destination )); continue; } let source = root.join(Path::new(&entry.destination)); if ensure_path_within_root(&root, &source).is_err() { index.warnings.push(format!( "忽略逃逸历史条目：release={} destination={}", root.display(), entry.destination )); continue; } let candidate = HistoricalReuseCandidate { root: root.clone(), source, destination: entry.destination.clone(), bytes: entry.bytes, blake3: entry.blake3.clone(), }; index .candidates .entry(reuse_key(&entry.destination)) .or_default() .push(candidate); } } Ok(index) } fn reuse_candidate_for_url( &self, _url: &str, destination: &Path, index: &ReleaseReuseIndex, ) -> Result, String> { let relative_destination = self.relative_destination(destination)?; Ok(index .candidates .get(&reuse_key(&relative_destination)) .filter(|candidates| !candidates.is_empty()) .cloned() .map(|candidates| ReuseCandidate { candidates })) } fn pull_task(&self, task: PlannedDownload) -> Result { if let Some(existing) = task.existing { return Ok(existing); } let mut reuse_warnings = Vec::new(); if let Some(candidate) = task.reuse_candidate.as_ref() { let (reused, warnings) = self.try_reuse_candidate(candidate, &task.url, &task.destination); reuse_warnings.extend(warnings); if let Some(mut result) = reused { result.reuse_warnings = reuse_warnings; return Ok(result); } } let mut result = self.pull_one(&task.url, &task.destination)?; result.reuse_warnings = reuse_warnings; Ok(result) } fn try_reuse_candidate( &self, candidate: &ReuseCandidate, url: &str, destination: &Path, ) -> (Option, Vec) { let mut warnings = Vec::new(); for historical in &candidate.candidates { match self.materialize_historical_candidate(historical, url, destination) { Ok(result) => return (Some(result), warnings), Err(error) => warnings.push(OfficialResourceReuseWarning { url: url.to_string(), source: "historical_release".to_string(), message: format!( "release={} destination={}：{error}", historical.root.display(), historical.destination ), }), } } if self.cas_reuse_root.is_some() { for historical in &candidate.candidates { match self.materialize_cas_candidate(historical, url, destination) { Ok(result) => return (Some(result), warnings), Err(error) => warnings.push(OfficialResourceReuseWarning { url: url.to_string(), source: "cas".to_string(), message: format!( "object={} destination={}：{error}", historical.blake3, historical.destination ), }), } } } (None, warnings) } fn materialize_historical_candidate( &self, candidate: &HistoricalReuseCandidate, url: &str, destination: &Path, ) -> Result { ensure_safe_directory_path(&candidate.root, "历史 release 根目录")?; ensure_safe_file_target(&candidate.root, &candidate.source, "历史 release 复用源")?; verify_file_attributes(&candidate.source, candidate.bytes, &candidate.blake3, url)?; self.materialize_verified_path( &candidate.source, url, destination, ReuseMaterialization { expected_bytes: candidate.bytes, expected_blake3: candidate.blake3.clone(), status: OfficialResourcePullStatus::ReleaseReused, cas_object_id: None, }, ) } fn materialize_cas_candidate( &self, candidate: &HistoricalReuseCandidate, url: &str, destination: &Path, ) -> Result { let cas_root = self .cas_reuse_root .as_ref() .ok_or_else(|| "未配置 CAS 根目录".to_string())?; let objects_root = cas_root.join("objects"); let metadata_path = cas_root.join("metadata.sqlite"); require_existing_directory(cas_root, "CAS 根目录")?; require_existing_directory(&objects_root, "CAS 对象目录")?; require_existing_file(cas_root, &metadata_path, "CAS 元数据库")?; let object_path = cas_object_path(cas_root, &candidate.blake3)?; ensure_safe_file_target(&objects_root, &object_path, "CAS 对象")?; let object_id = candidate.blake3.clone(); let cas_root_for_runtime = cas_root.clone(); let object_id_for_runtime = object_id.clone(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|error| format!("创建 CAS 校验 runtime 失败：{error}"))?; runtime .block_on(async move { let cas = crate::FileSystemCasRepository::new(cas_root_for_runtime); cas.get(&object_id_for_runtime) .await .map_err(|error| format!("CAS 对象完整性校验失败：{error}"))?; cas.get_reference_count(&object_id_for_runtime) .await .map_err(|error| format!("CAS 对象元数据不一致：{error}"))?; cas.add_reference(&object_id_for_runtime) .await .map_err(|error| format!("增加 CAS release 引用失败：{error}")) }) .map_err(|error| error.to_string())?; self.track_cas_reference(&object_id); match self.materialize_verified_path( &object_path, url, destination, ReuseMaterialization { expected_bytes: candidate.bytes, expected_blake3: candidate.blake3.clone(), status: OfficialResourcePullStatus::CasReused, cas_object_id: Some(object_id.clone()), }, ) { Ok(result) => Ok(result), Err(error) => { let cas_root_for_runtime = cas_root.clone(); let object_id_for_runtime = object_id.clone(); let rollback = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|runtime_error| { format!("创建 CAS 回滚 runtime 失败：{runtime_error}") })? .block_on(async move { let cas = crate::FileSystemCasRepository::new(cas_root_for_runtime); cas.remove_reference(&object_id_for_runtime) .await .map_err(|rollback_error| rollback_error.to_string()) }); if let Err(rollback_error) = rollback { return Err(format!("{error}；回滚 CAS 引用失败：{rollback_error}")); } self.untrack_cas_reference(&object_id); Err(error) } } } fn materialize_verified_path( &self, source: &Path, url: &str, destination: &Path, materialization: ReuseMaterialization, ) -> Result { ensure_safe_file_target(&self.output_root, destination, "复用目标文件")?; if source == destination { let verification = self.local_verification( url, destination, Some(materialization.expected_bytes), Some(materialization.expected_blake3), )?; return Ok(PullOneResult { bytes: verification.actual_bytes, transferred_bytes: 0, status: materialization.status, verification, cas_object_id: materialization.cas_object_id, reuse_warnings: Vec::new(), }); } let temporary = reuse_temporary_path(destination); ensure_safe_file_target(&self.output_root, &temporary, "复用临时文件")?; let result = (|| { if let Err(hard_link_error) = fs::hard_link(source, &temporary) { fs::copy(source, &temporary).map_err(|copy_error| { format!("硬链接复用失败：{hard_link_error}；跨文件系统复制也失败：{copy_error}") })?; } let verification = self.local_verification( url, &temporary, Some(materialization.expected_bytes), Some(materialization.expected_blake3), )?; ensure_safe_file_target(&self.output_root, destination, "复用目标文件")?; fs::rename(&temporary, destination).map_err(|error| { format!( "原子移动复用文件失败 {} -> {}：{error}", temporary.display(), destination.display() ) })?; Ok(PullOneResult { bytes: verification.actual_bytes, transferred_bytes: 0, status: materialization.status, verification, cas_object_id: materialization.cas_object_id, reuse_warnings: Vec::new(), }) })(); if result.is_err() { let _ = fs::remove_file(&temporary); } result } /// Fetches an official URL into memory. /// /// This is intended for small discovery inputs such as `server-info`, /// `TableCatalog.bytes`, `BundlePackingInfo.bytes`, and /// `MediaCatalog.bytes`. pub fn fetch_bytes(&self, url: &str) -> Result, DownloadError> { if !self.backend.is_official_url(url) { return Err(DownloadError::new( bat_core::ErrorCode::NON_OFFICIAL_URL, format!("拒绝拉取非官方 URL：{url}"), )); } let output = run_curl_with_retry_with_proxy( &self.curl_command, url, None, self.retry_attempts, &self.curl_proxy, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") .arg("--location") .arg("--silent") .arg("--show-error") .arg("--url") .arg(url); command }, ) .map_err(|error| { // 保留 curl 失败的准确网络域码；进程类失败归 INTERNAL。 let code = error .final_error_code() .unwrap_or(bat_core::ErrorCode::INTERNAL); DownloadError::new(code, format!("curl 拉取失败 {url}：{error}")) })?; Ok(output.stdout) } fn pull_one(&self, url: &str, destination: &Path) -> Result { ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; let partial = partial_path_for(destination); ensure_safe_file_target(&self.output_root, &partial, "临时下载文件") .map_err(PullOneError::plain)?; let partial_len_before = file_len_if_exists(&partial) .map_err(PullOneError::plain)? .unwrap_or_default(); let resumed = partial_len_before > 0; let mut status = if resumed { OfficialResourcePullStatus::Resumed } else { OfficialResourcePullStatus::Downloaded }; if resumed { if let Err(error) = self.download_one(url, &partial, true) { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { PullOneError::from_retry( format!("续传失败 {url}：{error}；清理后重新下载也失败：{retry_error}"), retry_error, ) })?; status = OfficialResourcePullStatus::Downloaded; } else if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { PullOneError::from_retry( format!("续传后的 ZIP 结构校验失败 {url}：{error}；清理后重新下载也失败：{retry_error}"), retry_error, ) })?; status = OfficialResourcePullStatus::Downloaded; } } else { self.download_one(url, &partial, false).map_err(|error| { PullOneError::from_retry(format!("下载失败 {url}：{error}"), error) })?; } if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); return Err(PullOneError::plain(error)); } ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; if file_len_if_exists(destination) .map_err(PullOneError::plain)? .is_some() { fs::remove_file(destination) .map_err(|error| format!("替换已有目标文件失败 {}：{error}", destination.display())) .map_err(PullOneError::plain)?; } ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; fs::rename(&partial, destination) .map_err(|error| { format!( "移动临时下载文件失败 {} -> {}：{error}", partial.display(), destination.display() ) }) .map_err(PullOneError::plain)?; let bytes = file_len_if_exists(destination) .map_err(PullOneError::plain)? .ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display())) .map_err(PullOneError::plain)?; let verification = self .local_verification(url, destination, None, None) .map_err(PullOneError::plain)?; Ok(PullOneResult { bytes, transferred_bytes: if status == OfficialResourcePullStatus::Resumed { bytes.saturating_sub(partial_len_before) } else { bytes }, status, verification, cas_object_id: None, reuse_warnings: Vec::new(), }) } fn download_one( &self, url: &str, destination: &Path, resume: bool, ) -> Result<(), CurlRetryError> { run_curl_with_retry_with_proxy( &self.curl_command, url, Some(destination), self.retry_attempts, &self.curl_proxy, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") .arg("--location") .arg("--silent") .arg("--show-error") .arg("--output") .arg(destination); if resume { command.arg("--continue-at").arg("-"); } command.arg("--url").arg(url); command }, ) .map(|_| ()) } fn verify_ready_official_hashes( &self, pairs: &[OfficialSeedHashPair], processed_urls: &HashSet, verified_hash_urls: &mut HashSet, verified_hashes: &mut Vec, manifest: &mut OfficialDownloadManifest, ) -> Result, String> { let mut newly_verified = Vec::new(); for pair in pairs { if verified_hash_urls.contains(&pair.hash_url) { continue; } if !processed_urls.contains(&pair.data_url) || !processed_urls.contains(&pair.hash_url) { continue; } if !self.hash_pair_files_exist(pair)? { continue; } let verification = match self.verify_official_seed_hash_pair_from_disk(pair) { Ok(verification) => verification, Err(error) => { if let Err(cleanup_error) = self.invalidate_download_manifest_hash_pair(manifest, pair) { return Err(format!( "{error}；同时清理官方 hash 对应的本地 manifest 条目失败：{cleanup_error}" )); } return Err(error); } }; newly_verified.push(verification.clone()); verified_hashes.push(verification); verified_hash_urls.insert(pair.hash_url.clone()); } Ok(newly_verified) } fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result { let data_path = self.destination_for_url(&pair.data_url)?; let hash_path = self.destination_for_url(&pair.hash_url)?; Ok(file_len_if_exists(&data_path)?.is_some() && file_len_if_exists(&hash_path)?.is_some()) } fn verify_official_seed_hash_pair_from_disk( &self, pair: &OfficialSeedHashPair, ) -> Result { let data_path = self.destination_for_url(&pair.data_url)?; let hash_path = self.destination_for_url(&pair.hash_url)?; let data = fs::read(&data_path).map_err(|error| { format!( "读取官方 hash 对应数据文件失败 {}：{error}", data_path.display() ) })?; let hash = fs::read(&hash_path).map_err(|error| { format!( "读取官方 hash sidecar 失败 {}：{error}", hash_path.display() ) })?; verify_official_seed_catalog_hash(&pair.data_url, &data, &pair.hash_url, &hash) } fn invalidate_download_manifest_hash_pair( &self, manifest: &mut OfficialDownloadManifest, pair: &OfficialSeedHashPair, ) -> Result<(), String> { manifest.entries.remove(&pair.data_url); manifest.entries.remove(&pair.hash_url); self.write_download_manifest(manifest) } fn verify_all_official_hashes_are_complete( &self, pairs: &[OfficialSeedHashPair], verified_hash_urls: &HashSet, ) -> Result<(), String> { for pair in pairs { if !verified_hash_urls.contains(&pair.hash_url) { return Err(format!( "官方 hash 对未完成校验：数据={} hash={}", pair.data_url, pair.hash_url )); } } Ok(()) } fn read_download_manifest(&self) -> Result { self.ensure_output_root_safe()?; let path = self.download_manifest_path(); ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?; let Some(bytes) = read_file_no_symlink(&path, "下载 manifest")? else { return Ok(OfficialDownloadManifest::default()); }; let manifest: OfficialDownloadManifest = serde_json::from_slice(&bytes) .map_err(|error| format!("解析下载 manifest 失败 {}：{error}", path.display()))?; if manifest.version != DOWNLOAD_MANIFEST_VERSION { return Err(format!( "不支持的下载 manifest 版本 {}，文件 {}", manifest.version, path.display() )); } Ok(manifest) } fn remove_managed_destination(&self, path: &Path, url: &str) -> Result<(), String> { ensure_path_within_root(&self.output_root, path)?; ensure_safe_file_target(&self.output_root, path, "旧官方资源清理目标")?; match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => Err(format!( "旧官方资源清理目标不能是 symlink：url={url} path={}", path.display() )), Ok(metadata) if metadata.is_file() => { fs::remove_file(path).map_err(|error| { format!( "清理旧官方资源失败：url={url} path={} error={error}", path.display() ) })?; remove_empty_parent_directories(&self.output_root, path.parent())?; Ok(()) } Ok(metadata) if metadata.is_dir() => Err(format!( "旧官方资源清理目标不能是目录：url={url} path={}", path.display() )), Ok(_) => Err(format!( "旧官方资源清理目标不是普通文件：url={url} path={}", path.display() )), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), Err(error) => Err(format!( "检查旧官方资源清理目标失败：url={url} path={} error={error}", path.display() )), } } fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> { self.ensure_output_root_ready()?; let path = self.download_manifest_path(); if let Some(parent) = path.parent() { ensure_safe_directory_path(parent, "下载 manifest 目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载 manifest 目录失败 {}：{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载 manifest 目录")?; } ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?; let mut persisted_manifest = manifest.clone(); persisted_manifest.distribution_mapping_identity = Some(official_distribution_mapping_identity(&persisted_manifest)); persisted_manifest.destination_index = official_distribution_destination_index(&persisted_manifest)?; let bytes = serde_json::to_vec_pretty(&persisted_manifest) .map_err(|error| format!("序列化下载 manifest 失败 {}：{error}", path.display()))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 manifest")?; Ok(()) } fn read_download_quarantine(&self) -> Result { self.ensure_output_root_safe()?; let path = self.download_quarantine_path(); ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; let Some(bytes) = read_file_no_symlink(&path, "下载 quarantine")? else { return Ok(OfficialDownloadQuarantineManifest::default()); }; let manifest: OfficialDownloadQuarantineManifest = serde_json::from_slice(&bytes) .map_err(|error| format!("解析下载 quarantine 失败 {}：{error}", path.display()))?; if manifest.version != DOWNLOAD_QUARANTINE_VERSION { return Err(format!( "不支持的下载 quarantine 版本 {}，文件 {}", manifest.version, path.display() )); } Ok(manifest) } fn write_download_quarantine( &self, manifest: &OfficialDownloadQuarantineManifest, ) -> Result<(), String> { self.ensure_output_root_ready()?; let path = self.download_quarantine_path(); if let Some(parent) = path.parent() { ensure_safe_directory_path(parent, "下载 quarantine 目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载 quarantine 目录失败 {}：{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载 quarantine 目录")?; } ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| format!("序列化下载 quarantine 失败 {}：{error}", path.display()))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 quarantine")?; Ok(()) } fn record_quarantine_entry( &self, url: &str, destination: &Path, error: &PullOneError, ) -> Result<(), String> { let mut manifest = self.read_download_quarantine()?; manifest.entries.insert( url.to_string(), OfficialDownloadQuarantineEntry { url: url.to_string(), destination: self.relative_destination(destination)?, failed_at_unix_seconds: unix_seconds_now(), attempts: error.attempts().unwrap_or(0), failure_kind: error.failure_kind().map(ToOwned::to_owned), http_status: error.http_status(), retryable: error.retryable(), last_error: error.message.clone(), }, ); self.write_download_quarantine(&manifest) } fn clear_quarantine_entry(&self, url: &str) -> Result<(), String> { let mut manifest = self.read_download_quarantine()?; if manifest.entries.remove(url).is_none() { return Ok(()); } self.write_download_quarantine(&manifest) } fn validated_existing_file( &self, url: &str, destination: &Path, manifest: &OfficialDownloadManifest, ) -> Result, String> { let Some(entry) = manifest.entries.get(url) else { return Ok(None); }; if entry.url != url { return Ok(None); } if entry.destination != self.relative_destination(destination)? { return Ok(None); } let Some(bytes) = file_len_if_exists(destination)? else { return Ok(None); }; if bytes != entry.bytes { return Ok(None); } let digest = blake3_file_hex(destination)?; if digest != entry.blake3 { return Ok(None); } if let Err(_error) = self.validate_zip_if_needed(url, destination) { return Ok(None); } let verification = self.local_verification( url, destination, Some(entry.bytes), Some(entry.blake3.clone()), )?; Ok(Some(PullOneResult { bytes, transferred_bytes: 0, status: OfficialResourcePullStatus::SkippedExisting, verification, cas_object_id: None, reuse_warnings: Vec::new(), })) } fn local_verification( &self, url: &str, destination: &Path, expected_bytes: Option, expected_blake3: Option, ) -> Result { let actual_bytes = file_len_if_exists(destination)? .ok_or_else(|| format!("读取资源文件信息失败 {}", destination.display()))?; let actual_blake3 = blake3_file_hex(destination)?; let zip_checked = url_or_path_has_zip_extension(url) || path_has_zip_extension(destination); if zip_checked { self.validate_zip_if_needed(url, destination)?; } Ok(OfficialResourceVerification { expected_bytes, actual_bytes, expected_blake3, actual_blake3, zip_checked, zip_structure_verified: zip_checked, }) } fn record_download_manifest_entry( &self, manifest: &mut OfficialDownloadManifest, url: &str, destination: &Path, ) -> Result { let mut verification = self.local_verification(url, destination, None, None)?; let relative_destination = self.relative_destination(destination)?; manifest.entries.insert( url.to_string(), OfficialDownloadManifestEntry { url: url.to_string(), destination: relative_destination, bytes: verification.actual_bytes, blake3: verification.actual_blake3.clone(), }, ); verification.expected_bytes = Some(verification.actual_bytes); verification.expected_blake3 = Some(verification.actual_blake3.clone()); Ok(verification) } fn validate_zip_if_needed(&self, url: &str, destination: &Path) -> Result<(), String> { ensure_safe_file_target(&self.output_root, destination, "ZIP 校验目标")?; if !url_or_path_has_zip_extension(url) && !path_has_zip_extension(destination) { return Ok(()); } validate_zip_structure(destination).map(|_report| ()) } fn relative_destination(&self, destination: &Path) -> Result { let relative = destination .strip_prefix(&self.output_root) .map_err(|error| { format!( "目标路径 {} 不在输出目录 {} 下：{error}", destination.display(), self.output_root.display() ) })?; Ok(relative.to_string_lossy().replace('\\', "/")) } fn audit_one( &self, url: &str, destination: PathBuf, manifest: &OfficialDownloadManifest, ) -> Result { let Some(entry) = manifest.entries.get(url) else { let actual_bytes = file_len_if_exists(&destination)?; return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::MissingManifestEntry, manifest_destination: None, expected_bytes: None, actual_bytes, expected_blake3: None, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); }; let manifest_destination = Some(entry.destination.clone()); let expected_bytes = Some(entry.bytes); let expected_blake3 = Some(entry.blake3.clone()); if entry.url != url { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::UrlMismatch, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let expected_destination = self.relative_destination(&destination)?; if entry.destination != expected_destination { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::DestinationMismatch, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let Some(actual_bytes) = file_len_if_exists(&destination)? else { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::MissingFile, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); }; if actual_bytes != entry.bytes { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::SizeMismatch, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let actual_blake3 = blake3_file_hex(&destination)?; if actual_blake3 != entry.blake3 { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::Blake3Mismatch, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: None, zip_structure_verified: false, }); } let zip_structure_required = url_or_path_has_zip_extension(url) || path_has_zip_extension(&destination); if let Err(error) = self.validate_zip_if_needed(url, &destination) { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::ZipStructureInvalid, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: Some(error), zip_structure_verified: false, }); } Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::Verified, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: None, zip_structure_verified: zip_structure_required, }) } fn destination_for_url(&self, url: &str) -> Result { self.ensure_output_root_safe()?; if !self.backend.is_official_url(url) { return Err(format!("URL 不是官方 JP host：{url}")); } let relative_destination = self.backend.relative_destination(url)?; let destination = destination_under_root(&self.output_root, &relative_destination)?; ensure_path_within_root(&self.output_root, &destination)?; Ok(destination) } fn ensure_output_root_safe(&self) -> Result<(), String> { validate_output_root(&self.output_root) } fn ensure_output_root_ready(&self) -> Result<(), String> { self.ensure_output_root_safe()?; ensure_safe_directory_path(&self.output_root, "资源输出目录")?; fs::create_dir_all(&self.output_root).map_err(|error| { format!( "创建资源输出目录失败 {}：{error}", self.output_root.display() ) })?; ensure_safe_directory_path(&self.output_root, "资源输出目录") } } /// Verifies an official seed catalog `.bytes` payload against its official /// `.hash` sidecar. /// /// The currently verified seed catalogs are `TableCatalog.bytes`, /// `BundlePackingInfo.bytes`, and `MediaCatalog.bytes`; their `.hash` files are /// decimal text `xxHash32` values with seed `0`. pub fn verify_official_seed_catalog_hash( data_url: &str, data: &[u8], hash_url: &str, hash_bytes: &[u8], ) -> Result { let strategy = XxHash32DecimalSeedZero; let verification = strategy .verify(data, hash_bytes) .map_err(|error| format!("{error}：{hash_url}"))?; if verification.expected != verification.actual { return Err(format!( "官方 hash 校验失败 {data_url}：期望 {}（来自 {hash_url}），实际 {}", verification.expected, verification.actual )); } Ok(OfficialResourceHashVerification { data_url: data_url.to_string(), hash_url: hash_url.to_string(), algorithm: OfficialResourceHashAlgorithm::XxHash32Decimal, expected: verification.expected, actual: verification.actual, }) } #[derive(Debug, Clone, PartialEq, Eq)] struct OfficialSeedHashPair { data_url: String, hash_url: String, } fn official_seed_hash_pairs(plan: &OfficialResourcePullPlan) -> Vec { plan.discovery .endpoints .iter() .filter_map(|endpoint| { let hash_kind = strong_seed_hash_kind(endpoint.kind)?; let hash_endpoint = plan.discovery.endpoints.iter().find(|candidate| { candidate.kind == hash_kind && candidate.platform == endpoint.platform })?; Some(OfficialSeedHashPair { data_url: endpoint.url.clone(), hash_url: hash_endpoint.url.clone(), }) }) .collect() } fn local_manifest_seed_hash_pairs( manifest: &OfficialDownloadManifest, ) -> Vec { manifest .entries .keys() .filter_map(|data_url| { let hash_url = data_url .strip_suffix(".bytes") .map(|prefix| format!("{prefix}.hash"))?; if !manifest.entries.contains_key(&hash_url) { return None; } Some(OfficialSeedHashPair { data_url: data_url.clone(), hash_url, }) }) .collect() } fn official_hash_refresh_urls(pairs: &[OfficialSeedHashPair]) -> HashSet { let mut urls = HashSet::new(); for pair in pairs { urls.insert(pair.data_url.clone()); urls.insert(pair.hash_url.clone()); } urls } fn strong_seed_hash_kind( kind: YostarJpResourceEndpointKind, ) -> Option { match kind { YostarJpResourceEndpointKind::TableCatalog => { Some(YostarJpResourceEndpointKind::TableCatalogHash) } YostarJpResourceEndpointKind::BundlePackingInfo => { Some(YostarJpResourceEndpointKind::BundlePackingInfoHash) } YostarJpResourceEndpointKind::MediaCatalog => { Some(YostarJpResourceEndpointKind::MediaCatalogHash) } YostarJpResourceEndpointKind::AddressablesCatalog | YostarJpResourceEndpointKind::TableCatalogHash | YostarJpResourceEndpointKind::AddressablesCatalogHash | YostarJpResourceEndpointKind::BundlePackingInfoHash | YostarJpResourceEndpointKind::MediaCatalogHash => None, } } /// 官方下载 manifest：按完整 URL 为键记录每个已下载资源的本地校验信息。 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialDownloadManifest { /// Manifest 结构版本。 #[serde(default = "default_download_manifest_version")] pub version: u32, /// 按 URL 为键的资源条目。 #[serde(default)] pub entries: BTreeMap, /// Persisted destination-to-URL index for single-entry distribution lookup. #[serde(default)] pub destination_index: BTreeMap, /// Deterministic identity of the complete distribution mapping. #[serde(default, skip_serializing_if = "Option::is_none")] pub distribution_mapping_identity: Option, } impl Default for OfficialDownloadManifest { fn default() -> Self { Self { version: DOWNLOAD_MANIFEST_VERSION, entries: BTreeMap::new(), destination_index: BTreeMap::new(), distribution_mapping_identity: None, } } } /// 官方下载 manifest 中的单个资源条目。 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialDownloadManifestEntry { /// 官方资源 URL。 pub url: String, /// 相对资源根目录的落盘路径。 pub destination: String, /// 文件字节数。 pub bytes: u64, /// 文件 BLAKE3 摘要（hex）。 pub blake3: String, } /// 读取指定资源根目录下的官方下载 manifest（daemon RPC 只读查询用）。 /// /// 文件缺失返回 `Ok(None)`；符号链接、解析失败或版本不支持返回 `Err`。 pub fn read_download_manifest_at( resource_root: &Path, ) -> Result, String> { let path = resource_root.join(DOWNLOAD_MANIFEST_FILE); let Some(bytes) = read_file_no_symlink(&path, "下载 manifest")? else { return Ok(None); }; let manifest: OfficialDownloadManifest = serde_json::from_slice(&bytes) .map_err(|error| format!("解析下载 manifest 失败 {}：{error}", path.display()))?; if manifest.version != DOWNLOAD_MANIFEST_VERSION { return Err(format!( "不支持的下载 manifest 版本 {}，文件 {}", manifest.version, path.display() )); } Ok(Some(manifest)) } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadQuarantineManifest { #[serde(default = "default_download_quarantine_version")] version: u32, #[serde(default)] entries: BTreeMap, } impl Default for OfficialDownloadQuarantineManifest { fn default() -> Self { Self { version: DOWNLOAD_QUARANTINE_VERSION, entries: BTreeMap::new(), } } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadQuarantineEntry { url: String, destination: String, failed_at_unix_seconds: u64, attempts: usize, failure_kind: Option, http_status: Option, retryable: Option, last_error: String, } fn default_download_manifest_version() -> u32 { DOWNLOAD_MANIFEST_VERSION } fn default_download_quarantine_version() -> u32 { DOWNLOAD_QUARANTINE_VERSION } #[derive(Debug, Clone, Default)] struct ReleaseReuseIndex { candidates: HashMap>, warnings: Vec, } #[derive(Debug, Clone)] struct HistoricalReuseCandidate { root: PathBuf, source: PathBuf, destination: String, bytes: u64, blake3: String, } #[derive(Debug, Clone)] struct ReuseCandidate { candidates: Vec, } struct CasReferenceRollbackGuard { cas_root: Option, tracker: Arc>>, committed: bool, } impl CasReferenceRollbackGuard { fn new(cas_root: Option, tracker: Arc>>) -> Self { Self { cas_root, tracker, committed: false, } } fn commit(mut self) { self.committed = true; if let Ok(mut references) = self.tracker.lock() { references.clear(); } } } impl Drop for CasReferenceRollbackGuard { fn drop(&mut self) { if self.committed { return; } let Some(cas_root) = self.cas_root.clone() else { return; }; let object_ids = self .tracker .lock() .map(|mut references| std::mem::take(&mut *references)) .unwrap_or_default(); if object_ids.is_empty() { return; } let Ok(runtime) = tokio::runtime::Builder::new_current_thread() .enable_all() .build() else { return; }; runtime.block_on(async move { let cas = crate::FileSystemCasRepository::new(cas_root); for object_id in object_ids { let _ = cas.remove_reference(&object_id).await; } }); } } /// Phase A 产出的单个下载计划项：URL、目标路径，以及若命中本地 manifest /// 校验则带上「已验证可跳过」的结果（`existing`）。 #[derive(Debug, Clone)] struct PlannedDownload { url: String, destination: PathBuf, existing: Option, reuse_candidate: Option, } #[derive(Debug, Clone, PartialEq, Eq)] struct PullOneResult { bytes: u64, transferred_bytes: u64, status: OfficialResourcePullStatus, verification: OfficialResourceVerification, cas_object_id: Option, reuse_warnings: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] struct ReuseMaterialization { expected_bytes: u64, expected_blake3: String, status: OfficialResourcePullStatus, cas_object_id: Option, } #[derive(Debug, Clone, PartialEq, Eq)] struct PullOneError { message: String, retry_error: Option, } struct CurlDownloadBackend<'a> { service: &'a OfficialResourcePullService, } impl DownloaderBackend for CurlDownloadBackend<'_> { type Output = PullOneResult; type Error = PullOneError; fn download(&self, task: PlannedDownload) -> Result { self.service.pull_task(task) } } impl PullOneError { fn plain(message: String) -> Self { Self { message, retry_error: None, } } fn from_retry(message: String, retry_error: CurlRetryError) -> Self { Self { message, retry_error: Some(retry_error), } } fn failure_kind(&self) -> Option<&'static str> { self.retry_error .as_ref() .and_then(CurlRetryError::final_kind_label) } /// 映射到统一错误码：curl 重试失败取网络域码，其余（进程/续传/替换等）归 INTERNAL。 fn error_code(&self) -> bat_core::ErrorCode { self.retry_error .as_ref() .and_then(CurlRetryError::final_error_code) .unwrap_or(bat_core::ErrorCode::INTERNAL) } fn http_status(&self) -> Option { self.retry_error .as_ref() .and_then(CurlRetryError::final_http_status) } fn retryable(&self) -> Option { self.retry_error .as_ref() .and_then(CurlRetryError::final_retryable) } fn attempts(&self) -> Option { self.retry_error.as_ref().map(CurlRetryError::attempt_count) } } impl std::fmt::Display for PullOneError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str(&self.message) } } impl std::error::Error for PullOneError {} fn file_len_if_exists(path: &Path) -> Result, String> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { Err(format!("拒绝跟随 symlink 文件：{}", path.display())) } Ok(metadata) if metadata.is_file() => Ok(Some(metadata.len())), Ok(_) => Err(format!("期望文件路径，但实际不是文件：{}", path.display())), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(format!("读取路径信息失败 {}：{error}", path.display())), } } fn require_existing_directory(path: &Path, label: &str) -> Result<(), String> { ensure_safe_directory_path(path, label)?; match fs::symlink_metadata(path) { Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => Ok(()), Ok(_) => Err(format!("{label} 不是可复用的目录：{}", path.display())), Err(error) => Err(format!( "{label} 不存在或不可读取 {}：{error}", path.display() )), } } fn require_existing_file(root: &Path, path: &Path, label: &str) -> Result<(), String> { ensure_safe_file_target(root, path, label)?; match fs::symlink_metadata(path) { Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(()), Ok(_) => Err(format!("{label} 不是可复用的普通文件：{}", path.display())), Err(error) => Err(format!( "{label} 不存在或不可读取 {}：{error}", path.display() )), } } fn verify_file_attributes( path: &Path, expected_bytes: u64, expected_blake3: &str, url: &str, ) -> Result<(), String> { let actual_bytes = file_len_if_exists(path)?.ok_or_else(|| format!("复用源文件不存在：{}", path.display()))?; if actual_bytes != expected_bytes { return Err(format!( "复用源 size 校验失败 {}：期望 {}，实际 {}", path.display(), expected_bytes, actual_bytes )); } let actual_blake3 = blake3_file_hex(path)?; if actual_blake3 != expected_blake3 { return Err(format!( "复用源 BLAKE3 校验失败 {}：期望 {}，实际 {}", path.display(), expected_blake3, actual_blake3 )); } if url_or_path_has_zip_extension(url) || path_has_zip_extension(path) { validate_zip_structure(path).map(|_| ())?; } Ok(()) } fn is_valid_blake3_hex(value: &str) -> bool { value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) } fn reuse_key(destination: &str) -> String { let normalized = destination.replace('\\', "/"); let components = normalized .split('/') .filter(|component| !component.is_empty()) .collect::>(); if components.len() <= 2 { return components.join("/"); } std::iter::once(components[0]) .chain(components.into_iter().skip(2)) .collect::>() .join("/") } fn cas_object_path(root: &Path, object_id: &str) -> Result { if !is_valid_blake3_hex(object_id) { return Err(format!("CAS 对象 ID 不是有效 BLAKE3：{object_id}")); } Ok(root .join("objects") .join(&object_id[..2]) .join(&object_id[2..4]) .join(object_id)) } fn reuse_temporary_path(destination: &Path) -> PathBuf { let nonce = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or_default(); let file_name = destination .file_name() .and_then(|name| name.to_str()) .unwrap_or("resource"); destination.with_file_name(format!(".{file_name}.reuse.{nonce}.tmp")) } fn partial_path_for(destination: &Path) -> PathBuf { let mut partial = destination.as_os_str().to_owned(); partial.push(".part"); PathBuf::from(partial) } fn remove_empty_parent_directories(root: &Path, parent: Option<&Path>) -> Result<(), String> { let mut current = parent; while let Some(path) = current { if path == root { break; } let metadata = match fs::symlink_metadata(path) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { current = path.parent(); continue; } Err(error) => { return Err(format!( "检查旧官方资源父目录失败 {}：{error}", path.display() )); } }; if metadata.file_type().is_symlink() { return Err(format!( "旧官方资源父目录不能是 symlink：{}", path.display() )); } if !metadata.is_dir() { break; } let mut entries = fs::read_dir(path) .map_err(|error| format!("读取旧官方资源父目录失败 {}：{error}", path.display()))?; if entries.next().is_some() { break; } fs::remove_dir(path) .map_err(|error| format!("清理旧官方资源空目录失败 {}：{error}", path.display()))?; current = path.parent(); } Ok(()) } fn unix_seconds_now() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs()) .unwrap_or(0) } fn blake3_file_hex(path: &Path) -> Result { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { return Err(format!("拒绝读取 symlink 文件：{}", path.display())); } Ok(metadata) if metadata.is_file() => {} Ok(_) => return Err(format!("期望文件路径，但实际不是文件：{}", path.display())), Err(error) => return Err(format!("读取路径信息失败 {}：{error}", path.display())), } let mut file = File::open(path).map_err(|error| format!("打开文件失败 {}：{error}", path.display()))?; let mut hasher = blake3::Hasher::new(); let mut buffer = [0u8; 64 * 1024]; loop { let read = file .read(&mut buffer) .map_err(|error| format!("读取文件失败 {}：{error}", path.display()))?; if read == 0 { break; } hasher.update(&buffer[..read]); } Ok(hasher.finalize().to_hex().to_string()) } #[cfg(test)] mod tests { use super::*; use crate::official_pull::{ build_official_pull_plan, build_official_pull_plan_for_platforms, OfficialResourcePullPlan, }; use bat_adapters::official::inventory::{ YostarJpDownloadInventory, YostarJpPlatformDownloadInventory, }; use bat_adapters::official::yostar_jp::{ PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, YostarJpResourceEndpointKind, }; use std::fs; use tempfile::TempDir; fn xxhash32(bytes: &[u8]) -> u32 { XxHash32DecimalSeedZero .digest(bytes) .parse() .expect("xxHash32 decimal digest") } #[test] fn fetch_bytes_maps_rejections_to_error_codes() { // 非官方 URL：安全边界拒绝。 let service = OfficialResourcePullService::new("/tmp/unused"); let error = service .fetch_bytes("https://evil.example/server_info.json") .unwrap_err(); assert_eq!(error.code(), bat_core::ErrorCode::NON_OFFICIAL_URL); // curl 404：保留网络域码。 let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_shell_script( &curl_path, r#"#!/bin/sh printf 'curl: (22) The requested URL returned error: 404\n' >&2 exit 22 "#, ); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let error = service .fetch_bytes("https://prod-clientpatch.bluearchiveyostar.com/r93_token/missing.bytes") .unwrap_err(); assert_eq!(error.code(), bat_core::ErrorCode::HTTP_NOT_FOUND); } #[test] fn read_download_manifest_at_handles_missing_and_bad_version() { let temp = TempDir::new().unwrap(); // 文件缺失：Ok(None)。 assert!(read_download_manifest_at(temp.path()).unwrap().is_none()); // 正常 manifest：读取条目。 let path = temp.path().join(DOWNLOAD_MANIFEST_FILE); fs::write( &path, br#"{"version":1,"entries":{"https://a":{"url":"https://a","destination":"a","bytes":1,"blake3":"aa"}}}"#, ) .unwrap(); let manifest = read_download_manifest_at(temp.path()).unwrap().unwrap(); assert_eq!(manifest.entries.len(), 1); // 不支持的版本：Err。 fs::write(&path, br#"{"version":999,"entries":{}}"#).unwrap(); assert!(read_download_manifest_at(temp.path()).is_err()); } fn discovery_plan() -> YostarJpResourceDiscoveryPlan { YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: Some("s8tloc7lo3".to_string()), addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![ YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(), }, YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalogHash, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.hash".to_string(), }, ], } } fn inventory() -> YostarJpDownloadInventory { YostarJpDownloadInventory::from_catalog_bytes( b"FullPatch_000.zip", b"ExcelDB.db ExcelDB.db", b"GameData\\Audio\\VOC_JP\\JP_Airi.zip", ) } fn platform_inventory() -> YostarJpPlatformDownloadInventory { YostarJpPlatformDownloadInventory::from_shared_inventory( inventory(), &[PatchPlatform::Windows], ) } fn fake_curl_script() -> String { format!( r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" case "$url" in */TableBundles/TableCatalog.hash) printf '%s' '{table_catalog_hash}' > "$out" ;; */TableBundles/TableCatalog.bytes) printf '%s' 'ExcelDB.db' > "$out" ;; */Windows_PatchPack/BundlePackingInfo.hash) printf '%s' '{windows_bundle_hash}' > "$out" ;; */Android_PatchPack/BundlePackingInfo.hash) printf '%s' '{android_bundle_hash}' > "$out" ;; */Windows_PatchPack/BundlePackingInfo.bytes) printf '%s' 'FullPatch_000.zip' > "$out" ;; */Android_PatchPack/BundlePackingInfo.bytes) printf '%s' 'FullPatch_001.zip' > "$out" ;; */MediaResources-Windows/Catalog/MediaCatalog.hash) printf '%s' '{windows_media_hash}' > "$out" ;; */MediaResources/Catalog/MediaCatalog.hash) printf '%s' '{android_media_hash}' > "$out" ;; */MediaResources-Windows/Catalog/MediaCatalog.bytes) printf '%s' 'GameData\Audio\VOC_JP\JP_Airi_Win.zip' > "$out" ;; */MediaResources/Catalog/MediaCatalog.bytes) printf '%s' 'GameData\Audio\VOC_JP\JP_Airi_Android.zip' > "$out" ;; *.zip) printf '\120\113\003\004\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000file.txtok\120\113\001\002\024\000\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000file.txt\120\113\005\006\000\000\000\000\001\000\001\000\066\000\000\000\050\000\000\000\000\000' > "$out" ;; *) printf '%s' "$url" > "$out" ;; esac else printf '%s' "$url" fi "#, table_catalog_hash = xxhash32(b"ExcelDB.db"), windows_bundle_hash = xxhash32(b"FullPatch_000.zip"), android_bundle_hash = xxhash32(b"FullPatch_001.zip"), windows_media_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Win.zip"), android_media_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"), ) } fn fake_resume_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" resume=0 while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --continue-at) resume=1 shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" if [ "$resume" -eq 1 ]; then printf '%s' "-resumed" >> "$out" else printf '%s' "$url" > "$out" fi else printf '%s' "$url" fi "# } fn fake_flaky_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done state="$0.state" count=0 if [ -f "$state" ]; then count="$(cat "$state")" fi count=$((count + 1)) printf '%s' "$count" > "$state" if [ "$count" -eq 1 ]; then printf '%s\n' "temporary failure" >&2 exit 22 fi if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" printf '%s' "$url" > "$out" else printf '%s' "$url" fi "# } fn fake_bad_hash_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" case "$url" in */TableBundles/TableCatalog.hash) printf '%s' '1' > "$out" ;; */TableBundles/TableCatalog.bytes) printf '%s' 'ExcelDB.db' > "$out" ;; *) printf '%s' "$url" > "$out" ;; esac else printf '%s' "$url" fi "# } fn fake_bad_zip_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" printf '%s' 'not a zip archive' > "$out" else printf '%s' "$url" fi "# } fn fake_http_error_curl_script(status: u16) -> String { format!( r#"#!/bin/sh set -eu url="" while [ "$#" -gt 0 ]; do case "$1" in --url) url="$2" shift 2 ;; *) shift ;; esac done state="$0.state" count=0 if [ -f "$state" ]; then count="$(cat "$state")" fi count=$((count + 1)) printf '%s' "$count" > "$state" printf 'curl: (22) The requested URL returned error: {status}\n' >&2 exit 22 "# ) } fn write_shell_script(path: &Path, script: &str) { let temp_path = path.with_extension("tmp"); fs::write(&temp_path, script).unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; let mut permissions = fs::metadata(&temp_path).unwrap().permissions(); permissions.set_mode(0o755); fs::set_permissions(&temp_path, permissions).unwrap(); } fs::rename(temp_path, path).unwrap(); } fn write_fake_curl(path: &Path) { write_shell_script(path, &fake_curl_script()); } fn write_fake_resume_curl(path: &Path) { write_shell_script(path, fake_resume_curl_script()); } fn write_fake_flaky_curl(path: &Path) { write_shell_script(path, fake_flaky_curl_script()); } fn write_fake_bad_hash_curl(path: &Path) { write_shell_script(path, fake_bad_hash_curl_script()); } fn write_fake_bad_zip_curl(path: &Path) { write_shell_script(path, fake_bad_zip_curl_script()); } fn write_fake_http_error_curl(path: &Path, status: u16) { write_shell_script(path, &fake_http_error_curl_script(status)); } fn one_url_plan(url: &str) -> OfficialResourcePullPlan { OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), } } #[test] fn official_distribution_mapping_identity_is_deterministic_and_sensitive() { let entries = [ ("https://example.invalid/z", "z", b"z-bytes".as_slice()), ("https://example.invalid/a", "a", b"a-bytes".as_slice()), ]; let mut first = OfficialDownloadManifest::default(); for (url, destination, bytes) in entries { first.entries.insert( url.to_string(), OfficialDownloadManifestEntry { url: url.to_string(), destination: destination.to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }, ); } let mut second = OfficialDownloadManifest::default(); for (url, destination, bytes) in entries.into_iter().rev() { second.entries.insert( url.to_string(), OfficialDownloadManifestEntry { url: url.to_string(), destination: destination.to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }, ); } let identity = official_distribution_mapping_identity(&first); assert_eq!(identity, official_distribution_mapping_identity(&second)); let mutators: &[fn(&mut OfficialDownloadManifest)] = &[ |manifest: &mut OfficialDownloadManifest| { manifest .entries .get_mut("https://example.invalid/a") .unwrap() .destination = "changed".to_string() }, |manifest: &mut OfficialDownloadManifest| { manifest .entries .get_mut("https://example.invalid/a") .unwrap() .url = "https://example.invalid/changed".to_string() }, |manifest: &mut OfficialDownloadManifest| { manifest .entries .get_mut("https://example.invalid/a") .unwrap() .bytes += 1 }, |manifest: &mut OfficialDownloadManifest| { manifest .entries .get_mut("https://example.invalid/a") .unwrap() .blake3 = "0".repeat(64) }, ]; for mutate in mutators { let mut changed = first.clone(); mutate(&mut changed); assert_ne!(identity, official_distribution_mapping_identity(&changed)); } } fn one_file_zip(name: &[u8], data: &[u8]) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(name.len() as u16).to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(name); bytes.extend_from_slice(data); let central_offset = bytes.len() as u32; bytes.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(name.len() as u16).to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(name); let central_size = bytes.len() as u32 - central_offset; bytes.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&1u16.to_le_bytes()); bytes.extend_from_slice(&1u16.to_le_bytes()); bytes.extend_from_slice(¢ral_size.to_le_bytes()); bytes.extend_from_slice(¢ral_offset.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes } #[test] fn downloads_verified_pull_plan_to_disk() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert!(report.total_bytes() > 0); assert_eq!(report.downloaded_count(), report.items.len()); assert_eq!(report.skipped_count(), 0); assert_eq!(report.resumed_count(), 0); assert_eq!(report.transferred_bytes(), report.total_bytes()); assert_eq!(report.items[0].url, plan.discovery.endpoints[0].url); for item in &report.items { assert!(item.destination.exists()); assert_eq!(item.status, OfficialResourcePullStatus::Downloaded); assert_eq!(item.bytes, item.transferred_bytes); } assert_eq!(report.official_hash_verified_count(), 1); assert_eq!( report.verified_hashes[0].algorithm, OfficialResourceHashAlgorithm::XxHash32Decimal ); assert_eq!(report.verified_hashes[0].expected, "2044170421"); assert_eq!(report.verified_hashes[0].actual, "2044170421"); let manifest = service.read_download_manifest().unwrap(); assert_eq!(manifest.version, DOWNLOAD_MANIFEST_VERSION); assert_eq!(manifest.entries.len(), report.items.len()); assert_eq!( manifest.distribution_mapping_identity.as_deref(), Some(official_distribution_mapping_identity(&manifest).as_str()) ); assert_eq!(manifest.destination_index.len(), manifest.entries.len()); for item in &report.items { let entry = manifest.entries.get(&item.url).unwrap(); assert_eq!(entry.url, item.url); assert_eq!( entry.destination, service.relative_destination(&item.destination).unwrap() ); assert_eq!(entry.bytes, item.bytes); assert_eq!(entry.blake3, blake3_file_hex(&item.destination).unwrap()); } } #[test] fn rejects_dangerous_output_root() { let service = OfficialResourcePullService::new(PathBuf::from("/")); let error = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap_err(); assert!(error.to_string().contains("危险路径")); } #[test] fn rejects_resource_url_with_query_or_fragment() { let temp = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(temp.path().join("out")); for url in [ "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes?v=1", "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes#frag", ] { let error = service.destination_for_url(url).unwrap_err(); assert!( error.contains("query 或 fragment"), "unexpected error for {url}: {error}" ); } // 无 query 的正常资源 URL 仍可映射。 assert!(service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .is_ok()); } #[cfg(unix)] #[test] fn pull_rejects_symlink_parent_escape() { use std::os::unix::fs::symlink; let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let escape_dir = temp.path().join("escape"); let bin_dir = TempDir::new().unwrap(); fs::create_dir_all(&out_dir).unwrap(); fs::create_dir_all(&escape_dir).unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path); let destination = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); let host_dir = destination .ancestors() .find(|path| { path.file_name().and_then(|name| name.to_str()) == Some("prod-clientpatch.bluearchiveyostar.com") }) .unwrap() .to_path_buf(); fs::create_dir_all(&host_dir).unwrap(); symlink(&escape_dir, host_dir.join("r93_token")).unwrap(); let error = service .pull(&build_official_pull_plan(discovery_plan(), inventory())) .unwrap_err(); assert!(error.to_string().contains("symlink")); assert!(fs::read_dir(&escape_dir).unwrap().next().is_none()); } #[cfg(unix)] #[test] fn pull_rejects_symlink_partial_file() { use std::os::unix::fs::symlink; let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let escape_file = temp.path().join("escape.txt"); let bin_dir = TempDir::new().unwrap(); fs::create_dir_all(&out_dir).unwrap(); fs::write(&escape_file, b"outside").unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes"; let destination = service.destination_for_url(url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); symlink(&escape_file, partial_path_for(&destination)).unwrap(); let error = service.pull_one(url, &destination).unwrap_err(); assert!(error.to_string().contains("symlink")); assert_eq!(fs::read(&escape_file).unwrap(), b"outside"); } #[test] fn local_manifest_audit_detects_and_repairs_corrupted_file() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let first_report = service.pull(&plan).unwrap(); let clean_audit = service.audit_local_manifest(&plan).unwrap(); assert!(clean_audit.is_clean()); assert_eq!(clean_audit.verified_count(), first_report.items.len()); assert_eq!(clean_audit.manifest_blake3_verified_count(), 7); assert_eq!(clean_audit.zip_structure_verified_count(), 4); let corrupted = first_report .items .iter() .find(|item| item.url.ends_with("/Windows_PatchPack/FullPatch_000.zip")) .unwrap(); let mut corrupted_bytes = fs::read(&corrupted.destination).unwrap(); corrupted_bytes[0] ^= 0xff; fs::write(&corrupted.destination, corrupted_bytes).unwrap(); let corrupted_audit = service.audit_local_manifest(&plan).unwrap(); assert!(!corrupted_audit.is_clean()); assert_eq!(corrupted_audit.repair_needed_count(), 1); assert!(corrupted_audit.items.iter().any(|item| { item.url == corrupted.url && item.status == OfficialLocalManifestAuditStatus::Blake3Mismatch })); let repair_report = service.pull(&plan).unwrap(); assert!(repair_report.downloaded_count() >= 1); assert!(service.audit_local_manifest(&plan).unwrap().is_clean()); } #[test] fn full_local_verification_checks_manifest_and_official_hash_pairs() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); service.pull(&plan).unwrap(); let verification = service.verify_local_download_manifest().unwrap(); assert!(verification.is_clean()); assert_eq!(verification.items.len(), 7); assert_eq!(verification.verified_count(), 7); assert_eq!(verification.manifest_blake3_verified_count(), 7); assert_eq!(verification.zip_structure_verified_count(), 4); assert_eq!(verification.official_hash_pair_count, 1); assert_eq!(verification.official_hash_verified_count, 1); let data_path = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); fs::write(data_path, b"corrupted").unwrap(); let corrupted = service.verify_local_download_manifest().unwrap(); assert!(!corrupted.is_clean()); assert_eq!(corrupted.failure_count(), 1); assert_eq!(corrupted.official_hash_errors.len(), 1); } #[test] fn refreshes_official_hash_pairs_and_skips_manifest_verified_content() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let mut manifest = OfficialDownloadManifest::default(); for url in plan.all_urls().unwrap() { let destination = service.destination_for_url(&url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); let bytes = if url.ends_with("/TableBundles/TableCatalog.bytes") { b"ExcelDB.db".to_vec() } else if url.ends_with("/TableBundles/TableCatalog.hash") { b"2044170421".to_vec() } else if url.ends_with(".zip") { one_file_zip(b"fixture.txt", b"ok") } else { url.as_bytes().to_vec() }; fs::write(&destination, bytes).unwrap(); service .record_download_manifest_entry(&mut manifest, &url, &destination) .unwrap(); } service.write_download_manifest(&manifest).unwrap(); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert_eq!(report.skipped_count(), 5); assert_eq!(report.downloaded_count(), 2); assert_eq!(report.resumed_count(), 0); assert!(report.transferred_bytes() > 0); assert_eq!(report.official_hash_verified_count(), 1); assert_eq!( fs::read_to_string( service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap() ) .unwrap(), "ExcelDB.db" ); } #[test] fn redownloads_existing_files_without_manifest() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); for url in plan.all_urls().unwrap() { let destination = service.destination_for_url(&url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); fs::write(destination, b"stale-local-file").unwrap(); } let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert_eq!(report.skipped_count(), 0); assert_eq!(report.downloaded_count(), 7); assert_eq!(report.resumed_count(), 0); assert_eq!(report.official_hash_verified_count(), 1); assert!(report.transferred_bytes() > 0); assert!(report .items .iter() .all(|item| item.status == OfficialResourcePullStatus::Downloaded)); } #[test] fn local_resource_state_reports_manifest_entries_and_existing_files() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let urls = plan.all_urls().unwrap(); let empty_state = service.local_resource_state(&plan).unwrap(); assert_eq!(empty_state.manifest_entry_count, 0); assert_eq!(empty_state.existing_file_count, 0); assert!(!empty_state.has_any_resources()); let first_destination = service.destination_for_url(&urls[0]).unwrap(); fs::create_dir_all(first_destination.parent().unwrap()).unwrap(); fs::write(&first_destination, b"local-only").unwrap(); let file_only_state = service.local_resource_state(&plan).unwrap(); assert_eq!(file_only_state.manifest_entry_count, 0); assert_eq!(file_only_state.existing_file_count, 1); assert!(file_only_state.has_any_resources()); let mut manifest = OfficialDownloadManifest::default(); service .record_download_manifest_entry(&mut manifest, &urls[0], &first_destination) .unwrap(); service.write_download_manifest(&manifest).unwrap(); let manifest_state = service.local_resource_state(&plan).unwrap(); assert_eq!(manifest_state.manifest_entry_count, 1); assert_eq!(manifest_state.existing_file_count, 1); assert!(manifest_state.has_any_resources()); } #[test] fn prune_stale_manifest_entries_removes_old_resource_and_partial_only() { let out_dir = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(out_dir.path()); let plan = build_official_pull_plan(discovery_plan(), inventory()); let expected_url = plan.all_urls().unwrap().into_iter().next().unwrap(); let expected_destination = service.destination_for_url(&expected_url).unwrap(); fs::create_dir_all(expected_destination.parent().unwrap()).unwrap(); fs::write(&expected_destination, b"keep").unwrap(); let stale_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle"; let stale_destination = service.destination_for_url(stale_url).unwrap(); fs::create_dir_all(stale_destination.parent().unwrap()).unwrap(); fs::write(&stale_destination, b"remove").unwrap(); fs::write(partial_path_for(&stale_destination), b"remove-partial").unwrap(); let mut manifest = OfficialDownloadManifest::default(); service .record_download_manifest_entry(&mut manifest, &expected_url, &expected_destination) .unwrap(); manifest.entries.insert( stale_url.to_string(), OfficialDownloadManifestEntry { url: stale_url.to_string(), destination: service.relative_destination(&stale_destination).unwrap(), bytes: 6, blake3: "stale".to_string(), }, ); service.write_download_manifest(&manifest).unwrap(); let snapshot_path = out_dir.path().join("official-sync-snapshot.json"); let quarantine_path = out_dir.path().join(DOWNLOAD_QUARANTINE_FILE); fs::write(&snapshot_path, b"snapshot").unwrap(); fs::write(&quarantine_path, b"quarantine").unwrap(); assert_eq!(service.prune_stale_manifest_entries(&plan).unwrap(), 1); assert!(!stale_destination.exists()); assert!(!partial_path_for(&stale_destination).exists()); assert!(!stale_destination.parent().unwrap().exists()); assert!(expected_destination.exists()); assert_eq!(fs::read(&snapshot_path).unwrap(), b"snapshot"); assert_eq!(fs::read(&quarantine_path).unwrap(), b"quarantine"); let pruned_manifest = service.read_download_manifest().unwrap(); assert!(pruned_manifest.entries.contains_key(&expected_url)); assert!(!pruned_manifest.entries.contains_key(stale_url)); } #[cfg(unix)] #[test] fn prune_stale_manifest_entries_rejects_symlink_destination() { use std::os::unix::fs::symlink; let out_dir = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(out_dir.path()); let plan = build_official_pull_plan(discovery_plan(), inventory()); let stale_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle"; let stale_destination = service.destination_for_url(stale_url).unwrap(); fs::create_dir_all(stale_destination.parent().unwrap()).unwrap(); let outside = out_dir.path().join("outside-resource"); fs::write(&outside, b"must-keep").unwrap(); symlink(&outside, &stale_destination).unwrap(); let mut manifest = OfficialDownloadManifest::default(); manifest.entries.insert( stale_url.to_string(), OfficialDownloadManifestEntry { url: stale_url.to_string(), destination: service.relative_destination(&stale_destination).unwrap(), bytes: 9, blake3: "stale".to_string(), }, ); service.write_download_manifest(&manifest).unwrap(); let error = service.prune_stale_manifest_entries(&plan).unwrap_err(); assert!(error.contains("symlink")); assert!(stale_destination.exists()); assert_eq!(fs::read(&outside).unwrap(), b"must-keep"); assert!(service .read_download_manifest() .unwrap() .entries .contains_key(stale_url)); } #[test] fn redownloads_existing_file_when_manifest_hash_mismatches() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let first_report = service.pull(&plan).unwrap(); assert_eq!(first_report.downloaded_count(), first_report.items.len()); let first_item = &first_report.items[0]; fs::write(&first_item.destination, b"corrupted").unwrap(); let second_report = service .pull(&OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: first_item.url.clone(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }) .unwrap(); assert_eq!(second_report.items.len(), 1); assert_eq!(second_report.skipped_count(), 0); assert_eq!(second_report.downloaded_count(), 1); assert_eq!( fs::read_to_string(&first_item.destination).unwrap(), "ExcelDB.db" ); } #[test] fn rejects_seed_catalog_when_official_hash_mismatches() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_bad_hash_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = OfficialResourcePullPlan { discovery: discovery_plan(), inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let error = service.pull(&plan).unwrap_err(); assert!(error.to_string().contains("官方 hash 校验失败")); let manifest = service.read_download_manifest().unwrap(); assert!(manifest.entries.is_empty()); let audit = service.audit_local_manifest(&plan).unwrap(); assert!(!audit.is_clean()); assert_eq!(audit.repair_needed_count(), 2); assert!(audit .items .iter() .all(|item| { item.status == OfficialLocalManifestAuditStatus::MissingManifestEntry })); } #[test] fn verifies_known_real_seed_catalog_hash_samples() { assert_eq!(xxhash32(b""), 46947589); assert_eq!(xxhash32(b"ExcelDB.db"), 2044170421); assert_eq!(xxhash32(b"FullPatch_000.zip"), 4038880697); assert_eq!( xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Win.zip"), 1865771294 ); assert_eq!(xxhash32(b"ExcelDB.db ExcelDB.db"), 2147704569); } #[test] fn rejects_downloaded_zip_when_structure_is_invalid() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_bad_zip_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let zip_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip"; let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::AddressablesCatalog, platform: Some(PatchPlatform::Windows), url: zip_url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let error = service.pull(&plan).unwrap_err(); assert!(error.to_string().contains("ZIP")); assert!(!partial_path_for(&service.destination_for_url(zip_url).unwrap()).exists()); assert!(service.read_download_manifest().unwrap().entries.is_empty()); } #[test] fn local_manifest_audit_rejects_zip_with_matching_size_and_blake3_but_bad_structure() { let out_dir = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(out_dir.path()); let zip_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip"; let destination = service.destination_for_url(zip_url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); fs::write(&destination, b"not a zip archive").unwrap(); let mut manifest = OfficialDownloadManifest::default(); let bytes = fs::metadata(&destination).unwrap().len(); let blake3 = blake3_file_hex(&destination).unwrap(); manifest.entries.insert( zip_url.to_string(), OfficialDownloadManifestEntry { url: zip_url.to_string(), destination: service.relative_destination(&destination).unwrap(), bytes, blake3, }, ); service.write_download_manifest(&manifest).unwrap(); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::AddressablesCatalog, platform: Some(PatchPlatform::Windows), url: zip_url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let audit = service.audit_local_manifest(&plan).unwrap(); let verification = service.verify_local_download_manifest().unwrap(); assert_eq!( audit.items[0].status, OfficialLocalManifestAuditStatus::ZipStructureInvalid ); assert!(audit.items[0].zip_error.as_ref().unwrap().contains("ZIP")); assert!(!verification.is_clean()); assert_eq!(verification.failure_count(), 1); } #[test] fn resumes_partial_files_before_atomic_completion() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_resume_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let first_url = plan.all_urls().unwrap().remove(0); let destination = service.destination_for_url(&first_url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); let partial = partial_path_for(&destination); fs::write(&partial, b"partial").unwrap(); let report = service .pull(&OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: first_url, }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }) .unwrap(); assert_eq!(report.items.len(), 1); assert_eq!(report.resumed_count(), 1); assert_eq!(report.items[0].status, OfficialResourcePullStatus::Resumed); assert_eq!(fs::read_to_string(&destination).unwrap(), "partial-resumed"); assert!(!partial.exists()); let manifest = service.read_download_manifest().unwrap(); assert_eq!(manifest.entries.len(), 1); assert!(manifest.entries.contains_key(&report.items[0].url)); } #[test] fn pull_emits_started_and_finished_progress_events() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); // 并发下事件顺序不固定，此处只断言与顺序无关的不变量；并发正确性另有 // downloads_run_concurrently_and_each_url_reports_once 专测。 let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let expected_urls = plan.all_urls().unwrap().len(); let mut events = Vec::new(); let report = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap(); assert_eq!(report.items.len(), expected_urls); let started: Vec<_> = events .iter() .filter(|event| event.kind == OfficialResourcePullProgressKind::Started) .collect(); let finished: Vec<_> = events .iter() .filter(|event| event.kind == OfficialResourcePullProgressKind::Finished) .collect(); let verifications: Vec<_> = events .iter() .filter(|event| event.kind == OfficialResourcePullProgressKind::Verification) .collect(); assert_eq!(started.len(), expected_urls); assert_eq!(finished.len(), expected_urls); assert_eq!(verifications.len(), report.verified_hashes.len()); for verification_event in &verifications { let hash = verification_event .official_hash .as_ref() .expect("verification event must carry official hash detail"); let pair_finished_indices = [hash.data_url.as_str(), hash.hash_url.as_str()] .into_iter() .map(|url| { events .iter() .position(|event| { event.kind == OfficialResourcePullProgressKind::Finished && event.url == url }) .expect("hash pair member must finish before verification") }) .collect::>(); let verification_index = events .iter() .position(|event| std::ptr::eq(event, *verification_event)) .expect("verification event must be present in event stream"); assert_eq!( verification_index, pair_finished_indices.into_iter().max().unwrap() + 1, "official hash verification must run immediately after the hash pair is complete" ); } // 每个 started 的 total 一致；index 表示已完成数量，不能超过总数。 assert!(started.iter().all(|event| event.index <= expected_urls)); assert!(started.iter().all(|event| event.total == expected_urls)); assert!(events.windows(2).all(|pair| pair[0].index <= pair[1].index)); let finished_indices: Vec = finished.iter().map(|event| event.index).collect(); assert_eq!(finished_indices, (1..=expected_urls).collect::>()); // started 与 finished 覆盖同一组 URL；finished 均为已下载。 let started_urls: HashSet<_> = started.iter().map(|event| event.url.clone()).collect(); let finished_urls: HashSet<_> = finished.iter().map(|event| event.url.clone()).collect(); assert_eq!(started_urls, finished_urls); assert!(finished .iter() .all(|event| event.status == Some(OfficialResourcePullStatus::Downloaded))); assert!(finished.iter().all(|event| { event.verification.as_ref().is_some_and(|verification| { verification.actual_bytes > 0 && !verification.actual_blake3.is_empty() }) })); assert!(verifications.iter().all(|event| { event.official_hash.as_ref().is_some_and(|hash| { hash.algorithm == OfficialResourceHashAlgorithm::XxHash32Decimal && hash.expected == hash.actual && !hash.hash_url.is_empty() }) })); assert!(events .iter() .any(|event| event.url.ends_with("/TableBundles/ExcelDB.db"))); } #[test] fn downloads_run_sequentially_and_each_url_reports_once() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); // 顺序下载：每个 URL 恰好一次 started + 一次 finished，全部文件落盘。 let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_max_concurrency(1); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let all_urls = plan.all_urls().unwrap(); let mut events = Vec::new(); let report = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap(); assert_eq!(report.items.len(), all_urls.len()); for url in &all_urls { let started = events .iter() .filter(|event| { &event.url == url && event.kind == OfficialResourcePullProgressKind::Started }) .count(); let finished = events .iter() .filter(|event| { &event.url == url && event.kind == OfficialResourcePullProgressKind::Finished }) .count(); assert_eq!(started, 1, "url {url} 的 started 次数"); assert_eq!(finished, 1, "url {url} 的 finished 次数"); } // 所有下载都记入 manifest。 let manifest = service.read_download_manifest().unwrap(); assert_eq!(manifest.entries.len(), all_urls.len()); } #[test] fn downloads_run_with_bounded_concurrency_and_keep_report_order() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_max_concurrency(3); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let all_urls = plan.all_urls().unwrap(); let mut events = Vec::new(); let report = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap(); assert_eq!(service.max_concurrency(), 3); assert_eq!(report.items.len(), all_urls.len()); assert_eq!( report .items .iter() .map(|item| &item.url) .collect::>(), all_urls.iter().collect::>() ); for url in &all_urls { assert_eq!( events .iter() .filter(|event| { event.url == *url && event.kind == OfficialResourcePullProgressKind::Started }) .count(), 1, "url {url} 的 started 次数" ); assert_eq!( events .iter() .filter(|event| { event.url == *url && event.kind == OfficialResourcePullProgressKind::Finished }) .count(), 1, "url {url} 的 finished 次数" ); } let finished_indices = events .iter() .filter(|event| event.kind == OfficialResourcePullProgressKind::Finished) .map(|event| event.index) .collect::>(); assert_eq!(finished_indices, (1..=all_urls.len()).collect::>()); assert_eq!( service.read_download_manifest().unwrap().entries.len(), all_urls.len() ); } #[test] fn retries_transient_download_failures() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_flaky_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(2); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 1); assert_eq!(report.downloaded_count(), 1); assert_eq!( fs::read_to_string(&report.items[0].destination).unwrap(), report.items[0].url ); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "2" ); } #[test] fn does_not_retry_terminal_http_404_and_records_quarantine() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_http_error_curl(&curl_path, 404); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Missing.bytes"; let plan = one_url_plan(url); let mut events = Vec::new(); let error = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap_err(); assert!(error.to_string().contains("quarantine")); assert!(error.to_string().contains("http_not_found")); assert!(error.to_string().contains("retryable=false")); // 类型化错误携带准确的 HTTP 404 网络域码。 assert_eq!(error.code(), bat_core::ErrorCode::HTTP_NOT_FOUND); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "1" ); let failed = events .iter() .find(|event| event.kind == OfficialResourcePullProgressKind::Failed) .unwrap(); assert_eq!(failed.failure_kind.as_deref(), Some("http_not_found")); assert_eq!(failed.failure_http_status, Some(404)); assert_eq!(failed.failure_retryable, Some(false)); assert_eq!(failed.failure_attempts, Some(1)); assert!(failed.quarantined); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.http_status, Some(404)); assert_eq!(entry.failure_kind.as_deref(), Some("http_not_found")); assert_eq!(entry.retryable, Some(false)); assert_eq!(entry.attempts, 1); assert!(service.read_download_manifest().unwrap().entries.is_empty()); } #[test] fn official_http_failure_regression_fixtures_are_enforced() { for fixture in [ include_str!("../tests/fixtures/official_regression/http_403.json"), include_str!("../tests/fixtures/official_regression/http_404.json"), ] { let fixture: serde_json::Value = serde_json::from_str(fixture).unwrap(); let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); let status = fixture["http_status"].as_u64().unwrap() as u16; write_fake_http_error_curl(&curl_path, status); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = fixture["url"].as_str().unwrap(); let error = service.pull(&one_url_plan(url)).unwrap_err(); let expected_kind = fixture["failure_kind"].as_str().unwrap(); let expected_retryable = fixture["retryable"].as_bool().unwrap(); let expected_attempts = fixture["expected_attempts"].as_u64().unwrap(); assert!(error.to_string().contains(expected_kind)); assert!(error .to_string() .contains(&format!("retryable={expected_retryable}"))); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), expected_attempts.to_string() ); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.failure_kind.as_deref(), Some(expected_kind)); assert_eq!(entry.retryable, Some(expected_retryable)); assert_eq!(entry.attempts, expected_attempts as usize); } } #[test] fn official_hash_mismatch_regression_fixture_is_enforced() { let fixture: serde_json::Value = serde_json::from_str(include_str!( "../tests/fixtures/official_regression/hash_mismatch_catalog.json" )) .unwrap(); let error = verify_official_seed_catalog_hash( fixture["data_url"].as_str().unwrap(), fixture["data"].as_str().unwrap().as_bytes(), fixture["hash_url"].as_str().unwrap(), fixture["official_hash"].as_str().unwrap().as_bytes(), ) .unwrap_err(); assert!(error .to_string() .contains(fixture["expected_error_contains"].as_str().unwrap())); } #[test] fn retries_http_5xx_before_quarantine() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_http_error_curl(&curl_path, 503); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Busy.bytes"; let plan = one_url_plan(url); let error = service.pull(&plan).unwrap_err(); assert!(error.to_string().contains("quarantine")); assert!(error.to_string().contains("http_server_error")); assert!(error.to_string().contains("retryable=true")); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "3" ); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.http_status, Some(503)); assert_eq!(entry.failure_kind.as_deref(), Some("http_server_error")); assert_eq!(entry.retryable, Some(true)); assert_eq!(entry.attempts, 3); assert!(entry.last_error.contains("重试次数已耗尽")); } #[test] fn custom_platform_selection_is_supported() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows, PatchPlatform::Android], ); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert!(report .items .iter() .any(|item| item.url.contains("Android_PatchPack"))); } #[test] fn reuses_verified_historical_release_when_cdn_root_changes() { let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let versions_dir = temp.path().join("versions"); let old_root = versions_dir.join("release-old"); let old_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let current_url = "https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes"; let destination = "prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let bytes = b"verified historical resource"; let source = old_root.join(destination); fs::create_dir_all(source.parent().unwrap()).unwrap(); fs::write(&source, bytes).unwrap(); write_historical_manifest(&old_root, old_url, destination, bytes); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path) .with_release_reuse_root(&versions_dir); let report = service.pull(&one_url_plan(current_url)).unwrap(); assert_eq!(report.release_reused_count(), 1); assert_eq!(report.cas_reused_count(), 0); assert_eq!(report.downloaded_count(), 0); assert_eq!(report.reused_bytes(), bytes.len() as u64); assert_eq!( fs::read(service.destination_for_url(current_url).unwrap()).unwrap(), bytes ); assert_eq!( report.items[0].status, OfficialResourcePullStatus::ReleaseReused ); assert!(report.items[0].cas_object_id.is_none()); assert!(report.reuse_warnings.is_empty()); let manifest = service.read_download_manifest().unwrap(); assert!(manifest.entries.contains_key(current_url)); } #[test] fn falls_back_to_cas_after_corrupt_historical_release() { let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let versions_dir = temp.path().join("versions"); let old_root = versions_dir.join("release-old"); let cas_root = temp.path().join("cas"); let old_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let current_url = "https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes"; let destination = "prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let bytes = b"cas-backed resource"; let source = old_root.join(destination); fs::create_dir_all(source.parent().unwrap()).unwrap(); fs::write(&source, b"corrupt").unwrap(); write_historical_manifest(&old_root, old_url, destination, bytes); let object_id = store_cas_object(&cas_root, bytes); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path) .with_release_reuse_root(&versions_dir) .with_cas_reuse_root(&cas_root); let report = service.pull(&one_url_plan(current_url)).unwrap(); assert_eq!(report.release_reused_count(), 0); assert_eq!(report.cas_reused_count(), 1); assert_eq!(report.downloaded_count(), 0); assert_eq!( report.items[0].cas_object_id.as_deref(), Some(object_id.as_str()) ); assert_eq!( fs::read(service.destination_for_url(current_url).unwrap()).unwrap(), bytes ); assert!(report .reuse_warnings .iter() .any(|warning| warning.source == "historical_release")); let references = read_cas_reuse_reference_manifest_at(&out_dir) .unwrap() .unwrap(); assert!(references.ownership_id.is_some()); assert_eq!(references.object_ids, vec![object_id.clone()]); let ownership_id = references.ownership_id.clone().unwrap(); assert_eq!(cas_reference_count(&cas_root, &object_id), 2); assert_eq!( release_cas_reuse_references(&out_dir, &cas_root).unwrap(), 1 ); assert_eq!(cas_reference_count(&cas_root, &object_id), 1); assert!(read_cas_reuse_reference_manifest_at(&out_dir) .unwrap() .is_none()); // 模拟 CAS 事务已提交但 release-local progress cursor 尚未写回； // 重试必须识别同一个 ownership pair，而不是再次递减。 fs::write( out_dir.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: Some(ownership_id), object_ids: vec![object_id.clone()], }) .unwrap(), ) .unwrap(); assert_eq!( release_cas_reuse_references(&out_dir, &cas_root).unwrap(), 0 ); assert_eq!(cas_reference_count(&cas_root, &object_id), 1); assert!(read_cas_reuse_reference_manifest_at(&out_dir) .unwrap() .is_none()); } #[test] fn cas_ownership_identity_isolated_for_same_release_basename() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let object_id = store_cas_object(&cas_root, b"shared object"); let cas = crate::FileSystemCasRepository::new(&cas_root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.add_reference(&object_id).await.unwrap(); cas.add_reference(&object_id).await.unwrap(); }); let first = temp.path().join("first").join("release"); let second = temp.path().join("second").join("release"); fs::create_dir_all(&first).unwrap(); fs::create_dir_all(&second).unwrap(); for (root, ownership_id) in [(&first, "owner-first"), (&second, "owner-second")] { fs::write( root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: Some(ownership_id.to_string()), object_ids: vec![object_id.clone()], }) .unwrap(), ) .unwrap(); } assert_eq!(release_cas_reuse_references(&first, &cas_root).unwrap(), 1); assert_eq!(cas_reference_count(&cas_root, &object_id), 2); assert_eq!(release_cas_reuse_references(&second, &cas_root).unwrap(), 1); assert_eq!(cas_reference_count(&cas_root, &object_id), 1); } #[test] fn legacy_cas_manifest_uses_basename_key_without_migration() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let object_id = store_cas_object(&cas_root, b"legacy object"); let release_root = temp.path().join("legacy-release"); fs::create_dir_all(&release_root).unwrap(); let mut scope = load_owner_scope_state(&release_root).unwrap(); scope .legacy_basename_compatibility .insert("legacy-release".to_string()); write_owner_scope_state(&release_root, &scope).unwrap(); fs::write( release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![object_id.clone()], }) .unwrap(), ) .unwrap(); let cas = crate::FileSystemCasRepository::new(&cas_root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.add_reference(&object_id).await.unwrap() }); assert_eq!( release_cas_reuse_references(&release_root, &cas_root).unwrap(), 1 ); assert_eq!(cas_reference_count(&cas_root, &object_id), 1); } #[test] fn legacy_basename_ledger_without_root_marker_is_ambiguous() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let output_a = temp.path().join("output-a/versions/release-x"); let output_b = temp.path().join("output-b/versions/release-x"); fs::create_dir_all(&output_a).unwrap(); fs::create_dir_all(&output_b).unwrap(); let object_id = store_cas_object(&cas_root, b"shared legacy object"); let cas = crate::FileSystemCasRepository::new(&cas_root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.add_reference(&object_id).await.unwrap(); cas.add_reference(&object_id).await.unwrap(); assert!(cas .release_reference_once("release-x", 0, &object_id) .await .unwrap()); }); let mut output_a_scope = load_owner_scope_state(&output_a).unwrap(); output_a_scope .legacy_basename_compatibility .insert("release-x".to_string()); write_owner_scope_state(&output_a, &output_a_scope).unwrap(); let manifest_path = output_b.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE); let manifest_bytes = serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![object_id.clone()], }) .unwrap(); fs::write(&manifest_path, &manifest_bytes).unwrap(); let result = release_cas_reuse_references(&output_b, &cas_root); let error = result.unwrap_err(); assert!(error.contains("ambiguous_legacy_ownership")); assert_eq!(cas_reference_count(&cas_root, &object_id), 2); let ledger_exists = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.has_release_ownership("release-x").await.unwrap() }); assert!(ledger_exists); assert_eq!(fs::read(&manifest_path).unwrap(), manifest_bytes); assert!(output_b.is_dir()); let output_b_scope = load_owner_scope_state(&output_b).unwrap(); assert!(!output_b_scope .legacy_basename_compatibility .contains("release-x")); } #[test] fn legacy_manifest_migration_persists_identity_before_retry() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let release_root = temp.path().join("versions/release-migrate"); fs::create_dir_all(&release_root).unwrap(); let object_id = store_cas_object(&cas_root, b"migrated legacy object"); let missing_object = "f".repeat(64); fs::write( release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![missing_object], }) .unwrap(), ) .unwrap(); assert!(release_cas_reuse_references(&release_root, &cas_root).is_err()); let migrated = read_cas_reuse_reference_manifest_at(&release_root) .unwrap() .unwrap(); let ownership_id = migrated.ownership_id.clone().unwrap(); assert!(ownership_id.starts_with("cas-legacy-owner-v1-")); fs::write( release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: Some(ownership_id), object_ids: vec![object_id.clone()], }) .unwrap(), ) .unwrap(); assert_eq!( release_cas_reuse_references(&release_root, &cas_root).unwrap(), 1 ); assert_eq!(cas_reference_count(&cas_root, &object_id), 0); } #[test] fn legacy_partial_cleanup_keeps_basename_ledger_and_new_generation_isolated() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let first = temp.path().join("versions").join("release-x"); fs::create_dir_all(&first).unwrap(); let mut scope = load_owner_scope_state(&first).unwrap(); scope .legacy_basename_compatibility .insert("release-x".to_string()); write_owner_scope_state(&first, &scope).unwrap(); let first_object = store_cas_object(&cas_root, b"legacy-first"); let second_object = store_cas_object(&cas_root, b"legacy-second"); let cas = crate::FileSystemCasRepository::new(&cas_root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.add_reference(&first_object).await.unwrap(); cas.add_reference(&second_object).await.unwrap(); assert!(cas .release_reference_once("release-x", 0, &first_object) .await .unwrap()); }); fs::write( first.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![first_object.clone()], }) .unwrap(), ) .unwrap(); assert_eq!(release_cas_reuse_references(&first, &cas_root).unwrap(), 0); assert_eq!(cas_reference_count(&cas_root, &first_object), 1); fs::create_dir_all(&first).unwrap(); fs::write( first.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![second_object.clone()], }) .unwrap(), ) .unwrap(); assert_eq!(release_cas_reuse_references(&first, &cas_root).unwrap(), 1); assert_eq!(cas_reference_count(&cas_root, &second_object), 1); assert_eq!(cas_reference_count(&cas_root, &first_object), 1); } #[test] fn legacy_generations_in_different_output_roots_do_not_share_scope() { let temp = TempDir::new().unwrap(); let cas_root = temp.path().join("cas"); let first = temp.path().join("output-a/versions/release-x"); let second = temp.path().join("output-b/versions/release-x"); let first_object = store_cas_object(&cas_root, b"root-a"); let second_object = store_cas_object(&cas_root, b"root-b"); for (root, object_id) in [(&first, &first_object), (&second, &second_object)] { fs::create_dir_all(root).unwrap(); fs::write( root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE), serde_json::to_vec(&OfficialCasReuseReferenceManifest { version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION, ownership_id: None, object_ids: vec![object_id.clone()], }) .unwrap(), ) .unwrap(); } assert_eq!(release_cas_reuse_references(&first, &cas_root).unwrap(), 1); assert_eq!(release_cas_reuse_references(&second, &cas_root).unwrap(), 1); assert_eq!(cas_reference_count(&cas_root, &first_object), 0); assert_eq!(cas_reference_count(&cas_root, &second_object), 0); assert_ne!( load_owner_scope_state(&first).unwrap().scope_id, load_owner_scope_state(&second).unwrap().scope_id ); } #[test] fn corrupted_cas_falls_back_to_network_with_diagnostic() { let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let versions_dir = temp.path().join("versions"); let old_root = versions_dir.join("release-old"); let cas_root = temp.path().join("cas"); let old_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let current_url = "https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes"; let destination = "prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let expected = b"expected cas resource"; write_historical_manifest(&old_root, old_url, destination, expected); let object_id = store_cas_object(&cas_root, expected); fs::write(cas_object_path(&cas_root, &object_id).unwrap(), b"corrupt").unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path) .with_release_reuse_root(&versions_dir) .with_cas_reuse_root(&cas_root); let report = service.pull(&one_url_plan(current_url)).unwrap(); assert_eq!(report.downloaded_count(), 1); assert_eq!(report.cas_reused_count(), 0); assert_eq!( fs::read(service.destination_for_url(current_url).unwrap()).unwrap(), current_url.as_bytes() ); assert!(report .reuse_warnings .iter() .any(|warning| warning.source == "cas" && warning.message.contains("完整性"))); assert!(read_cas_reuse_reference_manifest_at(&out_dir) .unwrap() .is_none()); } #[test] fn missing_cas_root_does_not_create_storage_during_fallback() { let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let versions_dir = temp.path().join("versions"); let old_root = versions_dir.join("release-old"); let cas_root = temp.path().join("cas"); let old_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let current_url = "https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes"; let destination = "prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes"; let expected = b"network fallback resource"; let source = old_root.join(destination); fs::create_dir_all(source.parent().unwrap()).unwrap(); fs::write(&source, b"corrupt").unwrap(); write_historical_manifest(&old_root, old_url, destination, expected); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path) .with_release_reuse_root(&versions_dir) .with_cas_reuse_root(&cas_root); let report = service.pull(&one_url_plan(current_url)).unwrap(); assert_eq!(report.downloaded_count(), 1); assert!(!cas_root.exists()); assert!(report.reuse_warnings.iter().any(|warning| { warning.source == "cas" && warning.message.contains("不存在或不可读取") })); } #[test] fn fetches_official_seed_bytes_with_curl() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let bytes = service .fetch_bytes( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes" ); } #[test] fn retries_transient_fetch_bytes_failures() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_flaky_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(2); let bytes = service .fetch_bytes( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes" ); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "2" ); } #[test] fn rejects_non_official_urls() { let service = OfficialResourcePullService::new("/tmp/unused"); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchive.cafe/r93_token/TableBundles/TableCatalog.bytes".to_string(), }], }, inventory: platform_inventory(), platforms: vec![PatchPlatform::Windows], }; assert!(service.pull(&plan).is_err()); } fn write_historical_manifest(root: &Path, url: &str, destination: &str, bytes: &[u8]) { fs::create_dir_all(root).unwrap(); let mut manifest = OfficialDownloadManifest::default(); manifest.entries.insert( url.to_string(), OfficialDownloadManifestEntry { url: url.to_string(), destination: destination.to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }, ); fs::write( root.join(DOWNLOAD_MANIFEST_FILE), serde_json::to_vec(&manifest).unwrap(), ) .unwrap(); } fn store_cas_object(root: &Path, bytes: &[u8]) -> String { let cas = crate::FileSystemCasRepository::new(root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.store(bytes).await.unwrap() }) } fn cas_reference_count(root: &Path, object_id: &str) -> u64 { let cas = crate::FileSystemCasRepository::new(root); tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { cas.get_reference_count(&object_id.to_string()) .await .unwrap() }) } }