//! Official JP resource download execution. use crate::curl_transfer::{run_curl_with_retry_with_proxy, CurlProxyConfig, CurlRetryError}; use crate::official_pull::OfficialResourcePullPlan; use crate::path_security::{ ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target, read_file_no_symlink, validate_output_root, write_file_atomic, STATE_FILE_MODE, }; use crate::zip_validation::{ path_has_zip_extension, url_or_path_has_zip_extension, validate_zip_structure, }; use bat_adapters::official::yostar_jp::{is_official_yostar_jp_url, YostarJpResourceEndpointKind}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashSet}; use std::fs::{self, File}; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json"; const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json"; const DOWNLOAD_MANIFEST_VERSION: u32 = 1; const DOWNLOAD_QUARANTINE_VERSION: u32 = 1; const DEFAULT_RETRY_ATTEMPTS: usize = 3; /// Outcome for one official resource pull item. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialResourcePullStatus { /// The destination already existed and passed local manifest validation. SkippedExisting, /// A partial `.part` file was resumed. Resumed, /// The resource was downloaded from scratch. Downloaded, } impl OfficialResourcePullStatus { /// Returns a stable label for reports and progress logs. pub fn as_str(self) -> &'static str { match self { Self::SkippedExisting => "skipped_existing", Self::Resumed => "resumed", Self::Downloaded => "downloaded", } } } /// Progress event kind emitted while executing an official pull plan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialResourcePullProgressKind { /// A URL is about to be checked or downloaded. Started, /// A URL finished as skipped, resumed, or downloaded. Finished, /// A URL failed after retry classification and was quarantined for this run. Failed, } impl OfficialResourcePullProgressKind { /// Returns a stable label for progress logs. pub fn as_str(self) -> &'static str { match self { Self::Started => "started", Self::Finished => "finished", Self::Failed => "failed", } } } /// Progress for one URL in an official pull plan. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialResourcePullProgress { /// Progress event kind. pub kind: OfficialResourcePullProgressKind, /// One-based URL index in the pull plan. pub index: usize, /// Total URL count in the pull plan. pub total: usize, /// Official URL currently being processed. pub url: String, /// Finished pull status, when `kind` is `Finished`. pub status: Option, /// Final local file size, when `kind` is `Finished`. pub bytes: Option, /// Bytes transferred during this run, when `kind` is `Finished`. pub transferred_bytes: Option, /// Stable failure kind label, when `kind` is `Failed`. pub failure_kind: Option, /// HTTP status parsed from curl stderr, when available. pub failure_http_status: Option, /// Whether retry policy considered the final failure retryable. pub failure_retryable: Option, /// Number of curl attempts that were executed before this failure. pub failure_attempts: Option, /// Whether the URL was written to the local quarantine manifest. pub quarantined: bool, } impl OfficialResourcePullProgress { fn started(index: usize, total: usize, url: String) -> Self { Self { kind: OfficialResourcePullProgressKind::Started, index, total, url, status: None, bytes: None, transferred_bytes: None, failure_kind: None, failure_http_status: None, failure_retryable: None, failure_attempts: None, quarantined: false, } } fn finished( index: usize, total: usize, url: String, status: OfficialResourcePullStatus, bytes: u64, transferred_bytes: u64, ) -> Self { Self { kind: OfficialResourcePullProgressKind::Finished, index, total, url, status: Some(status), bytes: Some(bytes), transferred_bytes: Some(transferred_bytes), failure_kind: None, failure_http_status: None, failure_retryable: None, failure_attempts: None, quarantined: false, } } fn failed(index: usize, total: usize, url: String, error: &PullOneError) -> Self { Self { kind: OfficialResourcePullProgressKind::Failed, index, total, url, status: None, bytes: None, transferred_bytes: None, failure_kind: error.failure_kind().map(ToOwned::to_owned), failure_http_status: error.http_status(), failure_retryable: error.retryable(), failure_attempts: error.attempts(), quarantined: true, } } } /// Official hash algorithm used by a verified resource sidecar. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialResourceHashAlgorithm { /// Decimal text form of `xxHash32` with seed `0`. XxHash32Decimal, } impl OfficialResourceHashAlgorithm { /// Returns a stable algorithm label for reports and logs. pub fn as_str(self) -> &'static str { match self { Self::XxHash32Decimal => "xxhash32_decimal", } } } /// Successful official hash verification for one downloaded resource. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialResourceHashVerification { /// Downloaded data URL that was verified. pub data_url: String, /// Official `.hash` URL used as the verification source. pub hash_url: String, /// Official hash algorithm. pub algorithm: OfficialResourceHashAlgorithm, /// Expected hash value read from the official `.hash` file. pub expected: String, /// Actual hash value computed from the downloaded data file. pub actual: String, } /// One downloaded official resource. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialResourcePullItem { /// Original URL. pub url: String, /// Local destination path under the output root. pub destination: PathBuf, /// Final local file size. pub bytes: u64, /// Bytes transferred during this pull execution. pub transferred_bytes: u64, /// Pull outcome. pub status: OfficialResourcePullStatus, } /// Download report for one executed pull plan. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialResourcePullReport { /// Downloaded resources in execution order. pub items: Vec, /// Official hash sidecars successfully verified after download. pub verified_hashes: Vec, } impl OfficialResourcePullReport { /// Returns the total downloaded byte count. pub fn total_bytes(&self) -> u64 { self.items.iter().map(|item| item.bytes).sum() } /// Returns bytes transferred by this pull execution. pub fn transferred_bytes(&self) -> u64 { self.items.iter().map(|item| item.transferred_bytes).sum() } /// Returns the number of resources reused from the local output tree. pub fn skipped_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::SkippedExisting) .count() } /// Returns the number of resources resumed from `.part` files. pub fn resumed_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::Resumed) .count() } /// Returns the number of resources downloaded from scratch. pub fn downloaded_count(&self) -> usize { self.items .iter() .filter(|item| item.status == OfficialResourcePullStatus::Downloaded) .count() } /// Returns the number of official hash sidecars verified after download. pub fn official_hash_verified_count(&self) -> usize { self.verified_hashes.len() } } /// Local download-manifest audit status for one expected official URL. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OfficialLocalManifestAuditStatus { /// The local file exists and matches the recorded manifest size and BLAKE3. Verified, /// No manifest entry exists for this URL. MissingManifestEntry, /// The manifest entry URL does not match the expected URL. UrlMismatch, /// The manifest destination does not match the URL-derived destination. DestinationMismatch, /// The manifest entry exists but the local file is missing. MissingFile, /// The local file size differs from the manifest entry. SizeMismatch, /// The local file BLAKE3 differs from the manifest entry. Blake3Mismatch, /// The local file has a `.zip` URL or destination but its ZIP structure is /// invalid. ZipStructureInvalid, } impl OfficialLocalManifestAuditStatus { /// Returns true when this status proves the local file can be reused. pub fn is_verified(self) -> bool { self == Self::Verified } /// Returns a stable status label for CLI and JSON reports. pub fn as_str(self) -> &'static str { match self { Self::Verified => "verified", Self::MissingManifestEntry => "missing_manifest_entry", Self::UrlMismatch => "url_mismatch", Self::DestinationMismatch => "destination_mismatch", Self::MissingFile => "missing_file", Self::SizeMismatch => "size_mismatch", Self::Blake3Mismatch => "blake3_mismatch", Self::ZipStructureInvalid => "zip_structure_invalid", } } } /// Local download-manifest audit result for one expected official URL. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialLocalManifestAuditItem { /// Expected official URL from the pull plan. pub url: String, /// URL-derived local destination. pub destination: PathBuf, /// Audit status. pub status: OfficialLocalManifestAuditStatus, /// Manifest-recorded destination, when an entry exists. pub manifest_destination: Option, /// Manifest-recorded byte size, when an entry exists. pub expected_bytes: Option, /// Actual local byte size, when the destination exists. pub actual_bytes: Option, /// Manifest-recorded BLAKE3, when an entry exists. pub expected_blake3: Option, /// Actual local BLAKE3, when it could be computed. pub actual_blake3: Option, /// ZIP structure validation error, when the URL or destination is a ZIP /// and the local file did not pass ZIP structure validation. pub zip_error: Option, /// Whether this item required and passed ZIP structure validation. pub zip_structure_verified: bool, } /// Local download-manifest audit result for a pull plan. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialLocalManifestAuditReport { /// Per-URL audit items in pull-plan order. pub items: Vec, } impl OfficialLocalManifestAuditReport { /// Returns true when every expected URL was locally verified. pub fn is_clean(&self) -> bool { self.items.iter().all(|item| item.status.is_verified()) } /// Returns the number of locally verified files. pub fn verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of files that require repair. pub fn repair_needed_count(&self) -> usize { self.items.len().saturating_sub(self.verified_count()) } /// Returns the number of files that passed local manifest path, size, and /// BLAKE3 validation. pub fn manifest_blake3_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of ZIP files that passed structural validation. pub fn zip_structure_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.zip_structure_verified) .count() } } /// Full local verification report for all entries currently recorded in the /// official download manifest. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OfficialLocalVerificationReport { /// Per-entry manifest audit results. pub items: Vec, /// Official seed hash pairs found in the local manifest. pub official_hash_pair_count: usize, /// Official seed hash pairs that passed xxHash32 verification. pub official_hash_verified_count: usize, /// Official seed hash verification failures. pub official_hash_errors: Vec, } impl OfficialLocalVerificationReport { /// Returns true when every local manifest entry and every local official /// seed hash pair passed verification. pub fn is_clean(&self) -> bool { self.items.iter().all(|item| item.status.is_verified()) && self.official_hash_errors.is_empty() && self.official_hash_verified_count == self.official_hash_pair_count } /// Returns the number of manifest entries that passed verification. pub fn verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of manifest entries that need attention. pub fn failure_count(&self) -> usize { self.items.len().saturating_sub(self.verified_count()) } /// Returns the number of local manifest entries that passed path, size, /// and BLAKE3 validation. pub fn manifest_blake3_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.status.is_verified()) .count() } /// Returns the number of ZIP files that passed structural validation. pub fn zip_structure_verified_count(&self) -> usize { self.items .iter() .filter(|item| item.zip_structure_verified) .count() } } /// Existing local state for an official pull plan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct OfficialLocalResourceState { /// Number of expected URLs that already have download-manifest entries. pub manifest_entry_count: usize, /// Number of expected URL destinations that already exist on disk. pub existing_file_count: usize, } impl OfficialLocalResourceState { /// Returns true when this plan has any reusable or auditable local state. pub fn has_any_resources(self) -> bool { self.manifest_entry_count > 0 || self.existing_file_count > 0 } } /// Executes an official pull plan with the system `curl` command. #[derive(Debug, Clone)] pub struct OfficialResourcePullService { output_root: PathBuf, curl_command: PathBuf, curl_proxy: CurlProxyConfig, retry_attempts: usize, } impl OfficialResourcePullService { /// Creates a pull service that uses `curl` from `PATH`. pub fn new(output_root: impl Into) -> Self { Self::with_curl_command(output_root, "curl") } /// Creates a pull service with an explicit `curl` binary path. pub fn with_curl_command( output_root: impl Into, curl_command: impl Into, ) -> Self { Self { output_root: output_root.into(), curl_command: curl_command.into(), curl_proxy: CurlProxyConfig::default(), retry_attempts: DEFAULT_RETRY_ATTEMPTS, } } /// Sets the proxy configuration for every `curl` transfer. pub fn with_proxy_config(mut self, curl_proxy: CurlProxyConfig) -> Self { self.curl_proxy = curl_proxy; self } /// Sets the number of attempts for each `curl` transfer. pub fn with_retry_attempts(mut self, retry_attempts: usize) -> Self { self.retry_attempts = retry_attempts.max(1); self } /// Returns the output root used for downloaded files. pub fn output_root(&self) -> &Path { &self.output_root } /// Returns the local manifest path used to validate completed downloads. pub fn download_manifest_path(&self) -> PathBuf { self.output_root.join(DOWNLOAD_MANIFEST_FILE) } /// Returns the local quarantine path used to diagnose failed downloads. pub fn download_quarantine_path(&self) -> PathBuf { self.output_root.join(DOWNLOAD_QUARANTINE_FILE) } /// Executes the plan and downloads every official URL to disk. pub fn pull( &self, plan: &OfficialResourcePullPlan, ) -> Result { self.pull_with_progress(plan, |_| {}) } /// Executes the plan and calls `progress` for every URL before and after it /// is checked or downloaded. pub fn pull_with_progress( &self, plan: &OfficialResourcePullPlan, mut progress: impl FnMut(OfficialResourcePullProgress), ) -> Result { self.pull_with_progress_and_cancellation(plan, &mut progress, || false) } /// Executes the plan, emits URL progress, and aborts between URLs when /// `should_cancel` returns true. pub fn pull_with_progress_and_cancellation( &self, plan: &OfficialResourcePullPlan, mut progress: impl FnMut(OfficialResourcePullProgress), mut should_cancel: impl FnMut() -> bool, ) -> Result { self.ensure_output_root_ready()?; let official_hash_pairs = official_seed_hash_pairs(plan); let force_refresh_urls = official_hash_refresh_urls(&official_hash_pairs); let mut manifest = self.read_download_manifest()?; let urls = plan.all_urls()?; let total = urls.len(); let mut items = Vec::new(); let mut verified_hashes = Vec::new(); let mut verified_hash_urls = HashSet::::new(); let mut processed_urls = HashSet::::new(); for (offset, url) in urls.into_iter().enumerate() { if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string()); } let index = offset + 1; progress(OfficialResourcePullProgress::started( index, total, url.clone(), )); if !is_official_yostar_jp_url(&url) { return Err(format!("拒绝下载非官方 URL:{url}")); } let destination = self.destination_for_url(&url)?; if let Some(parent) = destination.parent() { ensure_safe_directory_path(parent, "下载目标目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载目标目录失败 {}:{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载目标目录")?; } ensure_safe_file_target(&self.output_root, &destination, "下载目标文件")?; let force_refresh = force_refresh_urls.contains(&url); let result = if force_refresh { None } else { self.validated_existing_file(&url, &destination, &manifest)? }; let result = if let Some(result) = result { self.clear_quarantine_entry(&url)?; result } else { let result = match self.pull_one(&url, &destination) { Ok(result) => result, Err(error) => { self.record_quarantine_entry(&url, &destination, &error)?; progress(OfficialResourcePullProgress::failed( index, total, url.clone(), &error, )); return Err(format!( "官方资源下载中断:URL 已进入 quarantine 并跳过本轮,不会发布不完整资源;url={url} quarantine={};{}", self.download_quarantine_path().display(), error.message )); } }; self.clear_quarantine_entry(&url)?; self.record_download_manifest_entry(&mut manifest, &url, &destination)?; self.write_download_manifest(&manifest)?; result }; progress(OfficialResourcePullProgress::finished( index, total, url.clone(), result.status, result.bytes, result.transferred_bytes, )); processed_urls.insert(url.clone()); self.verify_ready_official_hashes( &official_hash_pairs, &processed_urls, &mut verified_hash_urls, &mut verified_hashes, &mut manifest, )?; items.push(OfficialResourcePullItem { url, destination, bytes: result.bytes, transferred_bytes: result.transferred_bytes, status: result.status, }); } if should_cancel() { return Err("官方资源拉取已被停止请求中断".to_string()); } self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?; Ok(OfficialResourcePullReport { items, verified_hashes, }) } /// Audits every URL in a pull plan against the local download manifest. /// /// This performs no network I/O. It checks that each URL has a manifest /// entry, that the entry maps to the URL-derived destination, and that the /// local file still matches the recorded size and BLAKE3 digest. pub fn audit_local_manifest( &self, plan: &OfficialResourcePullPlan, ) -> Result { let manifest = self.read_download_manifest()?; let mut items = Vec::new(); for url in plan.all_urls()? { let destination = self.destination_for_url(&url)?; items.push(self.audit_one(&url, destination, &manifest)?); } Ok(OfficialLocalManifestAuditReport { items }) } /// Verifies every entry currently present in the local download manifest. /// /// This is intentionally network-free. It validates each recorded file /// using the local manifest's size and BLAKE3 digest, then verifies every /// local seed `.bytes`/`.hash` pair that is present using the official /// decimal xxHash32 rule. pub fn verify_local_download_manifest( &self, ) -> Result { let manifest = self.read_download_manifest()?; let urls = manifest.entries.keys().cloned().collect::>(); let mut items = Vec::with_capacity(urls.len()); for url in urls { let destination = self.destination_for_url(&url)?; items.push(self.audit_one(&url, destination, &manifest)?); } let pairs = local_manifest_seed_hash_pairs(&manifest); let mut official_hash_verified_count = 0; let mut official_hash_errors = Vec::new(); for pair in &pairs { match self.verify_official_seed_hash_pair_from_disk(pair) { Ok(_) => official_hash_verified_count += 1, Err(error) => official_hash_errors.push(error), } } Ok(OfficialLocalVerificationReport { items, official_hash_pair_count: pairs.len(), official_hash_verified_count, official_hash_errors, }) } /// Returns whether the current output root already contains local state for /// this plan. This is intentionally cheaper than a full audit and is used /// to choose between audit-then-repair and first-time full pull flows. pub fn local_resource_state( &self, plan: &OfficialResourcePullPlan, ) -> Result { let manifest = self.read_download_manifest()?; let mut manifest_entry_count = 0; let mut existing_file_count = 0; for url in plan.all_urls()? { if manifest.entries.contains_key(&url) { manifest_entry_count += 1; } let destination = self.destination_for_url(&url)?; if file_len_if_exists(&destination)?.is_some() { existing_file_count += 1; } } Ok(OfficialLocalResourceState { manifest_entry_count, existing_file_count, }) } /// Fetches an official URL into memory. /// /// This is intended for small discovery inputs such as `server-info`, /// `TableCatalog.bytes`, `BundlePackingInfo.bytes`, and /// `MediaCatalog.bytes`. pub fn fetch_bytes(&self, url: &str) -> Result, String> { if !is_official_yostar_jp_url(url) { return Err(format!("拒绝拉取非官方 URL:{url}")); } let output = run_curl_with_retry_with_proxy( &self.curl_command, url, None, self.retry_attempts, &self.curl_proxy, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") .arg("--location") .arg("--silent") .arg("--show-error") .arg("--url") .arg(url); command }, ) .map_err(|error| format!("curl 拉取失败 {url}:{error}"))?; Ok(output.stdout) } fn pull_one(&self, url: &str, destination: &Path) -> Result { ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; let partial = partial_path_for(destination); ensure_safe_file_target(&self.output_root, &partial, "临时下载文件") .map_err(PullOneError::plain)?; let partial_len_before = file_len_if_exists(&partial) .map_err(PullOneError::plain)? .unwrap_or_default(); let resumed = partial_len_before > 0; let mut status = if resumed { OfficialResourcePullStatus::Resumed } else { OfficialResourcePullStatus::Downloaded }; if resumed { if let Err(error) = self.download_one(url, &partial, true) { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { PullOneError::from_retry( format!("续传失败 {url}:{error};清理后重新下载也失败:{retry_error}"), retry_error, ) })?; status = OfficialResourcePullStatus::Downloaded; } else if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); self.download_one(url, &partial, false) .map_err(|retry_error| { PullOneError::from_retry( format!("续传后的 ZIP 结构校验失败 {url}:{error};清理后重新下载也失败:{retry_error}"), retry_error, ) })?; status = OfficialResourcePullStatus::Downloaded; } } else { self.download_one(url, &partial, false).map_err(|error| { PullOneError::from_retry(format!("下载失败 {url}:{error}"), error) })?; } if let Err(error) = self.validate_zip_if_needed(url, &partial) { let _ = fs::remove_file(&partial); return Err(PullOneError::plain(error)); } ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; if file_len_if_exists(destination) .map_err(PullOneError::plain)? .is_some() { fs::remove_file(destination) .map_err(|error| format!("替换已有目标文件失败 {}:{error}", destination.display())) .map_err(PullOneError::plain)?; } ensure_safe_file_target(&self.output_root, destination, "下载目标文件") .map_err(PullOneError::plain)?; fs::rename(&partial, destination) .map_err(|error| { format!( "移动临时下载文件失败 {} -> {}:{error}", partial.display(), destination.display() ) }) .map_err(PullOneError::plain)?; let bytes = file_len_if_exists(destination) .map_err(PullOneError::plain)? .ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display())) .map_err(PullOneError::plain)?; Ok(PullOneResult { bytes, transferred_bytes: if status == OfficialResourcePullStatus::Resumed { bytes.saturating_sub(partial_len_before) } else { bytes }, status, }) } fn download_one( &self, url: &str, destination: &Path, resume: bool, ) -> Result<(), CurlRetryError> { run_curl_with_retry_with_proxy( &self.curl_command, url, Some(destination), self.retry_attempts, &self.curl_proxy, || { let mut command = Command::new(&self.curl_command); command .arg("--fail") .arg("--location") .arg("--silent") .arg("--show-error") .arg("--output") .arg(destination); if resume { command.arg("--continue-at").arg("-"); } command.arg("--url").arg(url); command }, ) .map(|_| ()) } fn verify_ready_official_hashes( &self, pairs: &[OfficialSeedHashPair], processed_urls: &HashSet, verified_hash_urls: &mut HashSet, verified_hashes: &mut Vec, manifest: &mut OfficialDownloadManifest, ) -> Result<(), String> { for pair in pairs { if verified_hash_urls.contains(&pair.hash_url) { continue; } if !processed_urls.contains(&pair.data_url) || !processed_urls.contains(&pair.hash_url) { continue; } if !self.hash_pair_files_exist(pair)? { continue; } let verification = match self.verify_official_seed_hash_pair_from_disk(pair) { Ok(verification) => verification, Err(error) => { if let Err(cleanup_error) = self.invalidate_download_manifest_hash_pair(manifest, pair) { return Err(format!( "{error};同时清理官方 hash 对应的本地 manifest 条目失败:{cleanup_error}" )); } return Err(error); } }; verified_hashes.push(verification); verified_hash_urls.insert(pair.hash_url.clone()); } Ok(()) } fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result { let data_path = self.destination_for_url(&pair.data_url)?; let hash_path = self.destination_for_url(&pair.hash_url)?; Ok(file_len_if_exists(&data_path)?.is_some() && file_len_if_exists(&hash_path)?.is_some()) } fn verify_official_seed_hash_pair_from_disk( &self, pair: &OfficialSeedHashPair, ) -> Result { let data_path = self.destination_for_url(&pair.data_url)?; let hash_path = self.destination_for_url(&pair.hash_url)?; let data = fs::read(&data_path).map_err(|error| { format!( "读取官方 hash 对应数据文件失败 {}:{error}", data_path.display() ) })?; let hash = fs::read(&hash_path).map_err(|error| { format!( "读取官方 hash sidecar 失败 {}:{error}", hash_path.display() ) })?; verify_official_seed_catalog_hash(&pair.data_url, &data, &pair.hash_url, &hash) } fn invalidate_download_manifest_hash_pair( &self, manifest: &mut OfficialDownloadManifest, pair: &OfficialSeedHashPair, ) -> Result<(), String> { manifest.entries.remove(&pair.data_url); manifest.entries.remove(&pair.hash_url); self.write_download_manifest(manifest) } fn verify_all_official_hashes_are_complete( &self, pairs: &[OfficialSeedHashPair], verified_hash_urls: &HashSet, ) -> Result<(), String> { for pair in pairs { if !verified_hash_urls.contains(&pair.hash_url) { return Err(format!( "官方 hash 对未完成校验:数据={} hash={}", pair.data_url, pair.hash_url )); } } Ok(()) } fn read_download_manifest(&self) -> Result { self.ensure_output_root_safe()?; let path = self.download_manifest_path(); ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?; let Some(bytes) = read_file_no_symlink(&path, "下载 manifest")? else { return Ok(OfficialDownloadManifest::default()); }; let manifest: OfficialDownloadManifest = serde_json::from_slice(&bytes) .map_err(|error| format!("解析下载 manifest 失败 {}:{error}", path.display()))?; if manifest.version != DOWNLOAD_MANIFEST_VERSION { return Err(format!( "不支持的下载 manifest 版本 {},文件 {}", manifest.version, path.display() )); } Ok(manifest) } fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> { self.ensure_output_root_ready()?; let path = self.download_manifest_path(); if let Some(parent) = path.parent() { ensure_safe_directory_path(parent, "下载 manifest 目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载 manifest 目录失败 {}:{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载 manifest 目录")?; } ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?; let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| format!("序列化下载 manifest 失败 {}:{error}", path.display()))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 manifest")?; Ok(()) } fn read_download_quarantine(&self) -> Result { self.ensure_output_root_safe()?; let path = self.download_quarantine_path(); ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; let Some(bytes) = read_file_no_symlink(&path, "下载 quarantine")? else { return Ok(OfficialDownloadQuarantineManifest::default()); }; let manifest: OfficialDownloadQuarantineManifest = serde_json::from_slice(&bytes) .map_err(|error| format!("解析下载 quarantine 失败 {}:{error}", path.display()))?; if manifest.version != DOWNLOAD_QUARANTINE_VERSION { return Err(format!( "不支持的下载 quarantine 版本 {},文件 {}", manifest.version, path.display() )); } Ok(manifest) } fn write_download_quarantine( &self, manifest: &OfficialDownloadQuarantineManifest, ) -> Result<(), String> { self.ensure_output_root_ready()?; let path = self.download_quarantine_path(); if let Some(parent) = path.parent() { ensure_safe_directory_path(parent, "下载 quarantine 目录")?; fs::create_dir_all(parent).map_err(|error| { format!("创建下载 quarantine 目录失败 {}:{error}", parent.display()) })?; ensure_safe_directory_path(parent, "下载 quarantine 目录")?; } ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?; let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| format!("序列化下载 quarantine 失败 {}:{error}", path.display()))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 quarantine")?; Ok(()) } fn record_quarantine_entry( &self, url: &str, destination: &Path, error: &PullOneError, ) -> Result<(), String> { let mut manifest = self.read_download_quarantine()?; manifest.entries.insert( url.to_string(), OfficialDownloadQuarantineEntry { url: url.to_string(), destination: self.relative_destination(destination)?, failed_at_unix_seconds: unix_seconds_now(), attempts: error.attempts().unwrap_or(0), failure_kind: error.failure_kind().map(ToOwned::to_owned), http_status: error.http_status(), retryable: error.retryable(), skipped_this_run: true, last_error: error.message.clone(), }, ); self.write_download_quarantine(&manifest) } fn clear_quarantine_entry(&self, url: &str) -> Result<(), String> { let mut manifest = self.read_download_quarantine()?; if manifest.entries.remove(url).is_none() { return Ok(()); } self.write_download_quarantine(&manifest) } fn validated_existing_file( &self, url: &str, destination: &Path, manifest: &OfficialDownloadManifest, ) -> Result, String> { let Some(entry) = manifest.entries.get(url) else { return Ok(None); }; if entry.url != url { return Ok(None); } if entry.destination != self.relative_destination(destination)? { return Ok(None); } let Some(bytes) = file_len_if_exists(destination)? else { return Ok(None); }; if bytes != entry.bytes { return Ok(None); } let digest = blake3_file_hex(destination)?; if digest != entry.blake3 { return Ok(None); } if let Err(_error) = self.validate_zip_if_needed(url, destination) { return Ok(None); } Ok(Some(PullOneResult { bytes, transferred_bytes: 0, status: OfficialResourcePullStatus::SkippedExisting, })) } fn record_download_manifest_entry( &self, manifest: &mut OfficialDownloadManifest, url: &str, destination: &Path, ) -> Result<(), String> { self.validate_zip_if_needed(url, destination)?; let bytes = file_len_if_exists(destination)? .ok_or_else(|| format!("读取已下载文件信息失败 {}", destination.display()))?; let digest = blake3_file_hex(destination)?; let relative_destination = self.relative_destination(destination)?; manifest.entries.insert( url.to_string(), OfficialDownloadManifestEntry { url: url.to_string(), destination: relative_destination, bytes, blake3: digest, }, ); Ok(()) } fn validate_zip_if_needed(&self, url: &str, destination: &Path) -> Result<(), String> { ensure_safe_file_target(&self.output_root, destination, "ZIP 校验目标")?; if !url_or_path_has_zip_extension(url) && !path_has_zip_extension(destination) { return Ok(()); } validate_zip_structure(destination).map(|_report| ()) } fn relative_destination(&self, destination: &Path) -> Result { let relative = destination .strip_prefix(&self.output_root) .map_err(|error| { format!( "目标路径 {} 不在输出目录 {} 下:{error}", destination.display(), self.output_root.display() ) })?; Ok(relative.to_string_lossy().replace('\\', "/")) } fn audit_one( &self, url: &str, destination: PathBuf, manifest: &OfficialDownloadManifest, ) -> Result { let Some(entry) = manifest.entries.get(url) else { let actual_bytes = file_len_if_exists(&destination)?; return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::MissingManifestEntry, manifest_destination: None, expected_bytes: None, actual_bytes, expected_blake3: None, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); }; let manifest_destination = Some(entry.destination.clone()); let expected_bytes = Some(entry.bytes); let expected_blake3 = Some(entry.blake3.clone()); if entry.url != url { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::UrlMismatch, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let expected_destination = self.relative_destination(&destination)?; if entry.destination != expected_destination { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::DestinationMismatch, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let Some(actual_bytes) = file_len_if_exists(&destination)? else { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::MissingFile, manifest_destination, expected_bytes, actual_bytes: None, expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); }; if actual_bytes != entry.bytes { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::SizeMismatch, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: None, zip_error: None, zip_structure_verified: false, }); } let actual_blake3 = blake3_file_hex(&destination)?; if actual_blake3 != entry.blake3 { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::Blake3Mismatch, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: None, zip_structure_verified: false, }); } let zip_structure_required = url_or_path_has_zip_extension(url) || path_has_zip_extension(&destination); if let Err(error) = self.validate_zip_if_needed(url, &destination) { return Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::ZipStructureInvalid, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: Some(error), zip_structure_verified: false, }); } Ok(OfficialLocalManifestAuditItem { url: url.to_string(), destination, status: OfficialLocalManifestAuditStatus::Verified, manifest_destination, expected_bytes, actual_bytes: Some(actual_bytes), expected_blake3, actual_blake3: Some(actual_blake3), zip_error: None, zip_structure_verified: zip_structure_required, }) } fn destination_for_url(&self, url: &str) -> Result { self.ensure_output_root_safe()?; if !is_official_yostar_jp_url(url) { return Err(format!("URL 不是官方 JP host:{url}")); } let rest = url .strip_prefix("https://") .ok_or_else(|| format!("官方 URL 必须使用 https:{url}"))?; let (host, path) = rest .split_once('/') .ok_or_else(|| format!("官方 URL 缺少路径:{url}"))?; let mut relative_destination = PathBuf::from(sanitize_segment(host)); for segment in path.split('/') { if segment.is_empty() { continue; } if segment == "." || segment == ".." { return Err(format!("官方 URL 包含不安全路径片段:{url}")); } // 官方资源 URL 不携带 query/fragment。若出现则直接拒绝,而非静默剥除—— // 否则仅 query 不同的两个 URL 会映射到同一目标文件而相互覆盖, // 并导致每轮 hash 复用校验失配、反复重下。 if segment.contains('?') || segment.contains('#') { return Err(format!("官方资源 URL 不允许包含 query 或 fragment:{url}")); } relative_destination.push(sanitize_segment(segment)); } let destination = self.output_root.join(relative_destination); ensure_path_within_root(&self.output_root, &destination)?; Ok(destination) } fn ensure_output_root_safe(&self) -> Result<(), String> { validate_output_root(&self.output_root) } fn ensure_output_root_ready(&self) -> Result<(), String> { self.ensure_output_root_safe()?; ensure_safe_directory_path(&self.output_root, "资源输出目录")?; fs::create_dir_all(&self.output_root).map_err(|error| { format!( "创建资源输出目录失败 {}:{error}", self.output_root.display() ) })?; ensure_safe_directory_path(&self.output_root, "资源输出目录") } } /// Verifies an official seed catalog `.bytes` payload against its official /// `.hash` sidecar. /// /// The currently verified seed catalogs are `TableCatalog.bytes`, /// `BundlePackingInfo.bytes`, and `MediaCatalog.bytes`; their `.hash` files are /// decimal text `xxHash32` values with seed `0`. pub fn verify_official_seed_catalog_hash( data_url: &str, data: &[u8], hash_url: &str, hash_bytes: &[u8], ) -> Result { let expected = parse_official_xxhash32_decimal(hash_url, hash_bytes)?; let actual = xxhash32(data); if actual != expected { return Err(format!( "官方 hash 校验失败 {data_url}:期望 {expected}(来自 {hash_url}),实际 {actual}" )); } Ok(OfficialResourceHashVerification { data_url: data_url.to_string(), hash_url: hash_url.to_string(), algorithm: OfficialResourceHashAlgorithm::XxHash32Decimal, expected: expected.to_string(), actual: actual.to_string(), }) } #[derive(Debug, Clone, PartialEq, Eq)] struct OfficialSeedHashPair { data_url: String, hash_url: String, } fn official_seed_hash_pairs(plan: &OfficialResourcePullPlan) -> Vec { plan.discovery .endpoints .iter() .filter_map(|endpoint| { let hash_kind = strong_seed_hash_kind(endpoint.kind)?; let hash_endpoint = plan.discovery.endpoints.iter().find(|candidate| { candidate.kind == hash_kind && candidate.platform == endpoint.platform })?; Some(OfficialSeedHashPair { data_url: endpoint.url.clone(), hash_url: hash_endpoint.url.clone(), }) }) .collect() } fn local_manifest_seed_hash_pairs( manifest: &OfficialDownloadManifest, ) -> Vec { manifest .entries .keys() .filter_map(|data_url| { let hash_url = data_url .strip_suffix(".bytes") .map(|prefix| format!("{prefix}.hash"))?; if !manifest.entries.contains_key(&hash_url) { return None; } Some(OfficialSeedHashPair { data_url: data_url.clone(), hash_url, }) }) .collect() } fn official_hash_refresh_urls(pairs: &[OfficialSeedHashPair]) -> HashSet { let mut urls = HashSet::new(); for pair in pairs { urls.insert(pair.data_url.clone()); urls.insert(pair.hash_url.clone()); } urls } fn strong_seed_hash_kind( kind: YostarJpResourceEndpointKind, ) -> Option { match kind { YostarJpResourceEndpointKind::TableCatalog => { Some(YostarJpResourceEndpointKind::TableCatalogHash) } YostarJpResourceEndpointKind::BundlePackingInfo => { Some(YostarJpResourceEndpointKind::BundlePackingInfoHash) } YostarJpResourceEndpointKind::MediaCatalog => { Some(YostarJpResourceEndpointKind::MediaCatalogHash) } YostarJpResourceEndpointKind::AddressablesCatalog | YostarJpResourceEndpointKind::TableCatalogHash | YostarJpResourceEndpointKind::AddressablesCatalogHash | YostarJpResourceEndpointKind::BundlePackingInfoHash | YostarJpResourceEndpointKind::MediaCatalogHash => None, } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadManifest { #[serde(default = "default_download_manifest_version")] version: u32, #[serde(default)] entries: BTreeMap, } impl Default for OfficialDownloadManifest { fn default() -> Self { Self { version: DOWNLOAD_MANIFEST_VERSION, entries: BTreeMap::new(), } } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadManifestEntry { url: String, destination: String, bytes: u64, blake3: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadQuarantineManifest { #[serde(default = "default_download_quarantine_version")] version: u32, #[serde(default)] entries: BTreeMap, } impl Default for OfficialDownloadQuarantineManifest { fn default() -> Self { Self { version: DOWNLOAD_QUARANTINE_VERSION, entries: BTreeMap::new(), } } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct OfficialDownloadQuarantineEntry { url: String, destination: String, failed_at_unix_seconds: u64, attempts: usize, failure_kind: Option, http_status: Option, retryable: Option, skipped_this_run: bool, last_error: String, } fn default_download_manifest_version() -> u32 { DOWNLOAD_MANIFEST_VERSION } fn default_download_quarantine_version() -> u32 { DOWNLOAD_QUARANTINE_VERSION } #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PullOneResult { bytes: u64, transferred_bytes: u64, status: OfficialResourcePullStatus, } #[derive(Debug, Clone, PartialEq, Eq)] struct PullOneError { message: String, retry_error: Option, } impl PullOneError { fn plain(message: String) -> Self { Self { message, retry_error: None, } } fn from_retry(message: String, retry_error: CurlRetryError) -> Self { Self { message, retry_error: Some(retry_error), } } fn failure_kind(&self) -> Option<&'static str> { self.retry_error .as_ref() .and_then(CurlRetryError::final_kind_label) } fn http_status(&self) -> Option { self.retry_error .as_ref() .and_then(CurlRetryError::final_http_status) } fn retryable(&self) -> Option { self.retry_error .as_ref() .and_then(CurlRetryError::final_retryable) } fn attempts(&self) -> Option { self.retry_error.as_ref().map(CurlRetryError::attempt_count) } } impl std::fmt::Display for PullOneError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str(&self.message) } } impl std::error::Error for PullOneError {} fn file_len_if_exists(path: &Path) -> Result, String> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { Err(format!("拒绝跟随 symlink 文件:{}", path.display())) } Ok(metadata) if metadata.is_file() => Ok(Some(metadata.len())), Ok(_) => Err(format!("期望文件路径,但实际不是文件:{}", path.display())), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(format!("读取路径信息失败 {}:{error}", path.display())), } } fn partial_path_for(destination: &Path) -> PathBuf { let mut partial = destination.as_os_str().to_owned(); partial.push(".part"); PathBuf::from(partial) } fn unix_seconds_now() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs()) .unwrap_or(0) } fn blake3_file_hex(path: &Path) -> Result { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { return Err(format!("拒绝读取 symlink 文件:{}", path.display())); } Ok(metadata) if metadata.is_file() => {} Ok(_) => return Err(format!("期望文件路径,但实际不是文件:{}", path.display())), Err(error) => return Err(format!("读取路径信息失败 {}:{error}", path.display())), } let mut file = File::open(path).map_err(|error| format!("打开文件失败 {}:{error}", path.display()))?; let mut hasher = blake3::Hasher::new(); let mut buffer = [0u8; 64 * 1024]; loop { let read = file .read(&mut buffer) .map_err(|error| format!("读取文件失败 {}:{error}", path.display()))?; if read == 0 { break; } hasher.update(&buffer[..read]); } Ok(hasher.finalize().to_hex().to_string()) } fn parse_official_xxhash32_decimal(hash_url: &str, bytes: &[u8]) -> Result { let text = std::str::from_utf8(bytes) .map_err(|error| format!("官方 hash sidecar 不是 UTF-8:{hash_url}:{error}"))? .trim(); text.parse::() .map_err(|error| format!("官方 hash sidecar 不是十进制 xxHash32:{hash_url}:{error}")) } fn xxhash32(bytes: &[u8]) -> u32 { const PRIME1: u32 = 0x9E37_79B1; const PRIME2: u32 = 0x85EB_CA77; const PRIME3: u32 = 0xC2B2_AE3D; const PRIME4: u32 = 0x27D4_EB2F; const PRIME5: u32 = 0x1656_67B1; let len = bytes.len(); let mut index = 0usize; let mut hash = if len >= 16 { let mut v1 = PRIME1.wrapping_add(PRIME2); let mut v2 = PRIME2; let mut v3 = 0; let mut v4 = 0u32.wrapping_sub(PRIME1); while index + 16 <= len { v1 = xxhash32_round(v1, read_u32_le(bytes, index)); v2 = xxhash32_round(v2, read_u32_le(bytes, index + 4)); v3 = xxhash32_round(v3, read_u32_le(bytes, index + 8)); v4 = xxhash32_round(v4, read_u32_le(bytes, index + 12)); index += 16; } v1.rotate_left(1) .wrapping_add(v2.rotate_left(7)) .wrapping_add(v3.rotate_left(12)) .wrapping_add(v4.rotate_left(18)) } else { PRIME5 } .wrapping_add(len as u32); while index + 4 <= len { hash = hash .wrapping_add(read_u32_le(bytes, index).wrapping_mul(PRIME3)) .rotate_left(17) .wrapping_mul(PRIME4); index += 4; } while index < len { hash = hash .wrapping_add((bytes[index] as u32).wrapping_mul(PRIME5)) .rotate_left(11) .wrapping_mul(PRIME1); index += 1; } xxhash32_avalanche(hash) } fn xxhash32_round(acc: u32, input: u32) -> u32 { const PRIME2: u32 = 0x85EB_CA77; const PRIME1: u32 = 0x9E37_79B1; acc.wrapping_add(input.wrapping_mul(PRIME2)) .rotate_left(13) .wrapping_mul(PRIME1) } fn xxhash32_avalanche(mut hash: u32) -> u32 { hash ^= hash >> 15; hash = hash.wrapping_mul(0x85EB_CA77); hash ^= hash >> 13; hash = hash.wrapping_mul(0xC2B2_AE3D); hash ^= hash >> 16; hash } fn read_u32_le(bytes: &[u8], offset: usize) -> u32 { u32::from_le_bytes([ bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3], ]) } fn sanitize_segment(segment: &str) -> String { segment .chars() .map(|ch| { if ch.is_control() || matches!(ch, '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|') { '_' } else { ch } }) .collect() } #[cfg(test)] mod tests { use super::*; use crate::official_pull::{ build_official_pull_plan, build_official_pull_plan_for_platforms, OfficialResourcePullPlan, }; use bat_adapters::official::inventory::{ YostarJpDownloadInventory, YostarJpPlatformDownloadInventory, }; use bat_adapters::official::yostar_jp::{ PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint, YostarJpResourceEndpointKind, }; use std::fs; use tempfile::TempDir; fn discovery_plan() -> YostarJpResourceDiscoveryPlan { YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: Some("s8tloc7lo3".to_string()), addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![ YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(), }, YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalogHash, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.hash".to_string(), }, ], } } fn inventory() -> YostarJpDownloadInventory { YostarJpDownloadInventory::from_catalog_bytes( b"FullPatch_000.zip", b"ExcelDB.db ExcelDB.db", b"JP_Airi.zip", ) } fn platform_inventory() -> YostarJpPlatformDownloadInventory { YostarJpPlatformDownloadInventory::from_shared_inventory( inventory(), &[PatchPlatform::Windows], ) } fn fake_curl_script() -> String { format!( r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" case "$url" in */TableBundles/TableCatalog.hash) printf '%s' '{table_catalog_hash}' > "$out" ;; */TableBundles/TableCatalog.bytes) printf '%s' 'ExcelDB.db' > "$out" ;; */Windows_PatchPack/BundlePackingInfo.hash) printf '%s' '{windows_bundle_hash}' > "$out" ;; */Android_PatchPack/BundlePackingInfo.hash) printf '%s' '{android_bundle_hash}' > "$out" ;; */Windows_PatchPack/BundlePackingInfo.bytes) printf '%s' 'FullPatch_000.zip' > "$out" ;; */Android_PatchPack/BundlePackingInfo.bytes) printf '%s' 'FullPatch_001.zip' > "$out" ;; */MediaResources-Windows/Catalog/MediaCatalog.hash) printf '%s' '{windows_media_hash}' > "$out" ;; */MediaResources/Catalog/MediaCatalog.hash) printf '%s' '{android_media_hash}' > "$out" ;; */MediaResources-Windows/Catalog/MediaCatalog.bytes) printf '%s' 'JP_Airi_Win.zip' > "$out" ;; */MediaResources/Catalog/MediaCatalog.bytes) printf '%s' 'JP_Airi_Android.zip' > "$out" ;; *.zip) printf '\120\113\003\004\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000file.txtok\120\113\001\002\024\000\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000file.txt\120\113\005\006\000\000\000\000\001\000\001\000\066\000\000\000\050\000\000\000\000\000' > "$out" ;; *) printf '%s' "$url" > "$out" ;; esac else printf '%s' "$url" fi "#, table_catalog_hash = xxhash32(b"ExcelDB.db"), windows_bundle_hash = xxhash32(b"FullPatch_000.zip"), android_bundle_hash = xxhash32(b"FullPatch_001.zip"), windows_media_hash = xxhash32(b"JP_Airi_Win.zip"), android_media_hash = xxhash32(b"JP_Airi_Android.zip"), ) } fn fake_resume_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" resume=0 while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --continue-at) resume=1 shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" if [ "$resume" -eq 1 ]; then printf '%s' "-resumed" >> "$out" else printf '%s' "$url" > "$out" fi else printf '%s' "$url" fi "# } fn fake_flaky_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done state="$0.state" count=0 if [ -f "$state" ]; then count="$(cat "$state")" fi count=$((count + 1)) printf '%s' "$count" > "$state" if [ "$count" -eq 1 ]; then printf '%s\n' "temporary failure" >&2 exit 22 fi if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" printf '%s' "$url" > "$out" else printf '%s' "$url" fi "# } fn fake_bad_hash_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" case "$url" in */TableBundles/TableCatalog.hash) printf '%s' '1' > "$out" ;; */TableBundles/TableCatalog.bytes) printf '%s' 'ExcelDB.db' > "$out" ;; *) printf '%s' "$url" > "$out" ;; esac else printf '%s' "$url" fi "# } fn fake_bad_zip_curl_script() -> &'static str { r#"#!/bin/sh set -eu out="" url="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2" shift 2 ;; --url) url="$2" shift 2 ;; *) shift ;; esac done if [ -n "$out" ]; then mkdir -p "$(dirname "$out")" printf '%s' 'not a zip archive' > "$out" else printf '%s' "$url" fi "# } fn fake_http_error_curl_script(status: u16) -> String { format!( r#"#!/bin/sh set -eu url="" while [ "$#" -gt 0 ]; do case "$1" in --url) url="$2" shift 2 ;; *) shift ;; esac done state="$0.state" count=0 if [ -f "$state" ]; then count="$(cat "$state")" fi count=$((count + 1)) printf '%s' "$count" > "$state" printf 'curl: (22) The requested URL returned error: {status}\n' >&2 exit 22 "# ) } fn write_shell_script(path: &Path, script: &str) { fs::write(path, script).unwrap(); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; let mut permissions = fs::metadata(path).unwrap().permissions(); permissions.set_mode(0o755); fs::set_permissions(path, permissions).unwrap(); } } fn write_fake_curl(path: &Path) { write_shell_script(path, &fake_curl_script()); } fn write_fake_resume_curl(path: &Path) { write_shell_script(path, fake_resume_curl_script()); } fn write_fake_flaky_curl(path: &Path) { write_shell_script(path, fake_flaky_curl_script()); } fn write_fake_bad_hash_curl(path: &Path) { write_shell_script(path, fake_bad_hash_curl_script()); } fn write_fake_bad_zip_curl(path: &Path) { write_shell_script(path, fake_bad_zip_curl_script()); } fn write_fake_http_error_curl(path: &Path, status: u16) { write_shell_script(path, &fake_http_error_curl_script(status)); } fn one_url_plan(url: &str) -> OfficialResourcePullPlan { OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), } } fn one_file_zip(name: &[u8], data: &[u8]) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(name.len() as u16).to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(name); bytes.extend_from_slice(data); let central_offset = bytes.len() as u32; bytes.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&20u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); bytes.extend_from_slice(&(name.len() as u16).to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(name); let central_size = bytes.len() as u32 - central_offset; bytes.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&1u16.to_le_bytes()); bytes.extend_from_slice(&1u16.to_le_bytes()); bytes.extend_from_slice(¢ral_size.to_le_bytes()); bytes.extend_from_slice(¢ral_offset.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes } #[test] fn downloads_verified_pull_plan_to_disk() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert!(report.total_bytes() > 0); assert_eq!(report.downloaded_count(), report.items.len()); assert_eq!(report.skipped_count(), 0); assert_eq!(report.resumed_count(), 0); assert_eq!(report.transferred_bytes(), report.total_bytes()); assert_eq!(report.items[0].url, plan.discovery.endpoints[0].url); for item in &report.items { assert!(item.destination.exists()); assert_eq!(item.status, OfficialResourcePullStatus::Downloaded); assert_eq!(item.bytes, item.transferred_bytes); } assert_eq!(report.official_hash_verified_count(), 1); assert_eq!( report.verified_hashes[0].algorithm, OfficialResourceHashAlgorithm::XxHash32Decimal ); assert_eq!(report.verified_hashes[0].expected, "2044170421"); assert_eq!(report.verified_hashes[0].actual, "2044170421"); let manifest = service.read_download_manifest().unwrap(); assert_eq!(manifest.version, DOWNLOAD_MANIFEST_VERSION); assert_eq!(manifest.entries.len(), report.items.len()); for item in &report.items { let entry = manifest.entries.get(&item.url).unwrap(); assert_eq!(entry.url, item.url); assert_eq!( entry.destination, service.relative_destination(&item.destination).unwrap() ); assert_eq!(entry.bytes, item.bytes); assert_eq!(entry.blake3, blake3_file_hex(&item.destination).unwrap()); } } #[test] fn rejects_dangerous_output_root() { let service = OfficialResourcePullService::new(PathBuf::from("/")); let error = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap_err(); assert!(error.contains("危险路径")); } #[test] fn rejects_resource_url_with_query_or_fragment() { let temp = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(temp.path().join("out")); for url in [ "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes?v=1", "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes#frag", ] { let error = service.destination_for_url(url).unwrap_err(); assert!( error.contains("query 或 fragment"), "unexpected error for {url}: {error}" ); } // 无 query 的正常资源 URL 仍可映射。 assert!(service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .is_ok()); } #[cfg(unix)] #[test] fn pull_rejects_symlink_parent_escape() { use std::os::unix::fs::symlink; let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let escape_dir = temp.path().join("escape"); let bin_dir = TempDir::new().unwrap(); fs::create_dir_all(&out_dir).unwrap(); fs::create_dir_all(&escape_dir).unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path); let destination = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); let host_dir = destination .ancestors() .find(|path| { path.file_name().and_then(|name| name.to_str()) == Some("prod-clientpatch.bluearchiveyostar.com") }) .unwrap() .to_path_buf(); fs::create_dir_all(&host_dir).unwrap(); symlink(&escape_dir, host_dir.join("r93_token")).unwrap(); let error = service .pull(&build_official_pull_plan(discovery_plan(), inventory())) .unwrap_err(); assert!(error.contains("symlink")); assert!(fs::read_dir(&escape_dir).unwrap().next().is_none()); } #[cfg(unix)] #[test] fn pull_rejects_symlink_partial_file() { use std::os::unix::fs::symlink; let temp = TempDir::new().unwrap(); let out_dir = temp.path().join("out"); let escape_file = temp.path().join("escape.txt"); let bin_dir = TempDir::new().unwrap(); fs::create_dir_all(&out_dir).unwrap(); fs::write(&escape_file, b"outside").unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes"; let destination = service.destination_for_url(url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); symlink(&escape_file, partial_path_for(&destination)).unwrap(); let error = service.pull_one(url, &destination).unwrap_err(); assert!(error.to_string().contains("symlink")); assert_eq!(fs::read(&escape_file).unwrap(), b"outside"); } #[test] fn local_manifest_audit_detects_and_repairs_corrupted_file() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let first_report = service.pull(&plan).unwrap(); let clean_audit = service.audit_local_manifest(&plan).unwrap(); assert!(clean_audit.is_clean()); assert_eq!(clean_audit.verified_count(), first_report.items.len()); assert_eq!(clean_audit.manifest_blake3_verified_count(), 7); assert_eq!(clean_audit.zip_structure_verified_count(), 4); let corrupted = first_report .items .iter() .find(|item| item.url.ends_with("/Windows_PatchPack/FullPatch_000.zip")) .unwrap(); let mut corrupted_bytes = fs::read(&corrupted.destination).unwrap(); corrupted_bytes[0] ^= 0xff; fs::write(&corrupted.destination, corrupted_bytes).unwrap(); let corrupted_audit = service.audit_local_manifest(&plan).unwrap(); assert!(!corrupted_audit.is_clean()); assert_eq!(corrupted_audit.repair_needed_count(), 1); assert!(corrupted_audit.items.iter().any(|item| { item.url == corrupted.url && item.status == OfficialLocalManifestAuditStatus::Blake3Mismatch })); let repair_report = service.pull(&plan).unwrap(); assert!(repair_report.downloaded_count() >= 1); assert!(service.audit_local_manifest(&plan).unwrap().is_clean()); } #[test] fn full_local_verification_checks_manifest_and_official_hash_pairs() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); service.pull(&plan).unwrap(); let verification = service.verify_local_download_manifest().unwrap(); assert!(verification.is_clean()); assert_eq!(verification.items.len(), 7); assert_eq!(verification.verified_count(), 7); assert_eq!(verification.manifest_blake3_verified_count(), 7); assert_eq!(verification.zip_structure_verified_count(), 4); assert_eq!(verification.official_hash_pair_count, 1); assert_eq!(verification.official_hash_verified_count, 1); let data_path = service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); fs::write(data_path, b"corrupted").unwrap(); let corrupted = service.verify_local_download_manifest().unwrap(); assert!(!corrupted.is_clean()); assert_eq!(corrupted.failure_count(), 1); assert_eq!(corrupted.official_hash_errors.len(), 1); } #[test] fn refreshes_official_hash_pairs_and_skips_manifest_verified_content() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let mut manifest = OfficialDownloadManifest::default(); for url in plan.all_urls().unwrap() { let destination = service.destination_for_url(&url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); let bytes = if url.ends_with("/TableBundles/TableCatalog.bytes") { b"ExcelDB.db".to_vec() } else if url.ends_with("/TableBundles/TableCatalog.hash") { b"2044170421".to_vec() } else if url.ends_with(".zip") { one_file_zip(b"fixture.txt", b"ok") } else { url.as_bytes().to_vec() }; fs::write(&destination, bytes).unwrap(); service .record_download_manifest_entry(&mut manifest, &url, &destination) .unwrap(); } service.write_download_manifest(&manifest).unwrap(); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert_eq!(report.skipped_count(), 5); assert_eq!(report.downloaded_count(), 2); assert_eq!(report.resumed_count(), 0); assert!(report.transferred_bytes() > 0); assert_eq!(report.official_hash_verified_count(), 1); assert_eq!( fs::read_to_string( service .destination_for_url( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap() ) .unwrap(), "ExcelDB.db" ); } #[test] fn redownloads_existing_files_without_manifest() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); for url in plan.all_urls().unwrap() { let destination = service.destination_for_url(&url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); fs::write(destination, b"stale-local-file").unwrap(); } let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert_eq!(report.skipped_count(), 0); assert_eq!(report.downloaded_count(), 7); assert_eq!(report.resumed_count(), 0); assert_eq!(report.official_hash_verified_count(), 1); assert!(report.transferred_bytes() > 0); assert!(report .items .iter() .all(|item| item.status == OfficialResourcePullStatus::Downloaded)); } #[test] fn local_resource_state_reports_manifest_entries_and_existing_files() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan(discovery_plan(), inventory()); let urls = plan.all_urls().unwrap(); let empty_state = service.local_resource_state(&plan).unwrap(); assert_eq!(empty_state.manifest_entry_count, 0); assert_eq!(empty_state.existing_file_count, 0); assert!(!empty_state.has_any_resources()); let first_destination = service.destination_for_url(&urls[0]).unwrap(); fs::create_dir_all(first_destination.parent().unwrap()).unwrap(); fs::write(&first_destination, b"local-only").unwrap(); let file_only_state = service.local_resource_state(&plan).unwrap(); assert_eq!(file_only_state.manifest_entry_count, 0); assert_eq!(file_only_state.existing_file_count, 1); assert!(file_only_state.has_any_resources()); let mut manifest = OfficialDownloadManifest::default(); service .record_download_manifest_entry(&mut manifest, &urls[0], &first_destination) .unwrap(); service.write_download_manifest(&manifest).unwrap(); let manifest_state = service.local_resource_state(&plan).unwrap(); assert_eq!(manifest_state.manifest_entry_count, 1); assert_eq!(manifest_state.existing_file_count, 1); assert!(manifest_state.has_any_resources()); } #[test] fn redownloads_existing_file_when_manifest_hash_mismatches() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let first_report = service.pull(&plan).unwrap(); assert_eq!(first_report.downloaded_count(), first_report.items.len()); let first_item = &first_report.items[0]; fs::write(&first_item.destination, b"corrupted").unwrap(); let second_report = service .pull(&OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: first_item.url.clone(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }) .unwrap(); assert_eq!(second_report.items.len(), 1); assert_eq!(second_report.skipped_count(), 0); assert_eq!(second_report.downloaded_count(), 1); assert_eq!( fs::read_to_string(&first_item.destination).unwrap(), "ExcelDB.db" ); } #[test] fn rejects_seed_catalog_when_official_hash_mismatches() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_bad_hash_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = OfficialResourcePullPlan { discovery: discovery_plan(), inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let error = service.pull(&plan).unwrap_err(); assert!(error.contains("官方 hash 校验失败")); let manifest = service.read_download_manifest().unwrap(); assert!(manifest.entries.is_empty()); let audit = service.audit_local_manifest(&plan).unwrap(); assert!(!audit.is_clean()); assert_eq!(audit.repair_needed_count(), 2); assert!(audit .items .iter() .all(|item| { item.status == OfficialLocalManifestAuditStatus::MissingManifestEntry })); } #[test] fn verifies_known_real_seed_catalog_hash_samples() { assert_eq!(xxhash32(b""), 46947589); assert_eq!(xxhash32(b"ExcelDB.db"), 2044170421); assert_eq!(xxhash32(b"FullPatch_000.zip"), 4038880697); assert_eq!(xxhash32(b"JP_Airi_Win.zip"), 1416778215); assert_eq!(xxhash32(b"ExcelDB.db ExcelDB.db"), 2147704569); } #[test] fn rejects_downloaded_zip_when_structure_is_invalid() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_bad_zip_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let zip_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip"; let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::AddressablesCatalog, platform: Some(PatchPlatform::Windows), url: zip_url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let error = service.pull(&plan).unwrap_err(); assert!(error.contains("ZIP")); assert!(!partial_path_for(&service.destination_for_url(zip_url).unwrap()).exists()); assert!(service.read_download_manifest().unwrap().entries.is_empty()); } #[test] fn local_manifest_audit_rejects_zip_with_matching_size_and_blake3_but_bad_structure() { let out_dir = TempDir::new().unwrap(); let service = OfficialResourcePullService::new(out_dir.path()); let zip_url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip"; let destination = service.destination_for_url(zip_url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); fs::write(&destination, b"not a zip archive").unwrap(); let mut manifest = OfficialDownloadManifest::default(); let bytes = fs::metadata(&destination).unwrap().len(); let blake3 = blake3_file_hex(&destination).unwrap(); manifest.entries.insert( zip_url.to_string(), OfficialDownloadManifestEntry { url: zip_url.to_string(), destination: service.relative_destination(&destination).unwrap(), bytes, blake3, }, ); service.write_download_manifest(&manifest).unwrap(); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::AddressablesCatalog, platform: Some(PatchPlatform::Windows), url: zip_url.to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let audit = service.audit_local_manifest(&plan).unwrap(); let verification = service.verify_local_download_manifest().unwrap(); assert_eq!( audit.items[0].status, OfficialLocalManifestAuditStatus::ZipStructureInvalid ); assert!(audit.items[0].zip_error.as_ref().unwrap().contains("ZIP")); assert!(!verification.is_clean()); assert_eq!(verification.failure_count(), 1); } #[test] fn resumes_partial_files_before_atomic_completion() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_resume_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let first_url = plan.all_urls().unwrap().remove(0); let destination = service.destination_for_url(&first_url).unwrap(); fs::create_dir_all(destination.parent().unwrap()).unwrap(); let partial = partial_path_for(&destination); fs::write(&partial, b"partial").unwrap(); let report = service .pull(&OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token" .to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: first_url, }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }) .unwrap(); assert_eq!(report.items.len(), 1); assert_eq!(report.resumed_count(), 1); assert_eq!(report.items[0].status, OfficialResourcePullStatus::Resumed); assert_eq!(fs::read_to_string(&destination).unwrap(), "partial-resumed"); assert!(!partial.exists()); let manifest = service.read_download_manifest().unwrap(); assert_eq!(manifest.entries.len(), 1); assert!(manifest.entries.contains_key(&report.items[0].url)); } #[test] fn pull_emits_started_and_finished_progress_events() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows], ); let expected_urls = plan.all_urls().unwrap().len(); let mut events = Vec::new(); let report = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap(); assert_eq!(report.items.len(), expected_urls); assert_eq!(events.len(), expected_urls * 2); assert_eq!(events[0].kind, OfficialResourcePullProgressKind::Started); assert_eq!(events[0].index, 1); assert_eq!(events[0].total, expected_urls); assert_eq!(events[1].kind, OfficialResourcePullProgressKind::Finished); assert_eq!( events[1].status, Some(OfficialResourcePullStatus::Downloaded) ); assert!(events .iter() .any(|event| event.url.ends_with("/TableBundles/ExcelDB.db"))); } #[test] fn retries_transient_download_failures() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_flaky_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(2); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(), }], }, inventory: YostarJpPlatformDownloadInventory::from_shared_inventory( YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""), &[], ), platforms: Vec::new(), }; let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 1); assert_eq!(report.downloaded_count(), 1); assert_eq!( fs::read_to_string(&report.items[0].destination).unwrap(), report.items[0].url ); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "2" ); } #[test] fn does_not_retry_terminal_http_404_and_records_quarantine() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_http_error_curl(&curl_path, 404); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Missing.bytes"; let plan = one_url_plan(url); let mut events = Vec::new(); let error = service .pull_with_progress(&plan, |event| events.push(event)) .unwrap_err(); assert!(error.contains("quarantine")); assert!(error.contains("http_not_found")); assert!(error.contains("retryable=false")); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "1" ); let failed = events .iter() .find(|event| event.kind == OfficialResourcePullProgressKind::Failed) .unwrap(); assert_eq!(failed.failure_kind.as_deref(), Some("http_not_found")); assert_eq!(failed.failure_http_status, Some(404)); assert_eq!(failed.failure_retryable, Some(false)); assert_eq!(failed.failure_attempts, Some(1)); assert!(failed.quarantined); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.http_status, Some(404)); assert_eq!(entry.failure_kind.as_deref(), Some("http_not_found")); assert_eq!(entry.retryable, Some(false)); assert_eq!(entry.attempts, 1); assert!(entry.skipped_this_run); assert!(service.read_download_manifest().unwrap().entries.is_empty()); } #[test] fn official_http_failure_regression_fixtures_are_enforced() { for fixture in [ include_str!("../tests/fixtures/official_regression/http_403.json"), include_str!("../tests/fixtures/official_regression/http_404.json"), ] { let fixture: serde_json::Value = serde_json::from_str(fixture).unwrap(); let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); let status = fixture["http_status"].as_u64().unwrap() as u16; write_fake_http_error_curl(&curl_path, status); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = fixture["url"].as_str().unwrap(); let error = service.pull(&one_url_plan(url)).unwrap_err(); let expected_kind = fixture["failure_kind"].as_str().unwrap(); let expected_retryable = fixture["retryable"].as_bool().unwrap(); let expected_attempts = fixture["expected_attempts"].as_u64().unwrap(); assert!(error.contains(expected_kind)); assert!(error.contains(&format!("retryable={expected_retryable}"))); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), expected_attempts.to_string() ); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.failure_kind.as_deref(), Some(expected_kind)); assert_eq!(entry.retryable, Some(expected_retryable)); assert_eq!(entry.attempts, expected_attempts as usize); } } #[test] fn official_hash_mismatch_regression_fixture_is_enforced() { let fixture: serde_json::Value = serde_json::from_str(include_str!( "../tests/fixtures/official_regression/hash_mismatch_catalog.json" )) .unwrap(); let error = verify_official_seed_catalog_hash( fixture["data_url"].as_str().unwrap(), fixture["data"].as_str().unwrap().as_bytes(), fixture["hash_url"].as_str().unwrap(), fixture["official_hash"].as_str().unwrap().as_bytes(), ) .unwrap_err(); assert!(error.contains(fixture["expected_error_contains"].as_str().unwrap())); } #[test] fn retries_http_5xx_before_quarantine() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_http_error_curl(&curl_path, 503); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(3); let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Busy.bytes"; let plan = one_url_plan(url); let error = service.pull(&plan).unwrap_err(); assert!(error.contains("quarantine")); assert!(error.contains("http_server_error")); assert!(error.contains("retryable=true")); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "3" ); let quarantine = service.read_download_quarantine().unwrap(); let entry = quarantine.entries.get(url).unwrap(); assert_eq!(entry.http_status, Some(503)); assert_eq!(entry.failure_kind.as_deref(), Some("http_server_error")); assert_eq!(entry.retryable, Some(true)); assert_eq!(entry.attempts, 3); assert!(entry.last_error.contains("重试次数已耗尽")); } #[test] fn custom_platform_selection_is_supported() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let plan = build_official_pull_plan_for_platforms( discovery_plan(), inventory(), &[PatchPlatform::Windows, PatchPlatform::Android], ); let report = service.pull(&plan).unwrap(); assert_eq!(report.items.len(), 7); assert!(report .items .iter() .any(|item| item.url.contains("Android_PatchPack"))); } #[test] fn fetches_official_seed_bytes_with_curl() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path); let bytes = service .fetch_bytes( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes" ); } #[test] fn retries_transient_fetch_bytes_failures() { let out_dir = TempDir::new().unwrap(); let bin_dir = TempDir::new().unwrap(); let curl_path = bin_dir.path().join("curl"); write_fake_flaky_curl(&curl_path); let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path) .with_retry_attempts(2); let bytes = service .fetch_bytes( "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes", ) .unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes" ); assert_eq!( fs::read_to_string(curl_path.with_extension("state")).unwrap(), "2" ); } #[test] fn rejects_non_official_urls() { let service = OfficialResourcePullService::new("/tmp/unused"); let plan = OfficialResourcePullPlan { discovery: YostarJpResourceDiscoveryPlan { connection_group_name: "Prod-Audit".to_string(), app_version: "1.70.0".to_string(), bundle_version: None, addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(), endpoints: vec![YostarJpResourceEndpoint { kind: YostarJpResourceEndpointKind::TableCatalog, platform: None, url: "https://prod-clientpatch.bluearchive.cafe/r93_token/TableBundles/TableCatalog.bytes".to_string(), }], }, inventory: platform_inventory(), platforms: vec![PatchPlatform::Windows], }; assert!(service.pull(&plan).is_err()); } }