Files
BlueArchiveToolkit/infrastructure/src/official_download.rs
T
nyaKazuha 30d1cd77e8
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s
fix(release): 修复发布一致性与并发边界
2026-09-12 16:35:55 +08:00

4884 lines
182 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Official JP resource download execution.
use crate::curl_transfer::{run_curl_with_retry_with_proxy, CurlProxyConfig, CurlRetryError};
use crate::downloader::{
DownloadScheduler, DownloaderBackend, DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY,
MIN_DOWNLOAD_CONCURRENCY,
};
use crate::official_pull::OfficialResourcePullPlan;
use crate::path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target,
read_file_no_symlink, validate_output_root, write_file_atomic, STATE_FILE_MODE,
};
use crate::zip_validation::{
path_has_zip_extension, url_or_path_has_zip_extension, validate_zip_structure,
};
use bat_adapters::official::yostar_jp::YostarJpResourceEndpointKind;
use bat_adapters::official::{
destination_under_root, DownloadUrlMapper, OfficialResourceBackend, SidecarHashStrategy,
XxHash32DecimalSeedZero, YostarJpBackend,
};
use bat_core::repositories::CasRepository;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
/// 官方资源下载错误:携带统一错误码,便于 CLI/RPC 归类。
///
/// 下载链路内部大量使用 `Result<_, String>`;这些经 `From<String>` 归入
/// `internal`,只有 curl 下载失败等有明确来源的错误会带上准确的网络域码。
#[derive(Debug)]
pub struct DownloadError {
code: bat_core::ErrorCode,
message: String,
}
impl DownloadError {
pub(crate) fn new(code: bat_core::ErrorCode, message: impl Into<String>) -> Self {
Self {
code,
message: message.into(),
}
}
/// 该错误的统一错误码。
pub fn code(&self) -> bat_core::ErrorCode {
self.code
}
}
impl From<String> for DownloadError {
fn from(message: String) -> Self {
Self::new(bat_core::ErrorCode::INTERNAL, message)
}
}
impl std::fmt::Display for DownloadError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for DownloadError {}
const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json";
const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json";
/// 记录一个已发布官方 release 获取的 CAS 引用。
pub const OFFICIAL_CAS_REUSE_REFERENCES_FILE: &str = "official-cas-reuse-references.json";
const OFFICIAL_CAS_REUSE_REFERENCES_VERSION: u32 = 1;
const DOWNLOAD_MANIFEST_VERSION: u32 = 1;
const DOWNLOAD_QUARANTINE_VERSION: u32 = 1;
const DEFAULT_RETRY_ATTEMPTS: usize = 3;
/// Outcome for one official resource pull item.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OfficialResourcePullStatus {
/// The destination already existed and passed local manifest validation.
SkippedExisting,
/// The resource was materialized from a previously published release.
ReleaseReused,
/// The resource was materialized from a CAS object.
CasReused,
/// A partial `.part` file was resumed.
Resumed,
/// The resource was downloaded from scratch.
Downloaded,
}
impl OfficialResourcePullStatus {
/// Returns a stable label for reports and progress logs.
pub fn as_str(self) -> &'static str {
match self {
Self::SkippedExisting => "skipped_existing",
Self::ReleaseReused => "release_reused",
Self::CasReused => "cas_reused",
Self::Resumed => "resumed",
Self::Downloaded => "downloaded",
}
}
}
/// Progress event kind emitted while executing an official pull plan.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OfficialResourcePullProgressKind {
/// A URL is about to be checked or reused/downloaded.
Started,
/// A URL finished as skipped, reused, resumed, or downloaded.
Finished,
/// An official sidecar hash pair passed verification.
Verification,
/// A URL failed after retry classification and was quarantined for this run.
Failed,
}
impl OfficialResourcePullProgressKind {
/// Returns a stable label for progress logs.
pub fn as_str(self) -> &'static str {
match self {
Self::Started => "started",
Self::Finished => "finished",
Self::Verification => "verification",
Self::Failed => "failed",
}
}
}
/// Local validation results for one completed resource.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceVerification {
/// Expected byte size from the local manifest, when reusing a file.
pub expected_bytes: Option<u64>,
/// Actual byte size observed after the transfer.
pub actual_bytes: u64,
/// Expected BLAKE3 from the local manifest, when reusing a file.
pub expected_blake3: Option<String>,
/// Actual BLAKE3 computed from the completed file.
pub actual_blake3: String,
/// Whether ZIP structure validation was required for this resource.
pub zip_checked: bool,
/// Whether the required ZIP structure validation passed.
pub zip_structure_verified: bool,
}
/// Progress for one URL in an official pull plan.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialResourcePullProgress {
/// Progress event kind.
pub kind: OfficialResourcePullProgressKind,
/// Monotonic completed item count for the whole pull.
///
/// `Started` events report the currently completed count before the URL
/// finishes; `Finished` events report the count after completion. This is
/// intentionally not the URL's plan position, so status percentages stay
/// monotonic even if execution order or skip/resume mix changes.
pub index: usize,
/// Total URL count in the pull plan.
pub total: usize,
/// Official URL currently being processed.
pub url: String,
/// Finished pull status, when `kind` is `Finished`.
pub status: Option<OfficialResourcePullStatus>,
/// Final local file size, when `kind` is `Finished`.
pub bytes: Option<u64>,
/// Bytes transferred during this run, when `kind` is `Finished`.
pub transferred_bytes: Option<u64>,
/// Local size/BLAKE3/ZIP validation for a completed URL.
pub verification: Option<OfficialResourceVerification>,
/// Official sidecar hash verification completed after this URL.
pub official_hash: Option<OfficialResourceHashVerification>,
/// Stable failure kind label, when `kind` is `Failed`.
pub failure_kind: Option<String>,
/// HTTP status parsed from curl stderr, when available.
pub failure_http_status: Option<u16>,
/// Whether retry policy considered the final failure retryable.
pub failure_retryable: Option<bool>,
/// Number of curl attempts that were executed before this failure.
pub failure_attempts: Option<usize>,
/// Whether the URL was written to the local quarantine manifest.
pub quarantined: bool,
}
impl OfficialResourcePullProgress {
fn started(index: usize, total: usize, url: String) -> Self {
Self {
kind: OfficialResourcePullProgressKind::Started,
index,
total,
url,
status: None,
bytes: None,
transferred_bytes: None,
verification: None,
official_hash: None,
failure_kind: None,
failure_http_status: None,
failure_retryable: None,
failure_attempts: None,
quarantined: false,
}
}
fn finished(
index: usize,
total: usize,
url: String,
status: OfficialResourcePullStatus,
bytes: u64,
transferred_bytes: u64,
verification: OfficialResourceVerification,
) -> Self {
Self {
kind: OfficialResourcePullProgressKind::Finished,
index,
total,
url,
status: Some(status),
bytes: Some(bytes),
transferred_bytes: Some(transferred_bytes),
verification: Some(verification),
official_hash: None,
failure_kind: None,
failure_http_status: None,
failure_retryable: None,
failure_attempts: None,
quarantined: false,
}
}
fn verification(
index: usize,
total: usize,
url: String,
official_hash: OfficialResourceHashVerification,
) -> Self {
Self {
kind: OfficialResourcePullProgressKind::Verification,
index,
total,
url,
status: None,
bytes: None,
transferred_bytes: None,
verification: None,
official_hash: Some(official_hash),
failure_kind: None,
failure_http_status: None,
failure_retryable: None,
failure_attempts: None,
quarantined: false,
}
}
fn failed(index: usize, total: usize, url: String, error: &PullOneError) -> Self {
Self {
kind: OfficialResourcePullProgressKind::Failed,
index,
total,
url,
status: None,
bytes: None,
transferred_bytes: None,
verification: None,
official_hash: None,
failure_kind: error.failure_kind().map(ToOwned::to_owned),
failure_http_status: error.http_status(),
failure_retryable: error.retryable(),
failure_attempts: error.attempts(),
quarantined: true,
}
}
}
/// Official hash algorithm used by a verified resource sidecar.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum OfficialResourceHashAlgorithm {
/// Decimal text form of `xxHash32` with seed `0`.
#[serde(rename = "xxhash32_decimal")]
XxHash32Decimal,
}
impl OfficialResourceHashAlgorithm {
/// Returns a stable algorithm label for reports and logs.
pub fn as_str(self) -> &'static str {
match self {
Self::XxHash32Decimal => "xxhash32_decimal",
}
}
}
/// Successful official hash verification for one downloaded resource.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceHashVerification {
/// Downloaded data URL that was verified.
pub data_url: String,
/// Official `.hash` URL used as the verification source.
pub hash_url: String,
/// Official hash algorithm.
pub algorithm: OfficialResourceHashAlgorithm,
/// Expected hash value read from the official `.hash` file.
pub expected: String,
/// Actual hash value computed from the downloaded data file.
pub actual: String,
}
/// One downloaded official resource.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourcePullItem {
/// Original URL.
pub url: String,
/// Local destination path under the output root.
pub destination: PathBuf,
/// Final local file size.
pub bytes: u64,
/// Bytes transferred during this pull execution.
pub transferred_bytes: u64,
/// Pull outcome.
pub status: OfficialResourcePullStatus,
/// CAS object referenced when this item was reused from CAS.
#[serde(default)]
pub cas_object_id: Option<String>,
}
/// Download report for one executed pull plan.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourcePullReport {
/// Downloaded resources in pull-plan order, independent of completion order.
pub items: Vec<OfficialResourcePullItem>,
/// Official hash sidecars successfully verified after download.
pub verified_hashes: Vec<OfficialResourceHashVerification>,
/// Non-fatal reuse diagnostics that fell back to the next source or network.
pub reuse_warnings: Vec<OfficialResourceReuseWarning>,
}
/// Non-fatal diagnostic emitted when a historical release or CAS candidate
/// could not be reused and the pull fell back to another source.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialResourceReuseWarning {
/// Current official URL being processed.
pub url: String,
/// Reuse source class, such as `historical_release` or `cas`.
pub source: String,
/// Human-readable diagnostic message.
pub message: String,
}
/// Versioned CAS references acquired while materializing one official release.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialCasReuseReferenceManifest {
/// 引用清单版本。
pub version: u32,
/// 每个 CAS 引用一项。允许重复,因为每个拉取项分别拥有一个引用。
pub object_ids: Vec<String>,
}
impl Default for OfficialCasReuseReferenceManifest {
fn default() -> Self {
Self {
version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION,
object_ids: Vec::new(),
}
}
}
/// Reads a release-local CAS reference manifest.
pub fn read_cas_reuse_reference_manifest_at(
release_root: &Path,
) -> Result<Option<OfficialCasReuseReferenceManifest>, String> {
let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 release CAS 引用清单")? else {
return Ok(None);
};
let manifest: OfficialCasReuseReferenceManifest =
serde_json::from_slice(&bytes).map_err(|error| {
format!(
"解析官方 release CAS 引用清单失败 {}{error}",
path.display()
)
})?;
if manifest.version != OFFICIAL_CAS_REUSE_REFERENCES_VERSION {
return Err(format!(
"不支持的官方 release CAS 引用清单版本 {},文件 {}",
manifest.version,
path.display()
));
}
Ok(Some(manifest))
}
/// Decrements and removes CAS references recorded for a release.
///
/// Each decrement is committed together with a durable `(release, ordinal)`
/// ownership record in CAS metadata. The release-local manifest remains a
/// resumable progress cursor, so a crash before its rewrite cannot decrement
/// the same ownership twice.
pub fn release_cas_reuse_references(release_root: &Path, cas_root: &Path) -> Result<usize, String> {
let Some(mut manifest) = read_cas_reuse_reference_manifest_at(release_root)? else {
return Ok(0);
};
let release_id = release_root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty() && *name != "." && *name != "..")
.ok_or_else(|| {
format!(
"无法从 release 路径确定 CAS ownership ID{}",
release_root.display()
)
})?
.to_string();
let objects_root = cas_root.join("objects");
let metadata_path = cas_root.join("metadata.sqlite");
require_existing_directory(cas_root, "CAS 根目录")?;
require_existing_directory(&objects_root, "CAS 对象目录")?;
require_existing_file(cas_root, &metadata_path, "CAS 元数据库")?;
let mut released = 0usize;
while let Some(object_id) = manifest.object_ids.pop() {
let ordinal = manifest.object_ids.len() as u64;
let cas_root = cas_root.to_path_buf();
let object_id_for_runtime = object_id.clone();
let release_id_for_runtime = release_id.clone();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| format!("创建 CAS 引用清理 runtime 失败:{error}"))?;
let did_release = runtime.block_on(async move {
let cas = crate::FileSystemCasRepository::new(cas_root);
cas.release_reference_once(&release_id_for_runtime, ordinal, &object_id_for_runtime)
.await
.map_err(|error| format!("减少 CAS release 引用失败 object={object_id}{error}"))
})?;
write_cas_reuse_reference_manifest(release_root, &manifest)?;
if did_release {
released += 1;
}
}
let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE);
match fs::symlink_metadata(&path) {
Ok(metadata) if metadata.file_type().is_symlink() => {
return Err(format!(
"官方 release CAS 引用清单不能是 symlink{}",
path.display()
))
}
Ok(_) => fs::remove_file(&path)
.map_err(|error| format!("删除空官方 release CAS 引用清单失败:{error}"))?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => {
return Err(format!(
"检查官方 release CAS 引用清单失败 {}{error}",
path.display()
))
}
}
Ok(released)
}
fn write_cas_reuse_reference_manifest(
release_root: &Path,
manifest: &OfficialCasReuseReferenceManifest,
) -> Result<(), String> {
let path = release_root.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE);
ensure_path_within_root(release_root, &path)?;
ensure_safe_file_target(release_root, &path, "官方 release CAS 引用清单")?;
let bytes = serde_json::to_vec_pretty(manifest)
.map_err(|error| format!("序列化官方 release CAS 引用清单失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 release CAS 引用清单")
}
impl OfficialResourcePullReport {
/// Returns the total downloaded byte count.
pub fn total_bytes(&self) -> u64 {
self.items.iter().map(|item| item.bytes).sum()
}
/// Returns bytes transferred by this pull execution.
pub fn transferred_bytes(&self) -> u64 {
self.items.iter().map(|item| item.transferred_bytes).sum()
}
/// Returns the number of resources reused from the local output tree.
pub fn skipped_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status == OfficialResourcePullStatus::SkippedExisting)
.count()
}
/// Returns the number of resources reused from historical releases.
pub fn release_reused_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status == OfficialResourcePullStatus::ReleaseReused)
.count()
}
/// Returns the number of resources reused from CAS.
pub fn cas_reused_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status == OfficialResourcePullStatus::CasReused)
.count()
}
/// Returns the byte count satisfied without network transfer.
pub fn reused_bytes(&self) -> u64 {
self.items
.iter()
.filter(|item| {
matches!(
item.status,
OfficialResourcePullStatus::SkippedExisting
| OfficialResourcePullStatus::ReleaseReused
| OfficialResourcePullStatus::CasReused
)
})
.map(|item| item.bytes)
.sum()
}
/// Returns the number of resources resumed from `.part` files.
pub fn resumed_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status == OfficialResourcePullStatus::Resumed)
.count()
}
/// Returns the number of resources downloaded from scratch.
pub fn downloaded_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status == OfficialResourcePullStatus::Downloaded)
.count()
}
/// Returns the number of official hash sidecars verified after download.
pub fn official_hash_verified_count(&self) -> usize {
self.verified_hashes.len()
}
}
/// Local download-manifest audit status for one expected official URL.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OfficialLocalManifestAuditStatus {
/// The local file exists and matches the recorded manifest size and BLAKE3.
Verified,
/// No manifest entry exists for this URL.
MissingManifestEntry,
/// The manifest entry URL does not match the expected URL.
UrlMismatch,
/// The manifest destination does not match the URL-derived destination.
DestinationMismatch,
/// The manifest entry exists but the local file is missing.
MissingFile,
/// The local file size differs from the manifest entry.
SizeMismatch,
/// The local file BLAKE3 differs from the manifest entry.
Blake3Mismatch,
/// The local file has a `.zip` URL or destination but its ZIP structure is
/// invalid.
ZipStructureInvalid,
}
impl OfficialLocalManifestAuditStatus {
/// Returns true when this status proves the local file can be reused.
pub fn is_verified(self) -> bool {
self == Self::Verified
}
/// Returns a stable status label for CLI and JSON reports.
pub fn as_str(self) -> &'static str {
match self {
Self::Verified => "verified",
Self::MissingManifestEntry => "missing_manifest_entry",
Self::UrlMismatch => "url_mismatch",
Self::DestinationMismatch => "destination_mismatch",
Self::MissingFile => "missing_file",
Self::SizeMismatch => "size_mismatch",
Self::Blake3Mismatch => "blake3_mismatch",
Self::ZipStructureInvalid => "zip_structure_invalid",
}
}
}
/// Local download-manifest audit result for one expected official URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialLocalManifestAuditItem {
/// Expected official URL from the pull plan.
pub url: String,
/// URL-derived local destination.
pub destination: PathBuf,
/// Audit status.
pub status: OfficialLocalManifestAuditStatus,
/// Manifest-recorded destination, when an entry exists.
pub manifest_destination: Option<String>,
/// Manifest-recorded byte size, when an entry exists.
pub expected_bytes: Option<u64>,
/// Actual local byte size, when the destination exists.
pub actual_bytes: Option<u64>,
/// Manifest-recorded BLAKE3, when an entry exists.
pub expected_blake3: Option<String>,
/// Actual local BLAKE3, when it could be computed.
pub actual_blake3: Option<String>,
/// ZIP structure validation error, when the URL or destination is a ZIP
/// and the local file did not pass ZIP structure validation.
pub zip_error: Option<String>,
/// Whether this item required and passed ZIP structure validation.
pub zip_structure_verified: bool,
}
/// Local download-manifest audit result for a pull plan.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialLocalManifestAuditReport {
/// Per-URL audit items in pull-plan order.
pub items: Vec<OfficialLocalManifestAuditItem>,
}
impl OfficialLocalManifestAuditReport {
/// Returns true when every expected URL was locally verified.
pub fn is_clean(&self) -> bool {
self.items.iter().all(|item| item.status.is_verified())
}
/// Returns the number of locally verified files.
pub fn verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status.is_verified())
.count()
}
/// Returns the number of files that require repair.
pub fn repair_needed_count(&self) -> usize {
self.items.len().saturating_sub(self.verified_count())
}
/// Returns the number of files that passed local manifest path, size, and
/// BLAKE3 validation.
pub fn manifest_blake3_verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status.is_verified())
.count()
}
/// Returns the number of ZIP files that passed structural validation.
pub fn zip_structure_verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.zip_structure_verified)
.count()
}
}
/// Full local verification report for all entries currently recorded in the
/// official download manifest.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialLocalVerificationReport {
/// Per-entry manifest audit results.
pub items: Vec<OfficialLocalManifestAuditItem>,
/// Official seed hash pairs found in the local manifest.
pub official_hash_pair_count: usize,
/// Official seed hash pairs that passed xxHash32 verification.
pub official_hash_verified_count: usize,
/// Official seed hash verification failures.
pub official_hash_errors: Vec<String>,
}
impl OfficialLocalVerificationReport {
/// Returns true when every local manifest entry and every local official
/// seed hash pair passed verification.
pub fn is_clean(&self) -> bool {
self.items.iter().all(|item| item.status.is_verified())
&& self.official_hash_errors.is_empty()
&& self.official_hash_verified_count == self.official_hash_pair_count
}
/// Returns the number of manifest entries that passed verification.
pub fn verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status.is_verified())
.count()
}
/// Returns the number of manifest entries that need attention.
pub fn failure_count(&self) -> usize {
self.items.len().saturating_sub(self.verified_count())
}
/// Returns the number of local manifest entries that passed path, size,
/// and BLAKE3 validation.
pub fn manifest_blake3_verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.status.is_verified())
.count()
}
/// Returns the number of ZIP files that passed structural validation.
pub fn zip_structure_verified_count(&self) -> usize {
self.items
.iter()
.filter(|item| item.zip_structure_verified)
.count()
}
}
/// Existing local state for an official pull plan.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct OfficialLocalResourceState {
/// Number of expected URLs that already have download-manifest entries.
pub manifest_entry_count: usize,
/// Number of expected URL destinations that already exist on disk.
pub existing_file_count: usize,
}
impl OfficialLocalResourceState {
/// Returns true when this plan has any reusable or auditable local state.
pub fn has_any_resources(self) -> bool {
self.manifest_entry_count > 0 || self.existing_file_count > 0
}
}
/// Executes an official pull plan with the system `curl` command.
#[derive(Debug, Clone)]
pub struct OfficialResourcePullService {
output_root: PathBuf,
backend: YostarJpBackend,
curl_command: PathBuf,
curl_proxy: CurlProxyConfig,
retry_attempts: usize,
max_concurrency: usize,
release_reuse_roots: Vec<PathBuf>,
cas_reuse_root: Option<PathBuf>,
cas_reference_tracker: Arc<Mutex<Vec<String>>>,
}
impl OfficialResourcePullService {
/// Creates a pull service that uses `curl` from `PATH`.
pub fn new(output_root: impl Into<PathBuf>) -> Self {
Self::with_curl_command(output_root, "curl")
}
/// Creates a pull service with an explicit `curl` binary path.
pub fn with_curl_command(
output_root: impl Into<PathBuf>,
curl_command: impl Into<PathBuf>,
) -> Self {
Self {
output_root: output_root.into(),
backend: YostarJpBackend,
curl_command: curl_command.into(),
curl_proxy: CurlProxyConfig::default(),
retry_attempts: DEFAULT_RETRY_ATTEMPTS,
max_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY,
release_reuse_roots: Vec::new(),
cas_reuse_root: None,
cas_reference_tracker: Arc::new(Mutex::new(Vec::new())),
}
}
/// Sets the proxy configuration for every `curl` transfer.
pub fn with_proxy_config(mut self, curl_proxy: CurlProxyConfig) -> Self {
self.curl_proxy = curl_proxy;
self
}
/// Sets the number of attempts for each `curl` transfer.
pub fn with_retry_attempts(mut self, retry_attempts: usize) -> Self {
self.retry_attempts = retry_attempts.max(1);
self
}
/// Sets the bounded number of concurrent downloads.
///
/// The default is [`DEFAULT_DOWNLOAD_CONCURRENCY`]. Values are kept
/// within the supported `1..=256` range; the public update configuration
/// validates input before constructing this service.
pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self {
self.max_concurrency =
max_concurrency.clamp(MIN_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY);
self
}
/// Enables best-effort reuse from immutable published release directories.
pub fn with_release_reuse_root(mut self, release_root: impl Into<PathBuf>) -> Self {
self.release_reuse_roots.push(release_root.into());
self
}
/// Enables best-effort reuse from several immutable release roots.
pub fn with_release_reuse_roots<I, P>(mut self, release_roots: I) -> Self
where
I: IntoIterator<Item = P>,
P: Into<PathBuf>,
{
self.release_reuse_roots
.extend(release_roots.into_iter().map(Into::into));
self
}
/// Enables best-effort reuse from an existing filesystem CAS root.
pub fn with_cas_reuse_root(mut self, cas_root: impl Into<PathBuf>) -> Self {
self.cas_reuse_root = Some(cas_root.into());
self
}
/// Returns the configured download concurrency.
pub fn max_concurrency(&self) -> usize {
self.max_concurrency
}
/// Returns the output root used for downloaded files.
pub fn output_root(&self) -> &Path {
&self.output_root
}
/// Returns the local manifest path used to validate completed downloads.
pub fn download_manifest_path(&self) -> PathBuf {
self.output_root.join(DOWNLOAD_MANIFEST_FILE)
}
/// Returns the local quarantine path used to diagnose failed downloads.
pub fn download_quarantine_path(&self) -> PathBuf {
self.output_root.join(DOWNLOAD_QUARANTINE_FILE)
}
fn track_cas_reference(&self, object_id: &str) {
if let Ok(mut references) = self.cas_reference_tracker.lock() {
references.push(object_id.to_string());
}
}
fn untrack_cas_reference(&self, object_id: &str) {
if let Ok(mut references) = self.cas_reference_tracker.lock() {
if let Some(index) = references.iter().position(|id| id == object_id) {
references.remove(index);
}
}
}
fn persist_cas_reuse_references(
&self,
report: &OfficialResourcePullReport,
) -> Result<(), String> {
let object_ids = report
.items
.iter()
.filter_map(|item| item.cas_object_id.clone())
.collect::<Vec<_>>();
if object_ids.is_empty() {
return Ok(());
}
let mut manifest =
read_cas_reuse_reference_manifest_at(&self.output_root)?.unwrap_or_default();
manifest.object_ids.extend(object_ids);
write_cas_reuse_reference_manifest(&self.output_root, &manifest)
}
/// Removes resource files recorded by an older manifest but absent from
/// the current pull plan.
///
/// This is intentionally limited to destinations owned by the previous
/// download manifest. Management files such as snapshots, version state,
/// and quarantine records are left untouched.
pub fn prune_stale_manifest_entries(
&self,
plan: &OfficialResourcePullPlan,
) -> Result<usize, String> {
self.ensure_output_root_ready()?;
let expected_urls = plan.all_urls()?.into_iter().collect::<HashSet<_>>();
let mut manifest = self.read_download_manifest()?;
let stale_entries = manifest
.entries
.iter()
.filter(|(url, _)| !expected_urls.contains(*url))
.map(|(url, entry)| (url.clone(), entry.destination.clone()))
.collect::<Vec<_>>();
for (url, relative_destination) in &stale_entries {
let destination = self.output_root.join(relative_destination);
self.remove_managed_destination(&destination, url)?;
self.remove_managed_destination(&partial_path_for(&destination), url)?;
manifest.entries.remove(url);
}
if !stale_entries.is_empty() {
self.write_download_manifest(&manifest)?;
}
Ok(stale_entries.len())
}
/// Executes the plan and downloads every official URL to disk.
pub fn pull(
&self,
plan: &OfficialResourcePullPlan,
) -> Result<OfficialResourcePullReport, DownloadError> {
self.pull_with_progress(plan, |_| {})
}
/// Executes the plan and calls `progress` for every URL before and after it
/// is checked or downloaded.
pub fn pull_with_progress(
&self,
plan: &OfficialResourcePullPlan,
mut progress: impl FnMut(OfficialResourcePullProgress),
) -> Result<OfficialResourcePullReport, DownloadError> {
self.pull_with_progress_and_cancellation(plan, &mut progress, || false)
}
/// Executes the plan, emits URL progress, and aborts between URLs when
/// `should_cancel` returns true.
pub fn pull_with_progress_and_cancellation(
&self,
plan: &OfficialResourcePullPlan,
mut progress: impl FnMut(OfficialResourcePullProgress),
mut should_cancel: impl FnMut() -> bool,
) -> Result<OfficialResourcePullReport, DownloadError> {
self.ensure_output_root_ready()?;
let cas_reference_guard = CasReferenceRollbackGuard::new(
self.cas_reuse_root.clone(),
Arc::clone(&self.cas_reference_tracker),
);
let official_hash_pairs = official_seed_hash_pairs(plan);
let force_refresh_urls = official_hash_refresh_urls(&official_hash_pairs);
let mut manifest = self.read_download_manifest()?;
let release_reuse_index = self.build_release_reuse_index()?;
let urls = plan.all_urls()?;
let total = urls.len();
// Phase A:无网络的前置校验(fail-fast)。逐个校验官方性、算目标路径、
// 建目录,并判定该 URL 是「已验证可跳过」还是「需下载」。任何非官方
// URL 在发起任何下载前即拒绝。
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let mut planned: Vec<PlannedDownload> = Vec::with_capacity(total);
let mut planned_destinations = HashSet::with_capacity(total);
for url in urls {
if !self.backend.is_official_url(&url) {
return Err(DownloadError::new(
bat_core::ErrorCode::NON_OFFICIAL_URL,
format!("拒绝下载非官方 URL{url}"),
));
}
let destination = self.destination_for_url(&url)?;
if !planned_destinations.insert(destination.clone()) {
return Err(DownloadError::new(
bat_core::ErrorCode::INVALID_ARGUMENT,
format!("官方资源拉取计划包含重复目标:{}", destination.display()),
));
}
if let Some(parent) = destination.parent() {
ensure_safe_directory_path(parent, "下载目标目录")?;
fs::create_dir_all(parent).map_err(|error| {
format!("创建下载目标目录失败 {}{error}", parent.display())
})?;
ensure_safe_directory_path(parent, "下载目标目录")?;
}
ensure_safe_file_target(&self.output_root, &destination, "下载目标文件")?;
let force_refresh = force_refresh_urls.contains(&url);
let existing = if force_refresh {
None
} else {
self.validated_existing_file(&url, &destination, &manifest)?
};
let reuse_candidate = if existing.is_none() && !force_refresh {
self.reuse_candidate_for_url(&url, &destination, &release_reuse_index)?
} else {
None
};
planned.push(PlannedDownload {
url,
destination,
existing,
reuse_candidate,
});
}
if self.max_concurrency > 1 {
let report = self.pull_planned_concurrently(
planned,
manifest,
official_hash_pairs,
release_reuse_index.warnings,
&mut progress,
&mut should_cancel,
)?;
self.persist_cas_reuse_references(&report)?;
cas_reference_guard.commit();
return Ok(report);
}
// Phase B:按 plan 顺序处理每个 URL。下载或复用完成并写入 manifest 后,
// 立即尝试校验已经到齐的官方 `.bytes/.hash` pair,避免把文件级问题延后到整轮末尾。
let mut completed_count = 0usize;
let mut items = Vec::with_capacity(planned.len());
let mut verified_hashes = Vec::new();
let mut reuse_warnings = release_reuse_index
.warnings
.into_iter()
.map(|message| OfficialResourceReuseWarning {
url: String::new(),
source: "historical_release_index".to_string(),
message,
})
.collect::<Vec<_>>();
let mut verified_hash_urls = HashSet::<String>::new();
let mut processed_urls = HashSet::<String>::new();
for item in &planned {
progress(OfficialResourcePullProgress::started(
completed_count,
total,
item.url.clone(),
));
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let result = if let Some(existing) = &item.existing {
self.clear_quarantine_entry(&item.url)?;
existing.clone()
} else {
match self.pull_task(item.clone()) {
Ok(mut pull_result) => {
reuse_warnings.append(&mut pull_result.reuse_warnings);
let verification_result = self
.clear_quarantine_entry(&item.url)
.and_then(|_| {
self.record_download_manifest_entry(
&mut manifest,
&item.url,
&item.destination,
)
})
.and_then(|verification| {
self.write_download_manifest(&manifest)
.map(|_| verification)
});
match verification_result {
Ok(verification) => {
pull_result.verification = verification;
}
Err(error) => {
let error =
PullOneError::plain(format!("记录下载 manifest 失败:{error}"));
self.record_quarantine_entry(&item.url, &item.destination, &error)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
&error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败:URL 已进入 quarantine,中止本轮同步、不发布不完整资源;url={} quarantine={}{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
}
pull_result
}
Err(error) => {
self.record_quarantine_entry(&item.url, &item.destination, &error)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
&error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败:URL 已进入 quarantine,中止本轮同步、不发布不完整资源;url={} quarantine={}{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
}
};
completed_count += 1;
progress(OfficialResourcePullProgress::finished(
completed_count,
total,
item.url.clone(),
result.status,
result.bytes,
result.transferred_bytes,
result.verification.clone(),
));
processed_urls.insert(item.url.clone());
let newly_verified_hashes = self.verify_ready_official_hashes(
&official_hash_pairs,
&processed_urls,
&mut verified_hash_urls,
&mut verified_hashes,
&mut manifest,
)?;
for verification in newly_verified_hashes {
progress(OfficialResourcePullProgress::verification(
completed_count,
total,
verification.data_url.clone(),
verification,
));
}
items.push(OfficialResourcePullItem {
url: item.url.clone(),
destination: item.destination.clone(),
bytes: result.bytes,
transferred_bytes: result.transferred_bytes,
status: result.status,
cas_object_id: result.cas_object_id.clone(),
});
}
self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?;
let report = OfficialResourcePullReport {
items,
verified_hashes,
reuse_warnings,
};
self.persist_cas_reuse_references(&report)?;
cas_reference_guard.commit();
Ok(report)
}
fn pull_planned_concurrently(
&self,
planned: Vec<PlannedDownload>,
mut manifest: OfficialDownloadManifest,
official_hash_pairs: Vec<OfficialSeedHashPair>,
mut reuse_warnings: Vec<String>,
progress: &mut impl FnMut(OfficialResourcePullProgress),
should_cancel: &mut impl FnMut() -> bool,
) -> Result<OfficialResourcePullReport, DownloadError> {
let total = planned.len();
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
for item in &planned {
progress(OfficialResourcePullProgress::started(
0,
total,
item.url.clone(),
));
}
let backend = CurlDownloadBackend { service: self };
let mut completed_count = 0usize;
let mut items_by_plan_index: Vec<Option<OfficialResourcePullItem>> =
(0..total).map(|_| None).collect();
let mut verified_hashes = Vec::new();
let mut reuse_warnings = reuse_warnings
.drain(..)
.map(|message| OfficialResourceReuseWarning {
url: String::new(),
source: "historical_release_index".to_string(),
message,
})
.collect::<Vec<_>>();
let mut verified_hash_urls = HashSet::<String>::new();
let mut processed_urls = HashSet::<String>::new();
DownloadScheduler::new(self.max_concurrency).execute_with_observer(
&backend,
planned.clone(),
|plan_index, result| {
let item = &planned[plan_index];
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let needs_manifest = item.existing.is_none();
let verification = match result {
Ok(pull_result) => {
reuse_warnings.extend(pull_result.reuse_warnings.clone());
let verification_result = self
.clear_quarantine_entry(&item.url)
.and_then(|_| {
if needs_manifest {
self.record_download_manifest_entry(
&mut manifest,
&item.url,
&item.destination,
)
} else {
Ok(pull_result.verification.clone())
}
})
.and_then(|verification| {
if needs_manifest {
self.write_download_manifest(&manifest)
.map(|_| verification)
} else {
Ok(verification)
}
});
match verification_result {
Ok(verification) => verification,
Err(error) => {
let error = PullOneError::plain(format!(
"记录下载 manifest 失败:{error}"
));
self.record_quarantine_entry(
&item.url,
&item.destination,
&error,
)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
&error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败:URL 已进入 quarantine,中止本轮同步、不发布不完整资源;url={} quarantine={}{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
}
}
Err(error) => {
self.record_quarantine_entry(&item.url, &item.destination, error)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败:URL 已进入 quarantine,中止本轮同步、不发布不完整资源;url={} quarantine={}{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
};
completed_count += 1;
progress(OfficialResourcePullProgress::finished(
completed_count,
total,
item.url.clone(),
result
.as_ref()
.expect("successful result handled above")
.status,
result
.as_ref()
.expect("successful result handled above")
.bytes,
result
.as_ref()
.expect("successful result handled above")
.transferred_bytes,
verification,
));
processed_urls.insert(item.url.clone());
let newly_verified_hashes = self.verify_ready_official_hashes(
&official_hash_pairs,
&processed_urls,
&mut verified_hash_urls,
&mut verified_hashes,
&mut manifest,
)?;
for verification in newly_verified_hashes {
progress(OfficialResourcePullProgress::verification(
completed_count,
total,
verification.data_url.clone(),
verification,
));
}
let pull_result = result
.as_ref()
.expect("successful result handled above");
items_by_plan_index[plan_index] = Some(OfficialResourcePullItem {
url: item.url.clone(),
destination: item.destination.clone(),
bytes: pull_result.bytes,
transferred_bytes: pull_result.transferred_bytes,
status: pull_result.status,
cas_object_id: pull_result.cas_object_id.clone(),
});
Ok(())
},
)?;
self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?;
let items = items_by_plan_index
.into_iter()
.enumerate()
.map(|(index, item)| {
item.ok_or_else(|| {
DownloadError::from(format!(
"并发下载结果缺少 plan index={index},拒绝发布不完整资源"
))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(OfficialResourcePullReport {
items,
verified_hashes,
reuse_warnings,
})
}
/// Audits every URL in a pull plan against the local download manifest.
///
/// This performs no network I/O. It checks that each URL has a manifest
/// entry, that the entry maps to the URL-derived destination, and that the
/// local file still matches the recorded size and BLAKE3 digest.
pub fn audit_local_manifest(
&self,
plan: &OfficialResourcePullPlan,
) -> Result<OfficialLocalManifestAuditReport, String> {
let manifest = self.read_download_manifest()?;
let mut items = Vec::new();
for url in plan.all_urls()? {
let destination = self.destination_for_url(&url)?;
items.push(self.audit_one(&url, destination, &manifest)?);
}
Ok(OfficialLocalManifestAuditReport { items })
}
/// Verifies every entry currently present in the local download manifest.
///
/// This is intentionally network-free. It validates each recorded file
/// using the local manifest's size and BLAKE3 digest, then verifies every
/// local seed `.bytes`/`.hash` pair that is present using the official
/// decimal xxHash32 rule.
pub fn verify_local_download_manifest(
&self,
) -> Result<OfficialLocalVerificationReport, String> {
let manifest = self.read_download_manifest()?;
let urls = manifest.entries.keys().cloned().collect::<Vec<_>>();
let mut items = Vec::with_capacity(urls.len());
for url in urls {
let destination = self.destination_for_url(&url)?;
items.push(self.audit_one(&url, destination, &manifest)?);
}
let pairs = local_manifest_seed_hash_pairs(&manifest);
let mut official_hash_verified_count = 0;
let mut official_hash_errors = Vec::new();
for pair in &pairs {
match self.verify_official_seed_hash_pair_from_disk(pair) {
Ok(_) => official_hash_verified_count += 1,
Err(error) => official_hash_errors.push(error),
}
}
Ok(OfficialLocalVerificationReport {
items,
official_hash_pair_count: pairs.len(),
official_hash_verified_count,
official_hash_errors,
})
}
/// Returns whether the current output root already contains local state for
/// this plan. This is intentionally cheaper than a full audit and is used
/// to choose between audit-then-repair and first-time full pull flows.
pub fn local_resource_state(
&self,
plan: &OfficialResourcePullPlan,
) -> Result<OfficialLocalResourceState, String> {
let manifest = self.read_download_manifest()?;
let mut manifest_entry_count = 0;
let mut existing_file_count = 0;
for url in plan.all_urls()? {
if manifest.entries.contains_key(&url) {
manifest_entry_count += 1;
}
let destination = self.destination_for_url(&url)?;
if file_len_if_exists(&destination)?.is_some() {
existing_file_count += 1;
}
}
Ok(OfficialLocalResourceState {
manifest_entry_count,
existing_file_count,
})
}
fn build_release_reuse_index(&self) -> Result<ReleaseReuseIndex, String> {
let mut index = ReleaseReuseIndex::default();
let mut roots = Vec::new();
for configured_root in &self.release_reuse_roots {
let metadata = match fs::symlink_metadata(configured_root) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => {
index.warnings.push(format!(
"读取历史 release 根失败 {}{error}",
configured_root.display()
));
continue;
}
};
if metadata.file_type().is_symlink() {
index.warnings.push(format!(
"历史 release 根不能是 symlink{}",
configured_root.display()
));
continue;
}
if !metadata.is_dir() {
index.warnings.push(format!(
"历史 release 根不是目录:{}",
configured_root.display()
));
continue;
}
if let Err(error) = ensure_safe_directory_path(configured_root, "历史 release 根目录")
{
index.warnings.push(error);
continue;
}
let manifest_path = configured_root.join(DOWNLOAD_MANIFEST_FILE);
if fs::symlink_metadata(&manifest_path)
.map(|metadata| !metadata.file_type().is_symlink() && metadata.is_file())
.unwrap_or(false)
{
roots.push(configured_root.clone());
continue;
}
let entries = match fs::read_dir(configured_root) {
Ok(entries) => entries,
Err(error) => {
index.warnings.push(format!(
"读取历史 release 目录失败 {}{error}",
configured_root.display()
));
continue;
}
};
let mut child_roots = entries
.filter_map(|entry| entry.ok().map(|entry| entry.path()))
.filter(|path| {
fs::symlink_metadata(path)
.map(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink())
.unwrap_or(false)
})
.collect::<Vec<_>>();
child_roots.sort();
child_roots.reverse();
roots.extend(child_roots);
}
for root in roots {
let manifest = match read_download_manifest_at(&root) {
Ok(Some(manifest)) => manifest,
Ok(None) => continue,
Err(error) => {
index.warnings.push(format!(
"忽略无效历史 release manifest {}{error}",
root.display()
));
continue;
}
};
for entry in manifest.entries.values() {
if !is_valid_blake3_hex(&entry.blake3) {
index.warnings.push(format!(
"忽略没有可靠 BLAKE3 的历史条目:release={} destination={}",
root.display(),
entry.destination
));
continue;
}
let source = root.join(Path::new(&entry.destination));
if ensure_path_within_root(&root, &source).is_err() {
index.warnings.push(format!(
"忽略逃逸历史条目:release={} destination={}",
root.display(),
entry.destination
));
continue;
}
let candidate = HistoricalReuseCandidate {
root: root.clone(),
source,
destination: entry.destination.clone(),
bytes: entry.bytes,
blake3: entry.blake3.clone(),
};
index
.candidates
.entry(reuse_key(&entry.destination))
.or_default()
.push(candidate);
}
}
Ok(index)
}
fn reuse_candidate_for_url(
&self,
_url: &str,
destination: &Path,
index: &ReleaseReuseIndex,
) -> Result<Option<ReuseCandidate>, String> {
let relative_destination = self.relative_destination(destination)?;
Ok(index
.candidates
.get(&reuse_key(&relative_destination))
.filter(|candidates| !candidates.is_empty())
.cloned()
.map(|candidates| ReuseCandidate { candidates }))
}
fn pull_task(&self, task: PlannedDownload) -> Result<PullOneResult, PullOneError> {
if let Some(existing) = task.existing {
return Ok(existing);
}
let mut reuse_warnings = Vec::new();
if let Some(candidate) = task.reuse_candidate.as_ref() {
let (reused, warnings) =
self.try_reuse_candidate(candidate, &task.url, &task.destination);
reuse_warnings.extend(warnings);
if let Some(mut result) = reused {
result.reuse_warnings = reuse_warnings;
return Ok(result);
}
}
let mut result = self.pull_one(&task.url, &task.destination)?;
result.reuse_warnings = reuse_warnings;
Ok(result)
}
fn try_reuse_candidate(
&self,
candidate: &ReuseCandidate,
url: &str,
destination: &Path,
) -> (Option<PullOneResult>, Vec<OfficialResourceReuseWarning>) {
let mut warnings = Vec::new();
for historical in &candidate.candidates {
match self.materialize_historical_candidate(historical, url, destination) {
Ok(result) => return (Some(result), warnings),
Err(error) => warnings.push(OfficialResourceReuseWarning {
url: url.to_string(),
source: "historical_release".to_string(),
message: format!(
"release={} destination={}{error}",
historical.root.display(),
historical.destination
),
}),
}
}
if self.cas_reuse_root.is_some() {
for historical in &candidate.candidates {
match self.materialize_cas_candidate(historical, url, destination) {
Ok(result) => return (Some(result), warnings),
Err(error) => warnings.push(OfficialResourceReuseWarning {
url: url.to_string(),
source: "cas".to_string(),
message: format!(
"object={} destination={}{error}",
historical.blake3, historical.destination
),
}),
}
}
}
(None, warnings)
}
fn materialize_historical_candidate(
&self,
candidate: &HistoricalReuseCandidate,
url: &str,
destination: &Path,
) -> Result<PullOneResult, String> {
ensure_safe_directory_path(&candidate.root, "历史 release 根目录")?;
ensure_safe_file_target(&candidate.root, &candidate.source, "历史 release 复用源")?;
verify_file_attributes(&candidate.source, candidate.bytes, &candidate.blake3, url)?;
self.materialize_verified_path(
&candidate.source,
url,
destination,
ReuseMaterialization {
expected_bytes: candidate.bytes,
expected_blake3: candidate.blake3.clone(),
status: OfficialResourcePullStatus::ReleaseReused,
cas_object_id: None,
},
)
}
fn materialize_cas_candidate(
&self,
candidate: &HistoricalReuseCandidate,
url: &str,
destination: &Path,
) -> Result<PullOneResult, String> {
let cas_root = self
.cas_reuse_root
.as_ref()
.ok_or_else(|| "未配置 CAS 根目录".to_string())?;
let objects_root = cas_root.join("objects");
let metadata_path = cas_root.join("metadata.sqlite");
require_existing_directory(cas_root, "CAS 根目录")?;
require_existing_directory(&objects_root, "CAS 对象目录")?;
require_existing_file(cas_root, &metadata_path, "CAS 元数据库")?;
let object_path = cas_object_path(cas_root, &candidate.blake3)?;
ensure_safe_file_target(&objects_root, &object_path, "CAS 对象")?;
let object_id = candidate.blake3.clone();
let cas_root_for_runtime = cas_root.clone();
let object_id_for_runtime = object_id.clone();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| format!("创建 CAS 校验 runtime 失败:{error}"))?;
runtime
.block_on(async move {
let cas = crate::FileSystemCasRepository::new(cas_root_for_runtime);
cas.get(&object_id_for_runtime)
.await
.map_err(|error| format!("CAS 对象完整性校验失败:{error}"))?;
cas.get_reference_count(&object_id_for_runtime)
.await
.map_err(|error| format!("CAS 对象元数据不一致:{error}"))?;
cas.add_reference(&object_id_for_runtime)
.await
.map_err(|error| format!("增加 CAS release 引用失败:{error}"))
})
.map_err(|error| error.to_string())?;
self.track_cas_reference(&object_id);
match self.materialize_verified_path(
&object_path,
url,
destination,
ReuseMaterialization {
expected_bytes: candidate.bytes,
expected_blake3: candidate.blake3.clone(),
status: OfficialResourcePullStatus::CasReused,
cas_object_id: Some(object_id.clone()),
},
) {
Ok(result) => Ok(result),
Err(error) => {
let cas_root_for_runtime = cas_root.clone();
let object_id_for_runtime = object_id.clone();
let rollback = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|runtime_error| {
format!("创建 CAS 回滚 runtime 失败:{runtime_error}")
})?
.block_on(async move {
let cas = crate::FileSystemCasRepository::new(cas_root_for_runtime);
cas.remove_reference(&object_id_for_runtime)
.await
.map_err(|rollback_error| rollback_error.to_string())
});
if let Err(rollback_error) = rollback {
return Err(format!("{error};回滚 CAS 引用失败:{rollback_error}"));
}
self.untrack_cas_reference(&object_id);
Err(error)
}
}
}
fn materialize_verified_path(
&self,
source: &Path,
url: &str,
destination: &Path,
materialization: ReuseMaterialization,
) -> Result<PullOneResult, String> {
ensure_safe_file_target(&self.output_root, destination, "复用目标文件")?;
if source == destination {
let verification = self.local_verification(
url,
destination,
Some(materialization.expected_bytes),
Some(materialization.expected_blake3),
)?;
return Ok(PullOneResult {
bytes: verification.actual_bytes,
transferred_bytes: 0,
status: materialization.status,
verification,
cas_object_id: materialization.cas_object_id,
reuse_warnings: Vec::new(),
});
}
let temporary = reuse_temporary_path(destination);
ensure_safe_file_target(&self.output_root, &temporary, "复用临时文件")?;
let result = (|| {
if let Err(hard_link_error) = fs::hard_link(source, &temporary) {
fs::copy(source, &temporary).map_err(|copy_error| {
format!("硬链接复用失败:{hard_link_error};跨文件系统复制也失败:{copy_error}")
})?;
}
let verification = self.local_verification(
url,
&temporary,
Some(materialization.expected_bytes),
Some(materialization.expected_blake3),
)?;
ensure_safe_file_target(&self.output_root, destination, "复用目标文件")?;
fs::rename(&temporary, destination).map_err(|error| {
format!(
"原子移动复用文件失败 {} -> {}{error}",
temporary.display(),
destination.display()
)
})?;
Ok(PullOneResult {
bytes: verification.actual_bytes,
transferred_bytes: 0,
status: materialization.status,
verification,
cas_object_id: materialization.cas_object_id,
reuse_warnings: Vec::new(),
})
})();
if result.is_err() {
let _ = fs::remove_file(&temporary);
}
result
}
/// Fetches an official URL into memory.
///
/// This is intended for small discovery inputs such as `server-info`,
/// `TableCatalog.bytes`, `BundlePackingInfo.bytes`, and
/// `MediaCatalog.bytes`.
pub fn fetch_bytes(&self, url: &str) -> Result<Vec<u8>, DownloadError> {
if !self.backend.is_official_url(url) {
return Err(DownloadError::new(
bat_core::ErrorCode::NON_OFFICIAL_URL,
format!("拒绝拉取非官方 URL{url}"),
));
}
let output = run_curl_with_retry_with_proxy(
&self.curl_command,
url,
None,
self.retry_attempts,
&self.curl_proxy,
|| {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--url")
.arg(url);
command
},
)
.map_err(|error| {
// 保留 curl 失败的准确网络域码;进程类失败归 INTERNAL。
let code = error
.final_error_code()
.unwrap_or(bat_core::ErrorCode::INTERNAL);
DownloadError::new(code, format!("curl 拉取失败 {url}{error}"))
})?;
Ok(output.stdout)
}
fn pull_one(&self, url: &str, destination: &Path) -> Result<PullOneResult, PullOneError> {
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
let partial = partial_path_for(destination);
ensure_safe_file_target(&self.output_root, &partial, "临时下载文件")
.map_err(PullOneError::plain)?;
let partial_len_before = file_len_if_exists(&partial)
.map_err(PullOneError::plain)?
.unwrap_or_default();
let resumed = partial_len_before > 0;
let mut status = if resumed {
OfficialResourcePullStatus::Resumed
} else {
OfficialResourcePullStatus::Downloaded
};
if resumed {
if let Err(error) = self.download_one(url, &partial, true) {
let _ = fs::remove_file(&partial);
self.download_one(url, &partial, false)
.map_err(|retry_error| {
PullOneError::from_retry(
format!("续传失败 {url}{error};清理后重新下载也失败:{retry_error}"),
retry_error,
)
})?;
status = OfficialResourcePullStatus::Downloaded;
} else if let Err(error) = self.validate_zip_if_needed(url, &partial) {
let _ = fs::remove_file(&partial);
self.download_one(url, &partial, false)
.map_err(|retry_error| {
PullOneError::from_retry(
format!("续传后的 ZIP 结构校验失败 {url}{error};清理后重新下载也失败:{retry_error}"),
retry_error,
)
})?;
status = OfficialResourcePullStatus::Downloaded;
}
} else {
self.download_one(url, &partial, false).map_err(|error| {
PullOneError::from_retry(format!("下载失败 {url}{error}"), error)
})?;
}
if let Err(error) = self.validate_zip_if_needed(url, &partial) {
let _ = fs::remove_file(&partial);
return Err(PullOneError::plain(error));
}
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
if file_len_if_exists(destination)
.map_err(PullOneError::plain)?
.is_some()
{
fs::remove_file(destination)
.map_err(|error| format!("替换已有目标文件失败 {}{error}", destination.display()))
.map_err(PullOneError::plain)?;
}
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
fs::rename(&partial, destination)
.map_err(|error| {
format!(
"移动临时下载文件失败 {} -> {}{error}",
partial.display(),
destination.display()
)
})
.map_err(PullOneError::plain)?;
let bytes = file_len_if_exists(destination)
.map_err(PullOneError::plain)?
.ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))
.map_err(PullOneError::plain)?;
let verification = self
.local_verification(url, destination, None, None)
.map_err(PullOneError::plain)?;
Ok(PullOneResult {
bytes,
transferred_bytes: if status == OfficialResourcePullStatus::Resumed {
bytes.saturating_sub(partial_len_before)
} else {
bytes
},
status,
verification,
cas_object_id: None,
reuse_warnings: Vec::new(),
})
}
fn download_one(
&self,
url: &str,
destination: &Path,
resume: bool,
) -> Result<(), CurlRetryError> {
run_curl_with_retry_with_proxy(
&self.curl_command,
url,
Some(destination),
self.retry_attempts,
&self.curl_proxy,
|| {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--output")
.arg(destination);
if resume {
command.arg("--continue-at").arg("-");
}
command.arg("--url").arg(url);
command
},
)
.map(|_| ())
}
fn verify_ready_official_hashes(
&self,
pairs: &[OfficialSeedHashPair],
processed_urls: &HashSet<String>,
verified_hash_urls: &mut HashSet<String>,
verified_hashes: &mut Vec<OfficialResourceHashVerification>,
manifest: &mut OfficialDownloadManifest,
) -> Result<Vec<OfficialResourceHashVerification>, String> {
let mut newly_verified = Vec::new();
for pair in pairs {
if verified_hash_urls.contains(&pair.hash_url) {
continue;
}
if !processed_urls.contains(&pair.data_url) || !processed_urls.contains(&pair.hash_url)
{
continue;
}
if !self.hash_pair_files_exist(pair)? {
continue;
}
let verification = match self.verify_official_seed_hash_pair_from_disk(pair) {
Ok(verification) => verification,
Err(error) => {
if let Err(cleanup_error) =
self.invalidate_download_manifest_hash_pair(manifest, pair)
{
return Err(format!(
"{error};同时清理官方 hash 对应的本地 manifest 条目失败:{cleanup_error}"
));
}
return Err(error);
}
};
newly_verified.push(verification.clone());
verified_hashes.push(verification);
verified_hash_urls.insert(pair.hash_url.clone());
}
Ok(newly_verified)
}
fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result<bool, String> {
let data_path = self.destination_for_url(&pair.data_url)?;
let hash_path = self.destination_for_url(&pair.hash_url)?;
Ok(file_len_if_exists(&data_path)?.is_some() && file_len_if_exists(&hash_path)?.is_some())
}
fn verify_official_seed_hash_pair_from_disk(
&self,
pair: &OfficialSeedHashPair,
) -> Result<OfficialResourceHashVerification, String> {
let data_path = self.destination_for_url(&pair.data_url)?;
let hash_path = self.destination_for_url(&pair.hash_url)?;
let data = fs::read(&data_path).map_err(|error| {
format!(
"读取官方 hash 对应数据文件失败 {}{error}",
data_path.display()
)
})?;
let hash = fs::read(&hash_path).map_err(|error| {
format!(
"读取官方 hash sidecar 失败 {}{error}",
hash_path.display()
)
})?;
verify_official_seed_catalog_hash(&pair.data_url, &data, &pair.hash_url, &hash)
}
fn invalidate_download_manifest_hash_pair(
&self,
manifest: &mut OfficialDownloadManifest,
pair: &OfficialSeedHashPair,
) -> Result<(), String> {
manifest.entries.remove(&pair.data_url);
manifest.entries.remove(&pair.hash_url);
self.write_download_manifest(manifest)
}
fn verify_all_official_hashes_are_complete(
&self,
pairs: &[OfficialSeedHashPair],
verified_hash_urls: &HashSet<String>,
) -> Result<(), String> {
for pair in pairs {
if !verified_hash_urls.contains(&pair.hash_url) {
return Err(format!(
"官方 hash 对未完成校验:数据={} hash={}",
pair.data_url, pair.hash_url
));
}
}
Ok(())
}
fn read_download_manifest(&self) -> Result<OfficialDownloadManifest, String> {
self.ensure_output_root_safe()?;
let path = self.download_manifest_path();
ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?;
let Some(bytes) = read_file_no_symlink(&path, "下载 manifest")? else {
return Ok(OfficialDownloadManifest::default());
};
let manifest: OfficialDownloadManifest = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析下载 manifest 失败 {}{error}", path.display()))?;
if manifest.version != DOWNLOAD_MANIFEST_VERSION {
return Err(format!(
"不支持的下载 manifest 版本 {},文件 {}",
manifest.version,
path.display()
));
}
Ok(manifest)
}
fn remove_managed_destination(&self, path: &Path, url: &str) -> Result<(), String> {
ensure_path_within_root(&self.output_root, path)?;
ensure_safe_file_target(&self.output_root, path, "旧官方资源清理目标")?;
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(format!(
"旧官方资源清理目标不能是 symlink:url={url} path={}",
path.display()
)),
Ok(metadata) if metadata.is_file() => {
fs::remove_file(path).map_err(|error| {
format!(
"清理旧官方资源失败:url={url} path={} error={error}",
path.display()
)
})?;
remove_empty_parent_directories(&self.output_root, path.parent())?;
Ok(())
}
Ok(metadata) if metadata.is_dir() => Err(format!(
"旧官方资源清理目标不能是目录:url={url} path={}",
path.display()
)),
Ok(_) => Err(format!(
"旧官方资源清理目标不是普通文件:url={url} path={}",
path.display()
)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(format!(
"检查旧官方资源清理目标失败:url={url} path={} error={error}",
path.display()
)),
}
}
fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> {
self.ensure_output_root_ready()?;
let path = self.download_manifest_path();
if let Some(parent) = path.parent() {
ensure_safe_directory_path(parent, "下载 manifest 目录")?;
fs::create_dir_all(parent).map_err(|error| {
format!("创建下载 manifest 目录失败 {}{error}", parent.display())
})?;
ensure_safe_directory_path(parent, "下载 manifest 目录")?;
}
ensure_safe_file_target(&self.output_root, &path, "下载 manifest")?;
let bytes = serde_json::to_vec_pretty(manifest)
.map_err(|error| format!("序列化下载 manifest 失败 {}{error}", path.display()))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 manifest")?;
Ok(())
}
fn read_download_quarantine(&self) -> Result<OfficialDownloadQuarantineManifest, String> {
self.ensure_output_root_safe()?;
let path = self.download_quarantine_path();
ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?;
let Some(bytes) = read_file_no_symlink(&path, "下载 quarantine")? else {
return Ok(OfficialDownloadQuarantineManifest::default());
};
let manifest: OfficialDownloadQuarantineManifest = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析下载 quarantine 失败 {}{error}", path.display()))?;
if manifest.version != DOWNLOAD_QUARANTINE_VERSION {
return Err(format!(
"不支持的下载 quarantine 版本 {},文件 {}",
manifest.version,
path.display()
));
}
Ok(manifest)
}
fn write_download_quarantine(
&self,
manifest: &OfficialDownloadQuarantineManifest,
) -> Result<(), String> {
self.ensure_output_root_ready()?;
let path = self.download_quarantine_path();
if let Some(parent) = path.parent() {
ensure_safe_directory_path(parent, "下载 quarantine 目录")?;
fs::create_dir_all(parent).map_err(|error| {
format!("创建下载 quarantine 目录失败 {}{error}", parent.display())
})?;
ensure_safe_directory_path(parent, "下载 quarantine 目录")?;
}
ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?;
let bytes = serde_json::to_vec_pretty(manifest)
.map_err(|error| format!("序列化下载 quarantine 失败 {}{error}", path.display()))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 quarantine")?;
Ok(())
}
fn record_quarantine_entry(
&self,
url: &str,
destination: &Path,
error: &PullOneError,
) -> Result<(), String> {
let mut manifest = self.read_download_quarantine()?;
manifest.entries.insert(
url.to_string(),
OfficialDownloadQuarantineEntry {
url: url.to_string(),
destination: self.relative_destination(destination)?,
failed_at_unix_seconds: unix_seconds_now(),
attempts: error.attempts().unwrap_or(0),
failure_kind: error.failure_kind().map(ToOwned::to_owned),
http_status: error.http_status(),
retryable: error.retryable(),
last_error: error.message.clone(),
},
);
self.write_download_quarantine(&manifest)
}
fn clear_quarantine_entry(&self, url: &str) -> Result<(), String> {
let mut manifest = self.read_download_quarantine()?;
if manifest.entries.remove(url).is_none() {
return Ok(());
}
self.write_download_quarantine(&manifest)
}
fn validated_existing_file(
&self,
url: &str,
destination: &Path,
manifest: &OfficialDownloadManifest,
) -> Result<Option<PullOneResult>, String> {
let Some(entry) = manifest.entries.get(url) else {
return Ok(None);
};
if entry.url != url {
return Ok(None);
}
if entry.destination != self.relative_destination(destination)? {
return Ok(None);
}
let Some(bytes) = file_len_if_exists(destination)? else {
return Ok(None);
};
if bytes != entry.bytes {
return Ok(None);
}
let digest = blake3_file_hex(destination)?;
if digest != entry.blake3 {
return Ok(None);
}
if let Err(_error) = self.validate_zip_if_needed(url, destination) {
return Ok(None);
}
let verification = self.local_verification(
url,
destination,
Some(entry.bytes),
Some(entry.blake3.clone()),
)?;
Ok(Some(PullOneResult {
bytes,
transferred_bytes: 0,
status: OfficialResourcePullStatus::SkippedExisting,
verification,
cas_object_id: None,
reuse_warnings: Vec::new(),
}))
}
fn local_verification(
&self,
url: &str,
destination: &Path,
expected_bytes: Option<u64>,
expected_blake3: Option<String>,
) -> Result<OfficialResourceVerification, String> {
let actual_bytes = file_len_if_exists(destination)?
.ok_or_else(|| format!("读取资源文件信息失败 {}", destination.display()))?;
let actual_blake3 = blake3_file_hex(destination)?;
let zip_checked = url_or_path_has_zip_extension(url) || path_has_zip_extension(destination);
if zip_checked {
self.validate_zip_if_needed(url, destination)?;
}
Ok(OfficialResourceVerification {
expected_bytes,
actual_bytes,
expected_blake3,
actual_blake3,
zip_checked,
zip_structure_verified: zip_checked,
})
}
fn record_download_manifest_entry(
&self,
manifest: &mut OfficialDownloadManifest,
url: &str,
destination: &Path,
) -> Result<OfficialResourceVerification, String> {
let mut verification = self.local_verification(url, destination, None, None)?;
let relative_destination = self.relative_destination(destination)?;
manifest.entries.insert(
url.to_string(),
OfficialDownloadManifestEntry {
url: url.to_string(),
destination: relative_destination,
bytes: verification.actual_bytes,
blake3: verification.actual_blake3.clone(),
},
);
verification.expected_bytes = Some(verification.actual_bytes);
verification.expected_blake3 = Some(verification.actual_blake3.clone());
Ok(verification)
}
fn validate_zip_if_needed(&self, url: &str, destination: &Path) -> Result<(), String> {
ensure_safe_file_target(&self.output_root, destination, "ZIP 校验目标")?;
if !url_or_path_has_zip_extension(url) && !path_has_zip_extension(destination) {
return Ok(());
}
validate_zip_structure(destination).map(|_report| ())
}
fn relative_destination(&self, destination: &Path) -> Result<String, String> {
let relative = destination
.strip_prefix(&self.output_root)
.map_err(|error| {
format!(
"目标路径 {} 不在输出目录 {} 下:{error}",
destination.display(),
self.output_root.display()
)
})?;
Ok(relative.to_string_lossy().replace('\\', "/"))
}
fn audit_one(
&self,
url: &str,
destination: PathBuf,
manifest: &OfficialDownloadManifest,
) -> Result<OfficialLocalManifestAuditItem, String> {
let Some(entry) = manifest.entries.get(url) else {
let actual_bytes = file_len_if_exists(&destination)?;
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::MissingManifestEntry,
manifest_destination: None,
expected_bytes: None,
actual_bytes,
expected_blake3: None,
actual_blake3: None,
zip_error: None,
zip_structure_verified: false,
});
};
let manifest_destination = Some(entry.destination.clone());
let expected_bytes = Some(entry.bytes);
let expected_blake3 = Some(entry.blake3.clone());
if entry.url != url {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::UrlMismatch,
manifest_destination,
expected_bytes,
actual_bytes: None,
expected_blake3,
actual_blake3: None,
zip_error: None,
zip_structure_verified: false,
});
}
let expected_destination = self.relative_destination(&destination)?;
if entry.destination != expected_destination {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::DestinationMismatch,
manifest_destination,
expected_bytes,
actual_bytes: None,
expected_blake3,
actual_blake3: None,
zip_error: None,
zip_structure_verified: false,
});
}
let Some(actual_bytes) = file_len_if_exists(&destination)? else {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::MissingFile,
manifest_destination,
expected_bytes,
actual_bytes: None,
expected_blake3,
actual_blake3: None,
zip_error: None,
zip_structure_verified: false,
});
};
if actual_bytes != entry.bytes {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::SizeMismatch,
manifest_destination,
expected_bytes,
actual_bytes: Some(actual_bytes),
expected_blake3,
actual_blake3: None,
zip_error: None,
zip_structure_verified: false,
});
}
let actual_blake3 = blake3_file_hex(&destination)?;
if actual_blake3 != entry.blake3 {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::Blake3Mismatch,
manifest_destination,
expected_bytes,
actual_bytes: Some(actual_bytes),
expected_blake3,
actual_blake3: Some(actual_blake3),
zip_error: None,
zip_structure_verified: false,
});
}
let zip_structure_required =
url_or_path_has_zip_extension(url) || path_has_zip_extension(&destination);
if let Err(error) = self.validate_zip_if_needed(url, &destination) {
return Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::ZipStructureInvalid,
manifest_destination,
expected_bytes,
actual_bytes: Some(actual_bytes),
expected_blake3,
actual_blake3: Some(actual_blake3),
zip_error: Some(error),
zip_structure_verified: false,
});
}
Ok(OfficialLocalManifestAuditItem {
url: url.to_string(),
destination,
status: OfficialLocalManifestAuditStatus::Verified,
manifest_destination,
expected_bytes,
actual_bytes: Some(actual_bytes),
expected_blake3,
actual_blake3: Some(actual_blake3),
zip_error: None,
zip_structure_verified: zip_structure_required,
})
}
fn destination_for_url(&self, url: &str) -> Result<PathBuf, String> {
self.ensure_output_root_safe()?;
if !self.backend.is_official_url(url) {
return Err(format!("URL 不是官方 JP host{url}"));
}
let relative_destination = self.backend.relative_destination(url)?;
let destination = destination_under_root(&self.output_root, &relative_destination)?;
ensure_path_within_root(&self.output_root, &destination)?;
Ok(destination)
}
fn ensure_output_root_safe(&self) -> Result<(), String> {
validate_output_root(&self.output_root)
}
fn ensure_output_root_ready(&self) -> Result<(), String> {
self.ensure_output_root_safe()?;
ensure_safe_directory_path(&self.output_root, "资源输出目录")?;
fs::create_dir_all(&self.output_root).map_err(|error| {
format!(
"创建资源输出目录失败 {}{error}",
self.output_root.display()
)
})?;
ensure_safe_directory_path(&self.output_root, "资源输出目录")
}
}
/// Verifies an official seed catalog `.bytes` payload against its official
/// `.hash` sidecar.
///
/// The currently verified seed catalogs are `TableCatalog.bytes`,
/// `BundlePackingInfo.bytes`, and `MediaCatalog.bytes`; their `.hash` files are
/// decimal text `xxHash32` values with seed `0`.
pub fn verify_official_seed_catalog_hash(
data_url: &str,
data: &[u8],
hash_url: &str,
hash_bytes: &[u8],
) -> Result<OfficialResourceHashVerification, String> {
let strategy = XxHash32DecimalSeedZero;
let verification = strategy
.verify(data, hash_bytes)
.map_err(|error| format!("{error}{hash_url}"))?;
if verification.expected != verification.actual {
return Err(format!(
"官方 hash 校验失败 {data_url}:期望 {}(来自 {hash_url}),实际 {}",
verification.expected, verification.actual
));
}
Ok(OfficialResourceHashVerification {
data_url: data_url.to_string(),
hash_url: hash_url.to_string(),
algorithm: OfficialResourceHashAlgorithm::XxHash32Decimal,
expected: verification.expected,
actual: verification.actual,
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct OfficialSeedHashPair {
data_url: String,
hash_url: String,
}
fn official_seed_hash_pairs(plan: &OfficialResourcePullPlan) -> Vec<OfficialSeedHashPair> {
plan.discovery
.endpoints
.iter()
.filter_map(|endpoint| {
let hash_kind = strong_seed_hash_kind(endpoint.kind)?;
let hash_endpoint = plan.discovery.endpoints.iter().find(|candidate| {
candidate.kind == hash_kind && candidate.platform == endpoint.platform
})?;
Some(OfficialSeedHashPair {
data_url: endpoint.url.clone(),
hash_url: hash_endpoint.url.clone(),
})
})
.collect()
}
fn local_manifest_seed_hash_pairs(
manifest: &OfficialDownloadManifest,
) -> Vec<OfficialSeedHashPair> {
manifest
.entries
.keys()
.filter_map(|data_url| {
let hash_url = data_url
.strip_suffix(".bytes")
.map(|prefix| format!("{prefix}.hash"))?;
if !manifest.entries.contains_key(&hash_url) {
return None;
}
Some(OfficialSeedHashPair {
data_url: data_url.clone(),
hash_url,
})
})
.collect()
}
fn official_hash_refresh_urls(pairs: &[OfficialSeedHashPair]) -> HashSet<String> {
let mut urls = HashSet::new();
for pair in pairs {
urls.insert(pair.data_url.clone());
urls.insert(pair.hash_url.clone());
}
urls
}
fn strong_seed_hash_kind(
kind: YostarJpResourceEndpointKind,
) -> Option<YostarJpResourceEndpointKind> {
match kind {
YostarJpResourceEndpointKind::TableCatalog => {
Some(YostarJpResourceEndpointKind::TableCatalogHash)
}
YostarJpResourceEndpointKind::BundlePackingInfo => {
Some(YostarJpResourceEndpointKind::BundlePackingInfoHash)
}
YostarJpResourceEndpointKind::MediaCatalog => {
Some(YostarJpResourceEndpointKind::MediaCatalogHash)
}
YostarJpResourceEndpointKind::AddressablesCatalog
| YostarJpResourceEndpointKind::TableCatalogHash
| YostarJpResourceEndpointKind::AddressablesCatalogHash
| YostarJpResourceEndpointKind::BundlePackingInfoHash
| YostarJpResourceEndpointKind::MediaCatalogHash => None,
}
}
/// 官方下载 manifest:按完整 URL 为键记录每个已下载资源的本地校验信息。
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialDownloadManifest {
/// Manifest 结构版本。
#[serde(default = "default_download_manifest_version")]
pub version: u32,
/// 按 URL 为键的资源条目。
#[serde(default)]
pub entries: BTreeMap<String, OfficialDownloadManifestEntry>,
}
impl Default for OfficialDownloadManifest {
fn default() -> Self {
Self {
version: DOWNLOAD_MANIFEST_VERSION,
entries: BTreeMap::new(),
}
}
}
/// 官方下载 manifest 中的单个资源条目。
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialDownloadManifestEntry {
/// 官方资源 URL。
pub url: String,
/// 相对资源根目录的落盘路径。
pub destination: String,
/// 文件字节数。
pub bytes: u64,
/// 文件 BLAKE3 摘要(hex)。
pub blake3: String,
}
/// 读取指定资源根目录下的官方下载 manifest(daemon RPC 只读查询用)。
///
/// 文件缺失返回 `Ok(None)`;符号链接、解析失败或版本不支持返回 `Err`。
pub fn read_download_manifest_at(
resource_root: &Path,
) -> Result<Option<OfficialDownloadManifest>, String> {
let path = resource_root.join(DOWNLOAD_MANIFEST_FILE);
let Some(bytes) = read_file_no_symlink(&path, "下载 manifest")? else {
return Ok(None);
};
let manifest: OfficialDownloadManifest = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析下载 manifest 失败 {}{error}", path.display()))?;
if manifest.version != DOWNLOAD_MANIFEST_VERSION {
return Err(format!(
"不支持的下载 manifest 版本 {},文件 {}",
manifest.version,
path.display()
));
}
Ok(Some(manifest))
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct OfficialDownloadQuarantineManifest {
#[serde(default = "default_download_quarantine_version")]
version: u32,
#[serde(default)]
entries: BTreeMap<String, OfficialDownloadQuarantineEntry>,
}
impl Default for OfficialDownloadQuarantineManifest {
fn default() -> Self {
Self {
version: DOWNLOAD_QUARANTINE_VERSION,
entries: BTreeMap::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct OfficialDownloadQuarantineEntry {
url: String,
destination: String,
failed_at_unix_seconds: u64,
attempts: usize,
failure_kind: Option<String>,
http_status: Option<u16>,
retryable: Option<bool>,
last_error: String,
}
fn default_download_manifest_version() -> u32 {
DOWNLOAD_MANIFEST_VERSION
}
fn default_download_quarantine_version() -> u32 {
DOWNLOAD_QUARANTINE_VERSION
}
#[derive(Debug, Clone, Default)]
struct ReleaseReuseIndex {
candidates: HashMap<String, Vec<HistoricalReuseCandidate>>,
warnings: Vec<String>,
}
#[derive(Debug, Clone)]
struct HistoricalReuseCandidate {
root: PathBuf,
source: PathBuf,
destination: String,
bytes: u64,
blake3: String,
}
#[derive(Debug, Clone)]
struct ReuseCandidate {
candidates: Vec<HistoricalReuseCandidate>,
}
struct CasReferenceRollbackGuard {
cas_root: Option<PathBuf>,
tracker: Arc<Mutex<Vec<String>>>,
committed: bool,
}
impl CasReferenceRollbackGuard {
fn new(cas_root: Option<PathBuf>, tracker: Arc<Mutex<Vec<String>>>) -> Self {
Self {
cas_root,
tracker,
committed: false,
}
}
fn commit(mut self) {
self.committed = true;
if let Ok(mut references) = self.tracker.lock() {
references.clear();
}
}
}
impl Drop for CasReferenceRollbackGuard {
fn drop(&mut self) {
if self.committed {
return;
}
let Some(cas_root) = self.cas_root.clone() else {
return;
};
let object_ids = self
.tracker
.lock()
.map(|mut references| std::mem::take(&mut *references))
.unwrap_or_default();
if object_ids.is_empty() {
return;
}
let Ok(runtime) = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
else {
return;
};
runtime.block_on(async move {
let cas = crate::FileSystemCasRepository::new(cas_root);
for object_id in object_ids {
let _ = cas.remove_reference(&object_id).await;
}
});
}
}
/// Phase A 产出的单个下载计划项:URL、目标路径,以及若命中本地 manifest
/// 校验则带上「已验证可跳过」的结果(`existing`)。
#[derive(Debug, Clone)]
struct PlannedDownload {
url: String,
destination: PathBuf,
existing: Option<PullOneResult>,
reuse_candidate: Option<ReuseCandidate>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct PullOneResult {
bytes: u64,
transferred_bytes: u64,
status: OfficialResourcePullStatus,
verification: OfficialResourceVerification,
cas_object_id: Option<String>,
reuse_warnings: Vec<OfficialResourceReuseWarning>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ReuseMaterialization {
expected_bytes: u64,
expected_blake3: String,
status: OfficialResourcePullStatus,
cas_object_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct PullOneError {
message: String,
retry_error: Option<CurlRetryError>,
}
struct CurlDownloadBackend<'a> {
service: &'a OfficialResourcePullService,
}
impl DownloaderBackend<PlannedDownload> for CurlDownloadBackend<'_> {
type Output = PullOneResult;
type Error = PullOneError;
fn download(&self, task: PlannedDownload) -> Result<Self::Output, Self::Error> {
self.service.pull_task(task)
}
}
impl PullOneError {
fn plain(message: String) -> Self {
Self {
message,
retry_error: None,
}
}
fn from_retry(message: String, retry_error: CurlRetryError) -> Self {
Self {
message,
retry_error: Some(retry_error),
}
}
fn failure_kind(&self) -> Option<&'static str> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_kind_label)
}
/// 映射到统一错误码:curl 重试失败取网络域码,其余(进程/续传/替换等)归 INTERNAL。
fn error_code(&self) -> bat_core::ErrorCode {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_error_code)
.unwrap_or(bat_core::ErrorCode::INTERNAL)
}
fn http_status(&self) -> Option<u16> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_http_status)
}
fn retryable(&self) -> Option<bool> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_retryable)
}
fn attempts(&self) -> Option<usize> {
self.retry_error.as_ref().map(CurlRetryError::attempt_count)
}
}
impl std::fmt::Display for PullOneError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for PullOneError {}
fn file_len_if_exists(path: &Path) -> Result<Option<u64>, String> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => {
Err(format!("拒绝跟随 symlink 文件:{}", path.display()))
}
Ok(metadata) if metadata.is_file() => Ok(Some(metadata.len())),
Ok(_) => Err(format!("期望文件路径,但实际不是文件:{}", path.display())),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(format!("读取路径信息失败 {}{error}", path.display())),
}
}
fn require_existing_directory(path: &Path, label: &str) -> Result<(), String> {
ensure_safe_directory_path(path, label)?;
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => Ok(()),
Ok(_) => Err(format!("{label} 不是可复用的目录:{}", path.display())),
Err(error) => Err(format!(
"{label} 不存在或不可读取 {}{error}",
path.display()
)),
}
}
fn require_existing_file(root: &Path, path: &Path, label: &str) -> Result<(), String> {
ensure_safe_file_target(root, path, label)?;
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(()),
Ok(_) => Err(format!("{label} 不是可复用的普通文件:{}", path.display())),
Err(error) => Err(format!(
"{label} 不存在或不可读取 {}{error}",
path.display()
)),
}
}
fn verify_file_attributes(
path: &Path,
expected_bytes: u64,
expected_blake3: &str,
url: &str,
) -> Result<(), String> {
let actual_bytes =
file_len_if_exists(path)?.ok_or_else(|| format!("复用源文件不存在:{}", path.display()))?;
if actual_bytes != expected_bytes {
return Err(format!(
"复用源 size 校验失败 {}:期望 {},实际 {}",
path.display(),
expected_bytes,
actual_bytes
));
}
let actual_blake3 = blake3_file_hex(path)?;
if actual_blake3 != expected_blake3 {
return Err(format!(
"复用源 BLAKE3 校验失败 {}:期望 {},实际 {}",
path.display(),
expected_blake3,
actual_blake3
));
}
if url_or_path_has_zip_extension(url) || path_has_zip_extension(path) {
validate_zip_structure(path).map(|_| ())?;
}
Ok(())
}
fn is_valid_blake3_hex(value: &str) -> bool {
value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn reuse_key(destination: &str) -> String {
let normalized = destination.replace('\\', "/");
let components = normalized
.split('/')
.filter(|component| !component.is_empty())
.collect::<Vec<_>>();
if components.len() <= 2 {
return components.join("/");
}
std::iter::once(components[0])
.chain(components.into_iter().skip(2))
.collect::<Vec<_>>()
.join("/")
}
fn cas_object_path(root: &Path, object_id: &str) -> Result<PathBuf, String> {
if !is_valid_blake3_hex(object_id) {
return Err(format!("CAS 对象 ID 不是有效 BLAKE3{object_id}"));
}
Ok(root
.join("objects")
.join(&object_id[..2])
.join(&object_id[2..4])
.join(object_id))
}
fn reuse_temporary_path(destination: &Path) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or_default();
let file_name = destination
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("resource");
destination.with_file_name(format!(".{file_name}.reuse.{nonce}.tmp"))
}
fn partial_path_for(destination: &Path) -> PathBuf {
let mut partial = destination.as_os_str().to_owned();
partial.push(".part");
PathBuf::from(partial)
}
fn remove_empty_parent_directories(root: &Path, parent: Option<&Path>) -> Result<(), String> {
let mut current = parent;
while let Some(path) = current {
if path == root {
break;
}
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
current = path.parent();
continue;
}
Err(error) => {
return Err(format!(
"检查旧官方资源父目录失败 {}{error}",
path.display()
));
}
};
if metadata.file_type().is_symlink() {
return Err(format!(
"旧官方资源父目录不能是 symlink:{}",
path.display()
));
}
if !metadata.is_dir() {
break;
}
let mut entries = fs::read_dir(path)
.map_err(|error| format!("读取旧官方资源父目录失败 {}{error}", path.display()))?;
if entries.next().is_some() {
break;
}
fs::remove_dir(path)
.map_err(|error| format!("清理旧官方资源空目录失败 {}{error}", path.display()))?;
current = path.parent();
}
Ok(())
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn blake3_file_hex(path: &Path) -> Result<String, String> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => {
return Err(format!("拒绝读取 symlink 文件:{}", path.display()));
}
Ok(metadata) if metadata.is_file() => {}
Ok(_) => return Err(format!("期望文件路径,但实际不是文件:{}", path.display())),
Err(error) => return Err(format!("读取路径信息失败 {}{error}", path.display())),
}
let mut file =
File::open(path).map_err(|error| format!("打开文件失败 {}{error}", path.display()))?;
let mut hasher = blake3::Hasher::new();
let mut buffer = [0u8; 64 * 1024];
loop {
let read = file
.read(&mut buffer)
.map_err(|error| format!("读取文件失败 {}{error}", path.display()))?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(hasher.finalize().to_hex().to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::official_pull::{
build_official_pull_plan, build_official_pull_plan_for_platforms, OfficialResourcePullPlan,
};
use bat_adapters::official::inventory::{
YostarJpDownloadInventory, YostarJpPlatformDownloadInventory,
};
use bat_adapters::official::yostar_jp::{
PatchPlatform, YostarJpResourceDiscoveryPlan, YostarJpResourceEndpoint,
YostarJpResourceEndpointKind,
};
use std::fs;
use tempfile::TempDir;
fn xxhash32(bytes: &[u8]) -> u32 {
XxHash32DecimalSeedZero
.digest(bytes)
.parse()
.expect("xxHash32 decimal digest")
}
#[test]
fn fetch_bytes_maps_rejections_to_error_codes() {
// 非官方 URL:安全边界拒绝。
let service = OfficialResourcePullService::new("/tmp/unused");
let error = service
.fetch_bytes("https://evil.example/server_info.json")
.unwrap_err();
assert_eq!(error.code(), bat_core::ErrorCode::NON_OFFICIAL_URL);
// curl 404:保留网络域码。
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_shell_script(
&curl_path,
r#"#!/bin/sh
printf 'curl: (22) The requested URL returned error: 404\n' >&2
exit 22
"#,
);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let error = service
.fetch_bytes("https://prod-clientpatch.bluearchiveyostar.com/r93_token/missing.bytes")
.unwrap_err();
assert_eq!(error.code(), bat_core::ErrorCode::HTTP_NOT_FOUND);
}
#[test]
fn read_download_manifest_at_handles_missing_and_bad_version() {
let temp = TempDir::new().unwrap();
// 文件缺失:Ok(None)。
assert!(read_download_manifest_at(temp.path()).unwrap().is_none());
// 正常 manifest:读取条目。
let path = temp.path().join(DOWNLOAD_MANIFEST_FILE);
fs::write(
&path,
br#"{"version":1,"entries":{"https://a":{"url":"https://a","destination":"a","bytes":1,"blake3":"aa"}}}"#,
)
.unwrap();
let manifest = read_download_manifest_at(temp.path()).unwrap().unwrap();
assert_eq!(manifest.entries.len(), 1);
// 不支持的版本:Err。
fs::write(&path, br#"{"version":999,"entries":{}}"#).unwrap();
assert!(read_download_manifest_at(temp.path()).is_err());
}
fn discovery_plan() -> YostarJpResourceDiscoveryPlan {
YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: Some("s8tloc7lo3".to_string()),
addressables_root:
"https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(),
endpoints: vec![
YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(),
},
YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalogHash,
platform: None,
url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.hash".to_string(),
},
],
}
}
fn inventory() -> YostarJpDownloadInventory {
YostarJpDownloadInventory::from_catalog_bytes(
b"FullPatch_000.zip",
b"ExcelDB.db ExcelDB.db",
b"GameData\\Audio\\VOC_JP\\JP_Airi.zip",
)
}
fn platform_inventory() -> YostarJpPlatformDownloadInventory {
YostarJpPlatformDownloadInventory::from_shared_inventory(
inventory(),
&[PatchPlatform::Windows],
)
}
fn fake_curl_script() -> String {
format!(
r#"#!/bin/sh
set -eu
out=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
if [ -n "$out" ]; then
mkdir -p "$(dirname "$out")"
case "$url" in
*/TableBundles/TableCatalog.hash)
printf '%s' '{table_catalog_hash}' > "$out"
;;
*/TableBundles/TableCatalog.bytes)
printf '%s' 'ExcelDB.db' > "$out"
;;
*/Windows_PatchPack/BundlePackingInfo.hash)
printf '%s' '{windows_bundle_hash}' > "$out"
;;
*/Android_PatchPack/BundlePackingInfo.hash)
printf '%s' '{android_bundle_hash}' > "$out"
;;
*/Windows_PatchPack/BundlePackingInfo.bytes)
printf '%s' 'FullPatch_000.zip' > "$out"
;;
*/Android_PatchPack/BundlePackingInfo.bytes)
printf '%s' 'FullPatch_001.zip' > "$out"
;;
*/MediaResources-Windows/Catalog/MediaCatalog.hash)
printf '%s' '{windows_media_hash}' > "$out"
;;
*/MediaResources/Catalog/MediaCatalog.hash)
printf '%s' '{android_media_hash}' > "$out"
;;
*/MediaResources-Windows/Catalog/MediaCatalog.bytes)
printf '%s' 'GameData\Audio\VOC_JP\JP_Airi_Win.zip' > "$out"
;;
*/MediaResources/Catalog/MediaCatalog.bytes)
printf '%s' 'GameData\Audio\VOC_JP\JP_Airi_Android.zip' > "$out"
;;
*.zip)
printf '\120\113\003\004\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000file.txtok\120\113\001\002\024\000\024\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\002\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000file.txt\120\113\005\006\000\000\000\000\001\000\001\000\066\000\000\000\050\000\000\000\000\000' > "$out"
;;
*)
printf '%s' "$url" > "$out"
;;
esac
else
printf '%s' "$url"
fi
"#,
table_catalog_hash = xxhash32(b"ExcelDB.db"),
windows_bundle_hash = xxhash32(b"FullPatch_000.zip"),
android_bundle_hash = xxhash32(b"FullPatch_001.zip"),
windows_media_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Win.zip"),
android_media_hash = xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip"),
)
}
fn fake_resume_curl_script() -> &'static str {
r#"#!/bin/sh
set -eu
out=""
url=""
resume=0
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--continue-at)
resume=1
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
if [ -n "$out" ]; then
mkdir -p "$(dirname "$out")"
if [ "$resume" -eq 1 ]; then
printf '%s' "-resumed" >> "$out"
else
printf '%s' "$url" > "$out"
fi
else
printf '%s' "$url"
fi
"#
}
fn fake_flaky_curl_script() -> &'static str {
r#"#!/bin/sh
set -eu
out=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
state="$0.state"
count=0
if [ -f "$state" ]; then
count="$(cat "$state")"
fi
count=$((count + 1))
printf '%s' "$count" > "$state"
if [ "$count" -eq 1 ]; then
printf '%s\n' "temporary failure" >&2
exit 22
fi
if [ -n "$out" ]; then
mkdir -p "$(dirname "$out")"
printf '%s' "$url" > "$out"
else
printf '%s' "$url"
fi
"#
}
fn fake_bad_hash_curl_script() -> &'static str {
r#"#!/bin/sh
set -eu
out=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
if [ -n "$out" ]; then
mkdir -p "$(dirname "$out")"
case "$url" in
*/TableBundles/TableCatalog.hash)
printf '%s' '1' > "$out"
;;
*/TableBundles/TableCatalog.bytes)
printf '%s' 'ExcelDB.db' > "$out"
;;
*)
printf '%s' "$url" > "$out"
;;
esac
else
printf '%s' "$url"
fi
"#
}
fn fake_bad_zip_curl_script() -> &'static str {
r#"#!/bin/sh
set -eu
out=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
if [ -n "$out" ]; then
mkdir -p "$(dirname "$out")"
printf '%s' 'not a zip archive' > "$out"
else
printf '%s' "$url"
fi
"#
}
fn fake_http_error_curl_script(status: u16) -> String {
format!(
r#"#!/bin/sh
set -eu
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
state="$0.state"
count=0
if [ -f "$state" ]; then
count="$(cat "$state")"
fi
count=$((count + 1))
printf '%s' "$count" > "$state"
printf 'curl: (22) The requested URL returned error: {status}\n' >&2
exit 22
"#
)
}
fn write_shell_script(path: &Path, script: &str) {
let temp_path = path.with_extension("tmp");
fs::write(&temp_path, script).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(&temp_path).unwrap().permissions();
permissions.set_mode(0o755);
fs::set_permissions(&temp_path, permissions).unwrap();
}
fs::rename(temp_path, path).unwrap();
}
fn write_fake_curl(path: &Path) {
write_shell_script(path, &fake_curl_script());
}
fn write_fake_resume_curl(path: &Path) {
write_shell_script(path, fake_resume_curl_script());
}
fn write_fake_flaky_curl(path: &Path) {
write_shell_script(path, fake_flaky_curl_script());
}
fn write_fake_bad_hash_curl(path: &Path) {
write_shell_script(path, fake_bad_hash_curl_script());
}
fn write_fake_bad_zip_curl(path: &Path) {
write_shell_script(path, fake_bad_zip_curl_script());
}
fn write_fake_http_error_curl(path: &Path, status: u16) {
write_shell_script(path, &fake_http_error_curl_script(status));
}
fn one_url_plan(url: &str) -> OfficialResourcePullPlan {
OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: url.to_string(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
}
}
fn one_file_zip(name: &[u8], data: &[u8]) -> Vec<u8> {
let mut bytes = Vec::new();
bytes.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
bytes.extend_from_slice(&20u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.extend_from_slice(&(data.len() as u32).to_le_bytes());
bytes.extend_from_slice(&(data.len() as u32).to_le_bytes());
bytes.extend_from_slice(&(name.len() as u16).to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(name);
bytes.extend_from_slice(data);
let central_offset = bytes.len() as u32;
bytes.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
bytes.extend_from_slice(&20u16.to_le_bytes());
bytes.extend_from_slice(&20u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.extend_from_slice(&(data.len() as u32).to_le_bytes());
bytes.extend_from_slice(&(data.len() as u32).to_le_bytes());
bytes.extend_from_slice(&(name.len() as u16).to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.extend_from_slice(name);
let central_size = bytes.len() as u32 - central_offset;
bytes.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes.extend_from_slice(&1u16.to_le_bytes());
bytes.extend_from_slice(&1u16.to_le_bytes());
bytes.extend_from_slice(&central_size.to_le_bytes());
bytes.extend_from_slice(&central_offset.to_le_bytes());
bytes.extend_from_slice(&0u16.to_le_bytes());
bytes
}
#[test]
fn downloads_verified_pull_plan_to_disk() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
let report = service.pull(&plan).unwrap();
assert_eq!(report.items.len(), 7);
assert!(report.total_bytes() > 0);
assert_eq!(report.downloaded_count(), report.items.len());
assert_eq!(report.skipped_count(), 0);
assert_eq!(report.resumed_count(), 0);
assert_eq!(report.transferred_bytes(), report.total_bytes());
assert_eq!(report.items[0].url, plan.discovery.endpoints[0].url);
for item in &report.items {
assert!(item.destination.exists());
assert_eq!(item.status, OfficialResourcePullStatus::Downloaded);
assert_eq!(item.bytes, item.transferred_bytes);
}
assert_eq!(report.official_hash_verified_count(), 1);
assert_eq!(
report.verified_hashes[0].algorithm,
OfficialResourceHashAlgorithm::XxHash32Decimal
);
assert_eq!(report.verified_hashes[0].expected, "2044170421");
assert_eq!(report.verified_hashes[0].actual, "2044170421");
let manifest = service.read_download_manifest().unwrap();
assert_eq!(manifest.version, DOWNLOAD_MANIFEST_VERSION);
assert_eq!(manifest.entries.len(), report.items.len());
for item in &report.items {
let entry = manifest.entries.get(&item.url).unwrap();
assert_eq!(entry.url, item.url);
assert_eq!(
entry.destination,
service.relative_destination(&item.destination).unwrap()
);
assert_eq!(entry.bytes, item.bytes);
assert_eq!(entry.blake3, blake3_file_hex(&item.destination).unwrap());
}
}
#[test]
fn rejects_dangerous_output_root() {
let service = OfficialResourcePullService::new(PathBuf::from("/"));
let error = service
.destination_for_url(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap_err();
assert!(error.to_string().contains("危险路径"));
}
#[test]
fn rejects_resource_url_with_query_or_fragment() {
let temp = TempDir::new().unwrap();
let service = OfficialResourcePullService::new(temp.path().join("out"));
for url in [
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes?v=1",
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes#frag",
] {
let error = service.destination_for_url(url).unwrap_err();
assert!(
error.contains("query 或 fragment"),
"unexpected error for {url}: {error}"
);
}
// 无 query 的正常资源 URL 仍可映射。
assert!(service
.destination_for_url(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.is_ok());
}
#[cfg(unix)]
#[test]
fn pull_rejects_symlink_parent_escape() {
use std::os::unix::fs::symlink;
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let escape_dir = temp.path().join("escape");
let bin_dir = TempDir::new().unwrap();
fs::create_dir_all(&out_dir).unwrap();
fs::create_dir_all(&escape_dir).unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path);
let destination = service
.destination_for_url(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap();
let host_dir = destination
.ancestors()
.find(|path| {
path.file_name().and_then(|name| name.to_str())
== Some("prod-clientpatch.bluearchiveyostar.com")
})
.unwrap()
.to_path_buf();
fs::create_dir_all(&host_dir).unwrap();
symlink(&escape_dir, host_dir.join("r93_token")).unwrap();
let error = service
.pull(&build_official_pull_plan(discovery_plan(), inventory()))
.unwrap_err();
assert!(error.to_string().contains("symlink"));
assert!(fs::read_dir(&escape_dir).unwrap().next().is_none());
}
#[cfg(unix)]
#[test]
fn pull_rejects_symlink_partial_file() {
use std::os::unix::fs::symlink;
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let escape_file = temp.path().join("escape.txt");
let bin_dir = TempDir::new().unwrap();
fs::create_dir_all(&out_dir).unwrap();
fs::write(&escape_file, b"outside").unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path);
let url = "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes";
let destination = service.destination_for_url(url).unwrap();
fs::create_dir_all(destination.parent().unwrap()).unwrap();
symlink(&escape_file, partial_path_for(&destination)).unwrap();
let error = service.pull_one(url, &destination).unwrap_err();
assert!(error.to_string().contains("symlink"));
assert_eq!(fs::read(&escape_file).unwrap(), b"outside");
}
#[test]
fn local_manifest_audit_detects_and_repairs_corrupted_file() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
let first_report = service.pull(&plan).unwrap();
let clean_audit = service.audit_local_manifest(&plan).unwrap();
assert!(clean_audit.is_clean());
assert_eq!(clean_audit.verified_count(), first_report.items.len());
assert_eq!(clean_audit.manifest_blake3_verified_count(), 7);
assert_eq!(clean_audit.zip_structure_verified_count(), 4);
let corrupted = first_report
.items
.iter()
.find(|item| item.url.ends_with("/Windows_PatchPack/FullPatch_000.zip"))
.unwrap();
let mut corrupted_bytes = fs::read(&corrupted.destination).unwrap();
corrupted_bytes[0] ^= 0xff;
fs::write(&corrupted.destination, corrupted_bytes).unwrap();
let corrupted_audit = service.audit_local_manifest(&plan).unwrap();
assert!(!corrupted_audit.is_clean());
assert_eq!(corrupted_audit.repair_needed_count(), 1);
assert!(corrupted_audit.items.iter().any(|item| {
item.url == corrupted.url
&& item.status == OfficialLocalManifestAuditStatus::Blake3Mismatch
}));
let repair_report = service.pull(&plan).unwrap();
assert!(repair_report.downloaded_count() >= 1);
assert!(service.audit_local_manifest(&plan).unwrap().is_clean());
}
#[test]
fn full_local_verification_checks_manifest_and_official_hash_pairs() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
service.pull(&plan).unwrap();
let verification = service.verify_local_download_manifest().unwrap();
assert!(verification.is_clean());
assert_eq!(verification.items.len(), 7);
assert_eq!(verification.verified_count(), 7);
assert_eq!(verification.manifest_blake3_verified_count(), 7);
assert_eq!(verification.zip_structure_verified_count(), 4);
assert_eq!(verification.official_hash_pair_count, 1);
assert_eq!(verification.official_hash_verified_count, 1);
let data_path = service
.destination_for_url(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap();
fs::write(data_path, b"corrupted").unwrap();
let corrupted = service.verify_local_download_manifest().unwrap();
assert!(!corrupted.is_clean());
assert_eq!(corrupted.failure_count(), 1);
assert_eq!(corrupted.official_hash_errors.len(), 1);
}
#[test]
fn refreshes_official_hash_pairs_and_skips_manifest_verified_content() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
let mut manifest = OfficialDownloadManifest::default();
for url in plan.all_urls().unwrap() {
let destination = service.destination_for_url(&url).unwrap();
fs::create_dir_all(destination.parent().unwrap()).unwrap();
let bytes = if url.ends_with("/TableBundles/TableCatalog.bytes") {
b"ExcelDB.db".to_vec()
} else if url.ends_with("/TableBundles/TableCatalog.hash") {
b"2044170421".to_vec()
} else if url.ends_with(".zip") {
one_file_zip(b"fixture.txt", b"ok")
} else {
url.as_bytes().to_vec()
};
fs::write(&destination, bytes).unwrap();
service
.record_download_manifest_entry(&mut manifest, &url, &destination)
.unwrap();
}
service.write_download_manifest(&manifest).unwrap();
let report = service.pull(&plan).unwrap();
assert_eq!(report.items.len(), 7);
assert_eq!(report.skipped_count(), 5);
assert_eq!(report.downloaded_count(), 2);
assert_eq!(report.resumed_count(), 0);
assert!(report.transferred_bytes() > 0);
assert_eq!(report.official_hash_verified_count(), 1);
assert_eq!(
fs::read_to_string(
service
.destination_for_url(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap()
)
.unwrap(),
"ExcelDB.db"
);
}
#[test]
fn redownloads_existing_files_without_manifest() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
for url in plan.all_urls().unwrap() {
let destination = service.destination_for_url(&url).unwrap();
fs::create_dir_all(destination.parent().unwrap()).unwrap();
fs::write(destination, b"stale-local-file").unwrap();
}
let report = service.pull(&plan).unwrap();
assert_eq!(report.items.len(), 7);
assert_eq!(report.skipped_count(), 0);
assert_eq!(report.downloaded_count(), 7);
assert_eq!(report.resumed_count(), 0);
assert_eq!(report.official_hash_verified_count(), 1);
assert!(report.transferred_bytes() > 0);
assert!(report
.items
.iter()
.all(|item| item.status == OfficialResourcePullStatus::Downloaded));
}
#[test]
fn local_resource_state_reports_manifest_entries_and_existing_files() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan(discovery_plan(), inventory());
let urls = plan.all_urls().unwrap();
let empty_state = service.local_resource_state(&plan).unwrap();
assert_eq!(empty_state.manifest_entry_count, 0);
assert_eq!(empty_state.existing_file_count, 0);
assert!(!empty_state.has_any_resources());
let first_destination = service.destination_for_url(&urls[0]).unwrap();
fs::create_dir_all(first_destination.parent().unwrap()).unwrap();
fs::write(&first_destination, b"local-only").unwrap();
let file_only_state = service.local_resource_state(&plan).unwrap();
assert_eq!(file_only_state.manifest_entry_count, 0);
assert_eq!(file_only_state.existing_file_count, 1);
assert!(file_only_state.has_any_resources());
let mut manifest = OfficialDownloadManifest::default();
service
.record_download_manifest_entry(&mut manifest, &urls[0], &first_destination)
.unwrap();
service.write_download_manifest(&manifest).unwrap();
let manifest_state = service.local_resource_state(&plan).unwrap();
assert_eq!(manifest_state.manifest_entry_count, 1);
assert_eq!(manifest_state.existing_file_count, 1);
assert!(manifest_state.has_any_resources());
}
#[test]
fn prune_stale_manifest_entries_removes_old_resource_and_partial_only() {
let out_dir = TempDir::new().unwrap();
let service = OfficialResourcePullService::new(out_dir.path());
let plan = build_official_pull_plan(discovery_plan(), inventory());
let expected_url = plan.all_urls().unwrap().into_iter().next().unwrap();
let expected_destination = service.destination_for_url(&expected_url).unwrap();
fs::create_dir_all(expected_destination.parent().unwrap()).unwrap();
fs::write(&expected_destination, b"keep").unwrap();
let stale_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle";
let stale_destination = service.destination_for_url(stale_url).unwrap();
fs::create_dir_all(stale_destination.parent().unwrap()).unwrap();
fs::write(&stale_destination, b"remove").unwrap();
fs::write(partial_path_for(&stale_destination), b"remove-partial").unwrap();
let mut manifest = OfficialDownloadManifest::default();
service
.record_download_manifest_entry(&mut manifest, &expected_url, &expected_destination)
.unwrap();
manifest.entries.insert(
stale_url.to_string(),
OfficialDownloadManifestEntry {
url: stale_url.to_string(),
destination: service.relative_destination(&stale_destination).unwrap(),
bytes: 6,
blake3: "stale".to_string(),
},
);
service.write_download_manifest(&manifest).unwrap();
let snapshot_path = out_dir.path().join("official-sync-snapshot.json");
let quarantine_path = out_dir.path().join(DOWNLOAD_QUARANTINE_FILE);
fs::write(&snapshot_path, b"snapshot").unwrap();
fs::write(&quarantine_path, b"quarantine").unwrap();
assert_eq!(service.prune_stale_manifest_entries(&plan).unwrap(), 1);
assert!(!stale_destination.exists());
assert!(!partial_path_for(&stale_destination).exists());
assert!(!stale_destination.parent().unwrap().exists());
assert!(expected_destination.exists());
assert_eq!(fs::read(&snapshot_path).unwrap(), b"snapshot");
assert_eq!(fs::read(&quarantine_path).unwrap(), b"quarantine");
let pruned_manifest = service.read_download_manifest().unwrap();
assert!(pruned_manifest.entries.contains_key(&expected_url));
assert!(!pruned_manifest.entries.contains_key(stale_url));
}
#[cfg(unix)]
#[test]
fn prune_stale_manifest_entries_rejects_symlink_destination() {
use std::os::unix::fs::symlink;
let out_dir = TempDir::new().unwrap();
let service = OfficialResourcePullService::new(out_dir.path());
let plan = build_official_pull_plan(discovery_plan(), inventory());
let stale_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle";
let stale_destination = service.destination_for_url(stale_url).unwrap();
fs::create_dir_all(stale_destination.parent().unwrap()).unwrap();
let outside = out_dir.path().join("outside-resource");
fs::write(&outside, b"must-keep").unwrap();
symlink(&outside, &stale_destination).unwrap();
let mut manifest = OfficialDownloadManifest::default();
manifest.entries.insert(
stale_url.to_string(),
OfficialDownloadManifestEntry {
url: stale_url.to_string(),
destination: service.relative_destination(&stale_destination).unwrap(),
bytes: 9,
blake3: "stale".to_string(),
},
);
service.write_download_manifest(&manifest).unwrap();
let error = service.prune_stale_manifest_entries(&plan).unwrap_err();
assert!(error.contains("symlink"));
assert!(stale_destination.exists());
assert_eq!(fs::read(&outside).unwrap(), b"must-keep");
assert!(service
.read_download_manifest()
.unwrap()
.entries
.contains_key(stale_url));
}
#[test]
fn redownloads_existing_file_when_manifest_hash_mismatches() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let first_report = service.pull(&plan).unwrap();
assert_eq!(first_report.downloaded_count(), first_report.items.len());
let first_item = &first_report.items[0];
fs::write(&first_item.destination, b"corrupted").unwrap();
let second_report = service
.pull(&OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: first_item.url.clone(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
})
.unwrap();
assert_eq!(second_report.items.len(), 1);
assert_eq!(second_report.skipped_count(), 0);
assert_eq!(second_report.downloaded_count(), 1);
assert_eq!(
fs::read_to_string(&first_item.destination).unwrap(),
"ExcelDB.db"
);
}
#[test]
fn rejects_seed_catalog_when_official_hash_mismatches() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_bad_hash_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = OfficialResourcePullPlan {
discovery: discovery_plan(),
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
};
let error = service.pull(&plan).unwrap_err();
assert!(error.to_string().contains("官方 hash 校验失败"));
let manifest = service.read_download_manifest().unwrap();
assert!(manifest.entries.is_empty());
let audit = service.audit_local_manifest(&plan).unwrap();
assert!(!audit.is_clean());
assert_eq!(audit.repair_needed_count(), 2);
assert!(audit
.items
.iter()
.all(|item| { item.status == OfficialLocalManifestAuditStatus::MissingManifestEntry }));
}
#[test]
fn verifies_known_real_seed_catalog_hash_samples() {
assert_eq!(xxhash32(b""), 46947589);
assert_eq!(xxhash32(b"ExcelDB.db"), 2044170421);
assert_eq!(xxhash32(b"FullPatch_000.zip"), 4038880697);
assert_eq!(
xxhash32(b"GameData\\Audio\\VOC_JP\\JP_Airi_Win.zip"),
1865771294
);
assert_eq!(xxhash32(b"ExcelDB.db ExcelDB.db"), 2147704569);
}
#[test]
fn rejects_downloaded_zip_when_structure_is_invalid() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_bad_zip_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let zip_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip";
let plan = OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::AddressablesCatalog,
platform: Some(PatchPlatform::Windows),
url: zip_url.to_string(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
};
let error = service.pull(&plan).unwrap_err();
assert!(error.to_string().contains("ZIP"));
assert!(!partial_path_for(&service.destination_for_url(zip_url).unwrap()).exists());
assert!(service.read_download_manifest().unwrap().entries.is_empty());
}
#[test]
fn local_manifest_audit_rejects_zip_with_matching_size_and_blake3_but_bad_structure() {
let out_dir = TempDir::new().unwrap();
let service = OfficialResourcePullService::new(out_dir.path());
let zip_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/Windows_PatchPack/Broken.zip";
let destination = service.destination_for_url(zip_url).unwrap();
fs::create_dir_all(destination.parent().unwrap()).unwrap();
fs::write(&destination, b"not a zip archive").unwrap();
let mut manifest = OfficialDownloadManifest::default();
let bytes = fs::metadata(&destination).unwrap().len();
let blake3 = blake3_file_hex(&destination).unwrap();
manifest.entries.insert(
zip_url.to_string(),
OfficialDownloadManifestEntry {
url: zip_url.to_string(),
destination: service.relative_destination(&destination).unwrap(),
bytes,
blake3,
},
);
service.write_download_manifest(&manifest).unwrap();
let plan = OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::AddressablesCatalog,
platform: Some(PatchPlatform::Windows),
url: zip_url.to_string(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
};
let audit = service.audit_local_manifest(&plan).unwrap();
let verification = service.verify_local_download_manifest().unwrap();
assert_eq!(
audit.items[0].status,
OfficialLocalManifestAuditStatus::ZipStructureInvalid
);
assert!(audit.items[0].zip_error.as_ref().unwrap().contains("ZIP"));
assert!(!verification.is_clean());
assert_eq!(verification.failure_count(), 1);
}
#[test]
fn resumes_partial_files_before_atomic_completion() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_resume_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let first_url = plan.all_urls().unwrap().remove(0);
let destination = service.destination_for_url(&first_url).unwrap();
fs::create_dir_all(destination.parent().unwrap()).unwrap();
let partial = partial_path_for(&destination);
fs::write(&partial, b"partial").unwrap();
let report = service
.pull(&OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: first_url,
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
})
.unwrap();
assert_eq!(report.items.len(), 1);
assert_eq!(report.resumed_count(), 1);
assert_eq!(report.items[0].status, OfficialResourcePullStatus::Resumed);
assert_eq!(fs::read_to_string(&destination).unwrap(), "partial-resumed");
assert!(!partial.exists());
let manifest = service.read_download_manifest().unwrap();
assert_eq!(manifest.entries.len(), 1);
assert!(manifest.entries.contains_key(&report.items[0].url));
}
#[test]
fn pull_emits_started_and_finished_progress_events() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
// 并发下事件顺序不固定,此处只断言与顺序无关的不变量;并发正确性另有
// downloads_run_concurrently_and_each_url_reports_once 专测。
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let expected_urls = plan.all_urls().unwrap().len();
let mut events = Vec::new();
let report = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap();
assert_eq!(report.items.len(), expected_urls);
let started: Vec<_> = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Started)
.collect();
let finished: Vec<_> = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
.collect();
let verifications: Vec<_> = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Verification)
.collect();
assert_eq!(started.len(), expected_urls);
assert_eq!(finished.len(), expected_urls);
assert_eq!(verifications.len(), report.verified_hashes.len());
for verification_event in &verifications {
let hash = verification_event
.official_hash
.as_ref()
.expect("verification event must carry official hash detail");
let pair_finished_indices = [hash.data_url.as_str(), hash.hash_url.as_str()]
.into_iter()
.map(|url| {
events
.iter()
.position(|event| {
event.kind == OfficialResourcePullProgressKind::Finished
&& event.url == url
})
.expect("hash pair member must finish before verification")
})
.collect::<Vec<_>>();
let verification_index = events
.iter()
.position(|event| std::ptr::eq(event, *verification_event))
.expect("verification event must be present in event stream");
assert_eq!(
verification_index,
pair_finished_indices.into_iter().max().unwrap() + 1,
"official hash verification must run immediately after the hash pair is complete"
);
}
// 每个 started 的 total 一致;index 表示已完成数量,不能超过总数。
assert!(started.iter().all(|event| event.index <= expected_urls));
assert!(started.iter().all(|event| event.total == expected_urls));
assert!(events.windows(2).all(|pair| pair[0].index <= pair[1].index));
let finished_indices: Vec<usize> = finished.iter().map(|event| event.index).collect();
assert_eq!(finished_indices, (1..=expected_urls).collect::<Vec<_>>());
// started 与 finished 覆盖同一组 URLfinished 均为已下载。
let started_urls: HashSet<_> = started.iter().map(|event| event.url.clone()).collect();
let finished_urls: HashSet<_> = finished.iter().map(|event| event.url.clone()).collect();
assert_eq!(started_urls, finished_urls);
assert!(finished
.iter()
.all(|event| event.status == Some(OfficialResourcePullStatus::Downloaded)));
assert!(finished.iter().all(|event| {
event.verification.as_ref().is_some_and(|verification| {
verification.actual_bytes > 0 && !verification.actual_blake3.is_empty()
})
}));
assert!(verifications.iter().all(|event| {
event.official_hash.as_ref().is_some_and(|hash| {
hash.algorithm == OfficialResourceHashAlgorithm::XxHash32Decimal
&& hash.expected == hash.actual
&& !hash.hash_url.is_empty()
})
}));
assert!(events
.iter()
.any(|event| event.url.ends_with("/TableBundles/ExcelDB.db")));
}
#[test]
fn downloads_run_sequentially_and_each_url_reports_once() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
// 顺序下载:每个 URL 恰好一次 started + 一次 finished,全部文件落盘。
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_max_concurrency(1);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let all_urls = plan.all_urls().unwrap();
let mut events = Vec::new();
let report = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap();
assert_eq!(report.items.len(), all_urls.len());
for url in &all_urls {
let started = events
.iter()
.filter(|event| {
&event.url == url && event.kind == OfficialResourcePullProgressKind::Started
})
.count();
let finished = events
.iter()
.filter(|event| {
&event.url == url && event.kind == OfficialResourcePullProgressKind::Finished
})
.count();
assert_eq!(started, 1, "url {url} 的 started 次数");
assert_eq!(finished, 1, "url {url} 的 finished 次数");
}
// 所有下载都记入 manifest。
let manifest = service.read_download_manifest().unwrap();
assert_eq!(manifest.entries.len(), all_urls.len());
}
#[test]
fn downloads_run_with_bounded_concurrency_and_keep_report_order() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_max_concurrency(3);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let all_urls = plan.all_urls().unwrap();
let mut events = Vec::new();
let report = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap();
assert_eq!(service.max_concurrency(), 3);
assert_eq!(report.items.len(), all_urls.len());
assert_eq!(
report
.items
.iter()
.map(|item| &item.url)
.collect::<Vec<_>>(),
all_urls.iter().collect::<Vec<_>>()
);
for url in &all_urls {
assert_eq!(
events
.iter()
.filter(|event| {
event.url == *url && event.kind == OfficialResourcePullProgressKind::Started
})
.count(),
1,
"url {url} 的 started 次数"
);
assert_eq!(
events
.iter()
.filter(|event| {
event.url == *url
&& event.kind == OfficialResourcePullProgressKind::Finished
})
.count(),
1,
"url {url} 的 finished 次数"
);
}
let finished_indices = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
.map(|event| event.index)
.collect::<Vec<_>>();
assert_eq!(finished_indices, (1..=all_urls.len()).collect::<Vec<_>>());
assert_eq!(
service.read_download_manifest().unwrap().entries.len(),
all_urls.len()
);
}
#[test]
fn retries_transient_download_failures() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_flaky_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(2);
let plan = OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root:
"https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes".to_string(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
};
let report = service.pull(&plan).unwrap();
assert_eq!(report.items.len(), 1);
assert_eq!(report.downloaded_count(), 1);
assert_eq!(
fs::read_to_string(&report.items[0].destination).unwrap(),
report.items[0].url
);
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"2"
);
}
#[test]
fn does_not_retry_terminal_http_404_and_records_quarantine() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_http_error_curl(&curl_path, 404);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(3);
let url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Missing.bytes";
let plan = one_url_plan(url);
let mut events = Vec::new();
let error = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap_err();
assert!(error.to_string().contains("quarantine"));
assert!(error.to_string().contains("http_not_found"));
assert!(error.to_string().contains("retryable=false"));
// 类型化错误携带准确的 HTTP 404 网络域码。
assert_eq!(error.code(), bat_core::ErrorCode::HTTP_NOT_FOUND);
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"1"
);
let failed = events
.iter()
.find(|event| event.kind == OfficialResourcePullProgressKind::Failed)
.unwrap();
assert_eq!(failed.failure_kind.as_deref(), Some("http_not_found"));
assert_eq!(failed.failure_http_status, Some(404));
assert_eq!(failed.failure_retryable, Some(false));
assert_eq!(failed.failure_attempts, Some(1));
assert!(failed.quarantined);
let quarantine = service.read_download_quarantine().unwrap();
let entry = quarantine.entries.get(url).unwrap();
assert_eq!(entry.http_status, Some(404));
assert_eq!(entry.failure_kind.as_deref(), Some("http_not_found"));
assert_eq!(entry.retryable, Some(false));
assert_eq!(entry.attempts, 1);
assert!(service.read_download_manifest().unwrap().entries.is_empty());
}
#[test]
fn official_http_failure_regression_fixtures_are_enforced() {
for fixture in [
include_str!("../tests/fixtures/official_regression/http_403.json"),
include_str!("../tests/fixtures/official_regression/http_404.json"),
] {
let fixture: serde_json::Value = serde_json::from_str(fixture).unwrap();
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
let status = fixture["http_status"].as_u64().unwrap() as u16;
write_fake_http_error_curl(&curl_path, status);
let service =
OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(3);
let url = fixture["url"].as_str().unwrap();
let error = service.pull(&one_url_plan(url)).unwrap_err();
let expected_kind = fixture["failure_kind"].as_str().unwrap();
let expected_retryable = fixture["retryable"].as_bool().unwrap();
let expected_attempts = fixture["expected_attempts"].as_u64().unwrap();
assert!(error.to_string().contains(expected_kind));
assert!(error
.to_string()
.contains(&format!("retryable={expected_retryable}")));
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
expected_attempts.to_string()
);
let quarantine = service.read_download_quarantine().unwrap();
let entry = quarantine.entries.get(url).unwrap();
assert_eq!(entry.failure_kind.as_deref(), Some(expected_kind));
assert_eq!(entry.retryable, Some(expected_retryable));
assert_eq!(entry.attempts, expected_attempts as usize);
}
}
#[test]
fn official_hash_mismatch_regression_fixture_is_enforced() {
let fixture: serde_json::Value = serde_json::from_str(include_str!(
"../tests/fixtures/official_regression/hash_mismatch_catalog.json"
))
.unwrap();
let error = verify_official_seed_catalog_hash(
fixture["data_url"].as_str().unwrap(),
fixture["data"].as_str().unwrap().as_bytes(),
fixture["hash_url"].as_str().unwrap(),
fixture["official_hash"].as_str().unwrap().as_bytes(),
)
.unwrap_err();
assert!(error
.to_string()
.contains(fixture["expected_error_contains"].as_str().unwrap()));
}
#[test]
fn retries_http_5xx_before_quarantine() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_http_error_curl(&curl_path, 503);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(3);
let url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Busy.bytes";
let plan = one_url_plan(url);
let error = service.pull(&plan).unwrap_err();
assert!(error.to_string().contains("quarantine"));
assert!(error.to_string().contains("http_server_error"));
assert!(error.to_string().contains("retryable=true"));
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"3"
);
let quarantine = service.read_download_quarantine().unwrap();
let entry = quarantine.entries.get(url).unwrap();
assert_eq!(entry.http_status, Some(503));
assert_eq!(entry.failure_kind.as_deref(), Some("http_server_error"));
assert_eq!(entry.retryable, Some(true));
assert_eq!(entry.attempts, 3);
assert!(entry.last_error.contains("重试次数已耗尽"));
}
#[test]
fn custom_platform_selection_is_supported() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows, PatchPlatform::Android],
);
let report = service.pull(&plan).unwrap();
assert_eq!(report.items.len(), 7);
assert!(report
.items
.iter()
.any(|item| item.url.contains("Android_PatchPack")));
}
#[test]
fn reuses_verified_historical_release_when_cdn_root_changes() {
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let versions_dir = temp.path().join("versions");
let old_root = versions_dir.join("release-old");
let old_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let current_url =
"https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes";
let destination =
"prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let bytes = b"verified historical resource";
let source = old_root.join(destination);
fs::create_dir_all(source.parent().unwrap()).unwrap();
fs::write(&source, bytes).unwrap();
write_historical_manifest(&old_root, old_url, destination, bytes);
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path)
.with_release_reuse_root(&versions_dir);
let report = service.pull(&one_url_plan(current_url)).unwrap();
assert_eq!(report.release_reused_count(), 1);
assert_eq!(report.cas_reused_count(), 0);
assert_eq!(report.downloaded_count(), 0);
assert_eq!(report.reused_bytes(), bytes.len() as u64);
assert_eq!(
fs::read(service.destination_for_url(current_url).unwrap()).unwrap(),
bytes
);
assert_eq!(
report.items[0].status,
OfficialResourcePullStatus::ReleaseReused
);
assert!(report.items[0].cas_object_id.is_none());
assert!(report.reuse_warnings.is_empty());
let manifest = service.read_download_manifest().unwrap();
assert!(manifest.entries.contains_key(current_url));
}
#[test]
fn falls_back_to_cas_after_corrupt_historical_release() {
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let versions_dir = temp.path().join("versions");
let old_root = versions_dir.join("release-old");
let cas_root = temp.path().join("cas");
let old_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let current_url =
"https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes";
let destination =
"prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let bytes = b"cas-backed resource";
let source = old_root.join(destination);
fs::create_dir_all(source.parent().unwrap()).unwrap();
fs::write(&source, b"corrupt").unwrap();
write_historical_manifest(&old_root, old_url, destination, bytes);
let object_id = store_cas_object(&cas_root, bytes);
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path)
.with_release_reuse_root(&versions_dir)
.with_cas_reuse_root(&cas_root);
let report = service.pull(&one_url_plan(current_url)).unwrap();
assert_eq!(report.release_reused_count(), 0);
assert_eq!(report.cas_reused_count(), 1);
assert_eq!(report.downloaded_count(), 0);
assert_eq!(
report.items[0].cas_object_id.as_deref(),
Some(object_id.as_str())
);
assert_eq!(
fs::read(service.destination_for_url(current_url).unwrap()).unwrap(),
bytes
);
assert!(report
.reuse_warnings
.iter()
.any(|warning| warning.source == "historical_release"));
let references = read_cas_reuse_reference_manifest_at(&out_dir)
.unwrap()
.unwrap();
assert_eq!(references.object_ids, vec![object_id.clone()]);
assert_eq!(cas_reference_count(&cas_root, &object_id), 2);
assert_eq!(
release_cas_reuse_references(&out_dir, &cas_root).unwrap(),
1
);
assert_eq!(cas_reference_count(&cas_root, &object_id), 1);
assert!(read_cas_reuse_reference_manifest_at(&out_dir)
.unwrap()
.is_none());
// 模拟 CAS 事务已提交但 release-local progress cursor 尚未写回;
// 重试必须识别同一个 ownership pair,而不是再次递减。
fs::write(
out_dir.join(OFFICIAL_CAS_REUSE_REFERENCES_FILE),
serde_json::to_vec(&OfficialCasReuseReferenceManifest {
version: OFFICIAL_CAS_REUSE_REFERENCES_VERSION,
object_ids: vec![object_id.clone()],
})
.unwrap(),
)
.unwrap();
assert_eq!(
release_cas_reuse_references(&out_dir, &cas_root).unwrap(),
0
);
assert_eq!(cas_reference_count(&cas_root, &object_id), 1);
assert!(read_cas_reuse_reference_manifest_at(&out_dir)
.unwrap()
.is_none());
}
#[test]
fn corrupted_cas_falls_back_to_network_with_diagnostic() {
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let versions_dir = temp.path().join("versions");
let old_root = versions_dir.join("release-old");
let cas_root = temp.path().join("cas");
let old_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let current_url =
"https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes";
let destination =
"prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let expected = b"expected cas resource";
write_historical_manifest(&old_root, old_url, destination, expected);
let object_id = store_cas_object(&cas_root, expected);
fs::write(cas_object_path(&cas_root, &object_id).unwrap(), b"corrupt").unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path)
.with_release_reuse_root(&versions_dir)
.with_cas_reuse_root(&cas_root);
let report = service.pull(&one_url_plan(current_url)).unwrap();
assert_eq!(report.downloaded_count(), 1);
assert_eq!(report.cas_reused_count(), 0);
assert_eq!(
fs::read(service.destination_for_url(current_url).unwrap()).unwrap(),
current_url.as_bytes()
);
assert!(report
.reuse_warnings
.iter()
.any(|warning| warning.source == "cas" && warning.message.contains("完整性")));
assert!(read_cas_reuse_reference_manifest_at(&out_dir)
.unwrap()
.is_none());
}
#[test]
fn missing_cas_root_does_not_create_storage_during_fallback() {
let temp = TempDir::new().unwrap();
let out_dir = temp.path().join("out");
let versions_dir = temp.path().join("versions");
let old_root = versions_dir.join("release-old");
let cas_root = temp.path().join("cas");
let old_url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let current_url =
"https://prod-clientpatch.bluearchiveyostar.com/r94_new/TableBundles/Reusable.bytes";
let destination =
"prod-clientpatch.bluearchiveyostar.com/r93_old/TableBundles/Reusable.bytes";
let expected = b"network fallback resource";
let source = old_root.join(destination);
fs::create_dir_all(source.parent().unwrap()).unwrap();
fs::write(&source, b"corrupt").unwrap();
write_historical_manifest(&old_root, old_url, destination, expected);
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(&out_dir, &curl_path)
.with_release_reuse_root(&versions_dir)
.with_cas_reuse_root(&cas_root);
let report = service.pull(&one_url_plan(current_url)).unwrap();
assert_eq!(report.downloaded_count(), 1);
assert!(!cas_root.exists());
assert!(report.reuse_warnings.iter().any(|warning| {
warning.source == "cas" && warning.message.contains("不存在或不可读取")
}));
}
#[test]
fn fetches_official_seed_bytes_with_curl() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let bytes = service
.fetch_bytes(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap();
assert_eq!(
String::from_utf8(bytes).unwrap(),
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes"
);
}
#[test]
fn retries_transient_fetch_bytes_failures() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_flaky_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(2);
let bytes = service
.fetch_bytes(
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
)
.unwrap();
assert_eq!(
String::from_utf8(bytes).unwrap(),
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes"
);
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"2"
);
}
#[test]
fn rejects_non_official_urls() {
let service = OfficialResourcePullService::new("/tmp/unused");
let plan = OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root:
"https://prod-clientpatch.bluearchiveyostar.com/r93_token".to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: "https://prod-clientpatch.bluearchive.cafe/r93_token/TableBundles/TableCatalog.bytes".to_string(),
}],
},
inventory: platform_inventory(),
platforms: vec![PatchPlatform::Windows],
};
assert!(service.pull(&plan).is_err());
}
fn write_historical_manifest(root: &Path, url: &str, destination: &str, bytes: &[u8]) {
fs::create_dir_all(root).unwrap();
let mut manifest = OfficialDownloadManifest::default();
manifest.entries.insert(
url.to_string(),
OfficialDownloadManifestEntry {
url: url.to_string(),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
);
fs::write(
root.join(DOWNLOAD_MANIFEST_FILE),
serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
}
fn store_cas_object(root: &Path, bytes: &[u8]) -> String {
let cas = crate::FileSystemCasRepository::new(root);
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async { cas.store(bytes).await.unwrap() })
}
fn cas_reference_count(root: &Path, object_id: &str) -> u64 {
let cas = crate::FileSystemCasRepository::new(root);
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
cas.get_reference_count(&object_id.to_string())
.await
.unwrap()
})
}
}