nyaKazuha/BlueArchiveToolkit
1
0
Fork 0
mirror of https://github.com/Yuyi-Oak/BlueArchiveToolkit.git synced 2026-07-22 01:15:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: classify official download failures

Browse Source 
Distinguish terminal HTTP errors from retryable CDN/network failures, record quarantined resources, and report failed download progress.

Fixes #7
This commit is contained in:
nyaKazuha
2026-07-14 01:30:12 +08:00
parent 25c2c4d40f
commit 4192d7ee4c
15 changed files with 1034 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infrastructure/src/bin/bat_official_sync.rs
+45
View File
@@ -503,6 +503,16 @@ struct DaemonDownloadProgress {
status: Option<String>,
bytes: Option<u64>,
transferred_bytes: Option<u64>,
#[serde(default)]
failure_kind: Option<String>,
#[serde(default)]
failure_http_status: Option<u16>,
#[serde(default)]
failure_retryable: Option<bool>,
#[serde(default)]
failure_attempts: Option<usize>,
#[serde(default)]
quarantined: Option<bool>,
}
#[derive(Debug)]
@@ -1980,6 +1990,31 @@ fn print_list(label: &str, values: &[String], limit: usize) {
fn format_daemon_download_progress(progress: &DaemonDownloadProgress) -> String {
let status = progress.status.as_deref().unwrap_or("running");
if status == "failed" {
return format!(
"{}/{} failed kind={} http={} retryable={} attempts={} quarantined={} {}",
progress.index,
progress.total,
progress.failure_kind.as_deref().unwrap_or("unknown"),
progress
.failure_http_status
.map(|status| status.to_string())
.unwrap_or_else(|| "none".to_string()),
progress
.failure_retryable
.map(|retryable| retryable.to_string())
.unwrap_or_else(|| "unknown".to_string()),
progress
.failure_attempts
.map(|attempts| attempts.to_string())
.unwrap_or_else(|| "0".to_string()),
progress
.quarantined
.map(|quarantined| quarantined.to_string())
.unwrap_or_else(|| "false".to_string()),
progress.url
);
}
format!(
"{}/{} {} {}",
progress.index, progress.total, status, progress.url
@@ -2900,6 +2935,11 @@ fn update_daemon_progress(state_dir: &Path, event: &OfficialUpdateProgress) -> a
status: event.download_status.clone(),
bytes: event.download_bytes,
transferred_bytes: event.download_transferred_bytes,
failure_kind: event.download_failure_kind.clone(),
failure_http_status: event.download_failure_http_status,
failure_retryable: event.download_failure_retryable,
failure_attempts: event.download_failure_attempts,
quarantined: event.download_quarantined,
}),
_ if event.stage != "download" => None,
_ => status.download_progress,
@@ -3354,6 +3394,11 @@ impl RotatingStructuredLogger {
"status": event.download_status.as_deref(),
"bytes": event.download_bytes,
"transferred_bytes": event.download_transferred_bytes,
"failure_kind": event.download_failure_kind.as_deref(),
"failure_http_status": event.download_failure_http_status,
"failure_retryable": event.download_failure_retryable,
"failure_attempts": event.download_failure_attempts,
"quarantined": event.download_quarantined,
})),
});
let mut line = serde_json::to_vec(&payload)?;
infrastructure/src/curl_transfer.rs
+308
View File
@@ -0,0 +1,308 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum CurlFailureKind {
HttpForbidden,
HttpNotFound,
HttpClientError,
HttpTooManyRequests,
HttpServerError,
HttpUnknown,
Dns,
Connect,
Timeout,
Tls,
Interrupted,
Network,
ProcessFailed,
Unknown,
}
impl CurlFailureKind {
pub(crate) fn as_str(&self) -> &'static str {
match self {
Self::HttpForbidden => "http_forbidden",
Self::HttpNotFound => "http_not_found",
Self::HttpClientError => "http_client_error",
Self::HttpTooManyRequests => "http_too_many_requests",
Self::HttpServerError => "http_server_error",
Self::HttpUnknown => "http_unknown",
Self::Dns => "dns",
Self::Connect => "connect",
Self::Timeout => "timeout",
Self::Tls => "tls",
Self::Interrupted => "interrupted",
Self::Network => "network",
Self::ProcessFailed => "process_failed",
Self::Unknown => "unknown",
}
}
pub(crate) fn is_retryable(&self) -> bool {
match self {
Self::HttpForbidden
| Self::HttpNotFound
| Self::HttpClientError
| Self::ProcessFailed => false,
Self::HttpTooManyRequests
| Self::HttpServerError
| Self::HttpUnknown
| Self::Dns
| Self::Connect
| Self::Timeout
| Self::Tls
| Self::Interrupted
| Self::Network
| Self::Unknown => true,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CurlFailure {
pub(crate) url: String,
pub(crate) destination: Option<PathBuf>,
pub(crate) exit_code: Option<i32>,
pub(crate) stderr: String,
pub(crate) http_status: Option<u16>,
pub(crate) kind: CurlFailureKind,
}
impl CurlFailure {
fn from_output(url: &str, destination: Option<&Path>, output: &Output) -> Self {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
let exit_code = output.status.code();
let http_status = parse_http_status(&stderr);
let kind = classify_failure(exit_code, http_status);
Self {
url: url.to_string(),
destination: destination.map(Path::to_path_buf),
exit_code,
stderr,
http_status,
kind,
}
}
fn process_failed(
url: &str,
destination: Option<&Path>,
curl_command: &Path,
error: std::io::Error,
) -> Self {
Self {
url: url.to_string(),
destination: destination.map(Path::to_path_buf),
exit_code: None,
stderr: format!("启动 curl 命令失败 {}：{error}", curl_command.display()),
http_status: None,
kind: CurlFailureKind::ProcessFailed,
}
}
pub(crate) fn retryable(&self) -> bool {
self.kind.is_retryable()
}
fn format_compact(&self) -> String {
let mut parts = vec![
format!("kind={}", self.kind.as_str()),
format!("retryable={}", self.retryable()),
format!("url={}", self.url),
];
if let Some(status) = self.http_status {
parts.push(format!("http_status={status}"));
}
if let Some(code) = self.exit_code {
parts.push(format!("curl_exit={code}"));
}
if let Some(destination) = self.destination.as_ref() {
parts.push(format!("destination={}", destination.display()));
}
if !self.stderr.is_empty() {
parts.push(format!("stderr={}", self.stderr));
}
parts.join(" ")
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CurlAttemptFailure {
pub(crate) attempt: usize,
pub(crate) attempts: usize,
pub(crate) failure: CurlFailure,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CurlRetryError {
pub(crate) attempts: Vec<CurlAttemptFailure>,
}
impl CurlRetryError {
pub(crate) fn final_failure(&self) -> Option<&CurlFailure> {
self.attempts.last().map(|attempt| &attempt.failure)
}
pub(crate) fn attempt_count(&self) -> usize {
self.attempts.len()
}
pub(crate) fn final_kind_label(&self) -> Option<&'static str> {
self.final_failure().map(|failure| failure.kind.as_str())
}
pub(crate) fn final_http_status(&self) -> Option<u16> {
self.final_failure().and_then(|failure| failure.http_status)
}
pub(crate) fn final_retryable(&self) -> Option<bool> {
self.final_failure().map(CurlFailure::retryable)
}
pub(crate) fn summary(&self) -> String {
let Some(final_failure) = self.final_failure() else {
return "curl 未执行就已失败".to_string();
};
let policy = if final_failure.retryable() {
"重试次数已耗尽"
} else {
"不可重试，已提前停止"
};
let attempts = self
.attempts
.iter()
.map(|attempt| {
format!(
"第 {}/{} 次尝试失败：{}",
attempt.attempt,
attempt.attempts,
attempt.failure.format_compact()
)
})
.collect::<Vec<_>>()
.join("；");
format!("{policy}；{attempts}")
}
}
impl std::fmt::Display for CurlRetryError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.summary())
}
}
impl std::error::Error for CurlRetryError {}
pub(crate) fn run_curl_with_retry(
curl_command: &Path,
url: &str,
destination: Option<&Path>,
attempts: usize,
mut build_command: impl FnMut() -> Command,
) -> Result<Output, CurlRetryError> {
let attempts = attempts.max(1);
let mut failures = Vec::new();
for attempt in 1..=attempts {
let failure = match build_command().output() {
Ok(output) if output.status.success() => return Ok(output),
Ok(output) => CurlFailure::from_output(url, destination, &output),
Err(error) => CurlFailure::process_failed(url, destination, curl_command, error),
};
let retryable = failure.retryable();
failures.push(CurlAttemptFailure {
attempt,
attempts,
failure,
});
if !retryable {
break;
}
}
Err(CurlRetryError { attempts: failures })
}
fn classify_failure(exit_code: Option<i32>, http_status: Option<u16>) -> CurlFailureKind {
if exit_code == Some(22) {
return match http_status {
Some(403) => CurlFailureKind::HttpForbidden,
Some(404) => CurlFailureKind::HttpNotFound,
Some(408 | 425 | 429) => CurlFailureKind::HttpTooManyRequests,
Some(400..=499) => CurlFailureKind::HttpClientError,
Some(500..=599) => CurlFailureKind::HttpServerError,
Some(_) => CurlFailureKind::HttpUnknown,
None => CurlFailureKind::HttpUnknown,
};
}
match exit_code {
Some(5 | 6) => CurlFailureKind::Dns,
Some(7) => CurlFailureKind::Connect,
Some(18) => CurlFailureKind::Interrupted,
Some(28) => CurlFailureKind::Timeout,
Some(35) => CurlFailureKind::Tls,
Some(52 | 56) => CurlFailureKind::Network,
Some(_) => CurlFailureKind::Unknown,
None => CurlFailureKind::ProcessFailed,
}
}
fn parse_http_status(stderr: &str) -> Option<u16> {
let bytes = stderr.as_bytes();
let mut index = 0usize;
while index < bytes.len() {
if !bytes[index].is_ascii_digit() {
index += 1;
continue;
}
let start = index;
while index < bytes.len() && bytes[index].is_ascii_digit() {
index += 1;
}
if index - start == 3 {
let status = stderr[start..index].parse::<u16>().ok()?;
if (100..=599).contains(&status) {
return Some(status);
}
}
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use std::os::unix::process::ExitStatusExt;
#[test]
fn classifies_http_status_codes_from_curl_stderr() {
let status = std::process::ExitStatus::from_raw(22 << 8);
let output = Output {
status,
stdout: Vec::new(),
stderr: b"curl: (22) The requested URL returned error: 404".to_vec(),
};
let failure = CurlFailure::from_output("https://example.test/a", None, &output);
assert_eq!(failure.http_status, Some(404));
assert_eq!(failure.kind, CurlFailureKind::HttpNotFound);
assert!(!failure.retryable());
}
#[test]
fn treats_5xx_as_retryable() {
let status = std::process::ExitStatus::from_raw(22 << 8);
let output = Output {
status,
stdout: Vec::new(),
stderr: b"curl: (22) The requested URL returned error: 503".to_vec(),
};
let failure = CurlFailure::from_output("https://example.test/a", None, &output);
assert_eq!(failure.http_status, Some(503));
assert_eq!(failure.kind, CurlFailureKind::HttpServerError);
assert!(failure.retryable());
}
}
infrastructure/src/lib.rs
+1
View File
@@ -11,6 +11,7 @@
#![warn(clippy::all)]
pub mod cas;
mod curl_transfer;
pub mod import;
pub mod official_download;
pub mod official_game_main_config;
infrastructure/src/official_download.rs
+453 -105
View File
@@ -1,5 +1,6 @@
//! Official JP resource download execution.
use crate::curl_transfer::{run_curl_with_retry, CurlRetryError};
use crate::official_pull::OfficialResourcePullPlan;
use crate::path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target,
@@ -15,9 +16,12 @@ use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
const DOWNLOAD_MANIFEST_FILE: &str = "official-download-manifest.json";
const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json";
const DOWNLOAD_MANIFEST_VERSION: u32 = 1;
const DOWNLOAD_QUARANTINE_VERSION: u32 = 1;
const DEFAULT_RETRY_ATTEMPTS: usize = 3;
/// Outcome for one official resource pull item.
@@ -49,6 +53,8 @@ pub enum OfficialResourcePullProgressKind {
Started,
/// A URL finished as skipped, resumed, or downloaded.
Finished,
/// A URL failed after retry classification and was quarantined for this run.
Failed,
}
impl OfficialResourcePullProgressKind {
@@ -57,6 +63,7 @@ impl OfficialResourcePullProgressKind {
match self {
Self::Started => "started",
Self::Finished => "finished",
Self::Failed => "failed",
}
}
}
@@ -78,6 +85,16 @@ pub struct OfficialResourcePullProgress {
pub bytes: Option<u64>,
/// Bytes transferred during this run, when `kind` is `Finished`.
pub transferred_bytes: Option<u64>,
/// Stable failure kind label, when `kind` is `Failed`.
pub failure_kind: Option<String>,
/// HTTP status parsed from curl stderr, when available.
pub failure_http_status: Option<u16>,
/// Whether retry policy considered the final failure retryable.
pub failure_retryable: Option<bool>,
/// Number of curl attempts that were executed before this failure.
pub failure_attempts: Option<usize>,
/// Whether the URL was written to the local quarantine manifest.
pub quarantined: bool,
}
impl OfficialResourcePullProgress {
@@ -90,6 +107,11 @@ impl OfficialResourcePullProgress {
status: None,
bytes: None,
transferred_bytes: None,
failure_kind: None,
failure_http_status: None,
failure_retryable: None,
failure_attempts: None,
quarantined: false,
}
}
@@ -109,6 +131,28 @@ impl OfficialResourcePullProgress {
status: Some(status),
bytes: Some(bytes),
transferred_bytes: Some(transferred_bytes),
failure_kind: None,
failure_http_status: None,
failure_retryable: None,
failure_attempts: None,
quarantined: false,
}
}
fn failed(index: usize, total: usize, url: String, error: &PullOneError) -> Self {
Self {
kind: OfficialResourcePullProgressKind::Failed,
index,
total,
url,
status: None,
bytes: None,
transferred_bytes: None,
failure_kind: error.failure_kind().map(ToOwned::to_owned),
failure_http_status: error.http_status(),
failure_retryable: error.retryable(),
failure_attempts: error.attempts(),
quarantined: true,
}
}
}
@@ -434,6 +478,11 @@ impl OfficialResourcePullService {
self.output_root.join(DOWNLOAD_MANIFEST_FILE)
}
/// Returns the local quarantine path used to diagnose failed downloads.
pub fn download_quarantine_path(&self) -> PathBuf {
self.output_root.join(DOWNLOAD_QUARANTINE_FILE)
}
/// Executes the plan and downloads every official URL to disk.
pub fn pull(
&self,
@@ -504,9 +553,27 @@ impl OfficialResourcePullService {
self.validated_existing_file(&url, &destination, &manifest)?
};
let result = if let Some(result) = result {
self.clear_quarantine_entry(&url)?;
result
} else {
let result = self.pull_one(&url, &destination)?;
let result = match self.pull_one(&url, &destination) {
Ok(result) => result,
Err(error) => {
self.record_quarantine_entry(&url, &destination, &error)?;
progress(OfficialResourcePullProgress::failed(
index,
total,
url.clone(),
&error,
));
return Err(format!(
"官方资源下载中断：URL 已进入 quarantine 并跳过本轮，不会发布不完整资源；url={url} quarantine={}；{}",
self.download_quarantine_path().display(),
error.message
));
}
};
self.clear_quarantine_entry(&url)?;
self.record_download_manifest_entry(&mut manifest, &url, &destination)?;
self.write_download_manifest(&manifest)?;
result
@@ -642,49 +709,32 @@ impl OfficialResourcePullService {
return Err(format!("拒绝拉取非官方 URL：{url}"));
}
let attempts = self.retry_attempts.max(1);
let mut last_error = None;
for attempt in 1..=attempts {
match self.fetch_bytes_once(url) {
Ok(bytes) => return Ok(bytes),
Err(error) => {
last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}"));
}
}
}
Err(last_error.unwrap_or_else(|| format!("curl 未执行就已失败：{url}")))
}
fn fetch_bytes_once(&self, url: &str) -> Result<Vec<u8>, String> {
let output = Command::new(&self.curl_command)
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--url")
.arg(url)
.output()
.map_err(|error| {
format!(
"启动 curl 命令失败 {}：{error}",
self.curl_command.display()
)
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!("curl 拉取失败 {url}：{}", stderr.trim()));
}
let output =
run_curl_with_retry(&self.curl_command, url, None, self.retry_attempts, || {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--url")
.arg(url);
command
})
.map_err(|error| format!("curl 拉取失败 {url}：{error}"))?;
Ok(output.stdout)
}
fn pull_one(&self, url: &str, destination: &Path) -> Result<PullOneResult, String> {
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?;
fn pull_one(&self, url: &str, destination: &Path) -> Result<PullOneResult, PullOneError> {
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
let partial = partial_path_for(destination);
ensure_safe_file_target(&self.output_root, &partial, "临时下载文件")?;
let partial_len_before = file_len_if_exists(&partial)?.unwrap_or_default();
ensure_safe_file_target(&self.output_root, &partial, "临时下载文件")
.map_err(PullOneError::plain)?;
let partial_len_before = file_len_if_exists(&partial)
.map_err(PullOneError::plain)?
.unwrap_or_default();
let resumed = partial_len_before > 0;
let mut status = if resumed {
OfficialResourcePullStatus::Resumed
@@ -697,43 +747,60 @@ impl OfficialResourcePullService {
let _ = fs::remove_file(&partial);
self.download_one(url, &partial, false)
.map_err(|retry_error| {
format!("续传失败 {url}：{error}；清理后重新下载也失败：{retry_error}")
PullOneError::from_retry(
format!("续传失败 {url}：{error}；清理后重新下载也失败：{retry_error}"),
retry_error,
)
})?;
status = OfficialResourcePullStatus::Downloaded;
} else if let Err(error) = self.validate_zip_if_needed(url, &partial) {
let _ = fs::remove_file(&partial);
self.download_one(url, &partial, false)
.map_err(|retry_error| {
format!("续传后的 ZIP 结构校验失败 {url}：{error}；清理后重新下载也失败：{retry_error}")
PullOneError::from_retry(
format!("续传后的 ZIP 结构校验失败 {url}：{error}；清理后重新下载也失败：{retry_error}"),
retry_error,
)
})?;
status = OfficialResourcePullStatus::Downloaded;
}
} else {
self.download_one(url, &partial, false)?;
self.download_one(url, &partial, false).map_err(|error| {
PullOneError::from_retry(format!("下载失败 {url}：{error}"), error)
})?;
}
if let Err(error) = self.validate_zip_if_needed(url, &partial) {
let _ = fs::remove_file(&partial);
return Err(error);
return Err(PullOneError::plain(error));
}
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?;
if file_len_if_exists(destination)?.is_some() {
fs::remove_file(destination).map_err(|error| {
format!("替换已有目标文件失败 {}：{error}", destination.display())
})?;
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
if file_len_if_exists(destination)
.map_err(PullOneError::plain)?
.is_some()
{
fs::remove_file(destination)
.map_err(|error| format!("替换已有目标文件失败 {}：{error}", destination.display()))
.map_err(PullOneError::plain)?;
}
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")?;
ensure_safe_file_target(&self.output_root, destination, "下载目标文件")
.map_err(PullOneError::plain)?;
fs::rename(&partial, destination).map_err(|error| {
format!(
"移动临时下载文件失败 {} -> {}：{error}",
partial.display(),
destination.display()
)
})?;
fs::rename(&partial, destination)
.map_err(|error| {
format!(
"移动临时下载文件失败 {} -> {}：{error}",
partial.display(),
destination.display()
)
})
.map_err(PullOneError::plain)?;
let bytes = file_len_if_exists(destination)?
.ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))?;
let bytes = file_len_if_exists(destination)
.map_err(PullOneError::plain)?
.ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))
.map_err(PullOneError::plain)?;
Ok(PullOneResult {
bytes,
@@ -746,53 +813,34 @@ impl OfficialResourcePullService {
})
}
fn download_one(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> {
let attempts = self.retry_attempts.max(1);
let mut last_error = None;
for attempt in 1..=attempts {
match self.download_one_once(url, destination, resume) {
Ok(()) => return Ok(()),
Err(error) => {
last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}"));
fn download_one(
&self,
url: &str,
destination: &Path,
resume: bool,
) -> Result<(), CurlRetryError> {
run_curl_with_retry(
&self.curl_command,
url,
Some(destination),
self.retry_attempts,
|| {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--output")
.arg(destination);
if resume {
command.arg("--continue-at").arg("-");
}
}
}
Err(last_error
.unwrap_or_else(|| format!("curl 未执行就已失败：{url} -> {}", destination.display())))
}
fn download_one_once(&self, url: &str, destination: &Path, resume: bool) -> Result<(), String> {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--output")
.arg(destination);
if resume {
command.arg("--continue-at").arg("-");
}
command.arg("--url").arg(url);
let output = command.output().map_err(|error| {
format!(
"启动 curl 命令失败 {}：{error}",
self.curl_command.display()
)
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!(
"curl 下载失败 {url} -> {}：{}",
destination.display(),
stderr.trim()
));
}
Ok(())
command.arg("--url").arg(url);
command
},
)
.map(|_| ())
}
fn verify_ready_official_hashes(
@@ -935,6 +983,82 @@ impl OfficialResourcePullService {
Ok(())
}
fn read_download_quarantine(&self) -> Result<OfficialDownloadQuarantineManifest, String> {
self.ensure_output_root_safe()?;
let path = self.download_quarantine_path();
ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?;
let Some(bytes) = read_file_no_symlink(&path, "下载 quarantine")? else {
return Ok(OfficialDownloadQuarantineManifest::default());
};
let manifest: OfficialDownloadQuarantineManifest = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析下载 quarantine 失败 {}：{error}", path.display()))?;
if manifest.version != DOWNLOAD_QUARANTINE_VERSION {
return Err(format!(
"不支持的下载 quarantine 版本 {}，文件 {}",
manifest.version,
path.display()
));
}
Ok(manifest)
}
fn write_download_quarantine(
&self,
manifest: &OfficialDownloadQuarantineManifest,
) -> Result<(), String> {
self.ensure_output_root_ready()?;
let path = self.download_quarantine_path();
if let Some(parent) = path.parent() {
ensure_safe_directory_path(parent, "下载 quarantine 目录")?;
fs::create_dir_all(parent).map_err(|error| {
format!("创建下载 quarantine 目录失败 {}：{error}", parent.display())
})?;
ensure_safe_directory_path(parent, "下载 quarantine 目录")?;
}
ensure_safe_file_target(&self.output_root, &path, "下载 quarantine")?;
let bytes = serde_json::to_vec_pretty(manifest)
.map_err(|error| format!("序列化下载 quarantine 失败 {}：{error}", path.display()))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "下载 quarantine")?;
Ok(())
}
fn record_quarantine_entry(
&self,
url: &str,
destination: &Path,
error: &PullOneError,
) -> Result<(), String> {
let mut manifest = self.read_download_quarantine()?;
manifest.entries.insert(
url.to_string(),
OfficialDownloadQuarantineEntry {
url: url.to_string(),
destination: self.relative_destination(destination)?,
failed_at_unix_seconds: unix_seconds_now(),
attempts: error.attempts().unwrap_or(0),
failure_kind: error.failure_kind().map(ToOwned::to_owned),
http_status: error.http_status(),
retryable: error.retryable(),
skipped_this_run: true,
last_error: error.message.clone(),
},
);
self.write_download_quarantine(&manifest)
}
fn clear_quarantine_entry(&self, url: &str) -> Result<(), String> {
let mut manifest = self.read_download_quarantine()?;
if manifest.entries.remove(url).is_none() {
return Ok(());
}
self.write_download_quarantine(&manifest)
}
fn validated_existing_file(
&self,
url: &str,
@@ -1341,10 +1465,44 @@ struct OfficialDownloadManifestEntry {
blake3: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct OfficialDownloadQuarantineManifest {
#[serde(default = "default_download_quarantine_version")]
version: u32,
#[serde(default)]
entries: BTreeMap<String, OfficialDownloadQuarantineEntry>,
}
impl Default for OfficialDownloadQuarantineManifest {
fn default() -> Self {
Self {
version: DOWNLOAD_QUARANTINE_VERSION,
entries: BTreeMap::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct OfficialDownloadQuarantineEntry {
url: String,
destination: String,
failed_at_unix_seconds: u64,
attempts: usize,
failure_kind: Option<String>,
http_status: Option<u16>,
retryable: Option<bool>,
skipped_this_run: bool,
last_error: String,
}
fn default_download_manifest_version() -> u32 {
DOWNLOAD_MANIFEST_VERSION
}
fn default_download_quarantine_version() -> u32 {
DOWNLOAD_QUARANTINE_VERSION
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct PullOneResult {
bytes: u64,
@@ -1352,6 +1510,58 @@ struct PullOneResult {
status: OfficialResourcePullStatus,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct PullOneError {
message: String,
retry_error: Option<CurlRetryError>,
}
impl PullOneError {
fn plain(message: String) -> Self {
Self {
message,
retry_error: None,
}
}
fn from_retry(message: String, retry_error: CurlRetryError) -> Self {
Self {
message,
retry_error: Some(retry_error),
}
}
fn failure_kind(&self) -> Option<&'static str> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_kind_label)
}
fn http_status(&self) -> Option<u16> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_http_status)
}
fn retryable(&self) -> Option<bool> {
self.retry_error
.as_ref()
.and_then(CurlRetryError::final_retryable)
}
fn attempts(&self) -> Option<usize> {
self.retry_error.as_ref().map(CurlRetryError::attempt_count)
}
}
impl std::fmt::Display for PullOneError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for PullOneError {}
fn file_len_if_exists(path: &Path) -> Result<Option<u64>, String> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => {
@@ -1370,6 +1580,13 @@ fn partial_path_for(destination: &Path) -> PathBuf {
PathBuf::from(partial)
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn blake3_file_hex(path: &Path) -> Result<String, String> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => {
@@ -1769,6 +1986,35 @@ fi
"#
}
fn fake_http_error_curl_script(status: u16) -> String {
format!(
r#"#!/bin/sh
set -eu
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
state="$0.state"
count=0
if [ -f "$state" ]; then
count="$(cat "$state")"
fi
count=$((count + 1))
printf '%s' "$count" > "$state"
printf 'curl: (22) The requested URL returned error: {status}\n' >&2
exit 22
"#
)
}
fn write_shell_script(path: &Path, script: &str) {
fs::write(path, script).unwrap();
#[cfg(unix)]
@@ -1800,6 +2046,32 @@ fi
write_shell_script(path, fake_bad_zip_curl_script());
}
fn write_fake_http_error_curl(path: &Path, status: u16) {
write_shell_script(path, &fake_http_error_curl_script(status));
}
fn one_url_plan(url: &str) -> OfficialResourcePullPlan {
OfficialResourcePullPlan {
discovery: YostarJpResourceDiscoveryPlan {
connection_group_name: "Prod-Audit".to_string(),
app_version: "1.70.0".to_string(),
bundle_version: None,
addressables_root: "https://prod-clientpatch.bluearchiveyostar.com/r93_token"
.to_string(),
endpoints: vec![YostarJpResourceEndpoint {
kind: YostarJpResourceEndpointKind::TableCatalog,
platform: None,
url: url.to_string(),
}],
},
inventory: YostarJpPlatformDownloadInventory::from_shared_inventory(
YostarJpDownloadInventory::from_catalog_bytes(b"", b"", b""),
&[],
),
platforms: Vec::new(),
}
}
fn one_file_zip(name: &[u8], data: &[u8]) -> Vec<u8> {
let mut bytes = Vec::new();
bytes.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
@@ -1965,7 +2237,7 @@ fi
symlink(&escape_file, partial_path_for(&destination)).unwrap();
let error = service.pull_one(url, &destination).unwrap_err();
assert!(error.contains("symlink"));
assert!(error.to_string().contains("symlink"));
assert_eq!(fs::read(&escape_file).unwrap(), b"outside");
}
@@ -2470,6 +2742,82 @@ fi
);
}
#[test]
fn does_not_retry_terminal_http_404_and_records_quarantine() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_http_error_curl(&curl_path, 404);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(3);
let url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Missing.bytes";
let plan = one_url_plan(url);
let mut events = Vec::new();
let error = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap_err();
assert!(error.contains("quarantine"));
assert!(error.contains("http_not_found"));
assert!(error.contains("retryable=false"));
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"1"
);
let failed = events
.iter()
.find(|event| event.kind == OfficialResourcePullProgressKind::Failed)
.unwrap();
assert_eq!(failed.failure_kind.as_deref(), Some("http_not_found"));
assert_eq!(failed.failure_http_status, Some(404));
assert_eq!(failed.failure_retryable, Some(false));
assert_eq!(failed.failure_attempts, Some(1));
assert!(failed.quarantined);
let quarantine = service.read_download_quarantine().unwrap();
let entry = quarantine.entries.get(url).unwrap();
assert_eq!(entry.http_status, Some(404));
assert_eq!(entry.failure_kind.as_deref(), Some("http_not_found"));
assert_eq!(entry.retryable, Some(false));
assert_eq!(entry.attempts, 1);
assert!(entry.skipped_this_run);
assert!(service.read_download_manifest().unwrap().entries.is_empty());
}
#[test]
fn retries_http_5xx_before_quarantine() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_http_error_curl(&curl_path, 503);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_retry_attempts(3);
let url =
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/Busy.bytes";
let plan = one_url_plan(url);
let error = service.pull(&plan).unwrap_err();
assert!(error.contains("quarantine"));
assert!(error.contains("http_server_error"));
assert!(error.contains("retryable=true"));
assert_eq!(
fs::read_to_string(curl_path.with_extension("state")).unwrap(),
"3"
);
let quarantine = service.read_download_quarantine().unwrap();
let entry = quarantine.entries.get(url).unwrap();
assert_eq!(entry.http_status, Some(503));
assert_eq!(entry.failure_kind.as_deref(), Some("http_server_error"));
assert_eq!(entry.retryable, Some(true));
assert_eq!(entry.attempts, 3);
assert!(entry.last_error.contains("重试次数已耗尽"));
}
#[test]
fn custom_platform_selection_is_supported() {
let out_dir = TempDir::new().unwrap();
infrastructure/src/official_game_main_config.rs
+105 -39
View File
@@ -4,6 +4,7 @@
//! artifacts into a temporary directory, extracts only the data needed to read
//! `GameMainConfig`, and does not install or execute the official launcher.
use crate::curl_transfer::run_curl_with_retry;
use crate::official_launcher::launcher_package_url;
use crate::official_launcher::OfficialLauncherBootstrapService;
use crate::official_launcher::{
@@ -115,44 +116,27 @@ impl OfficialGameMainConfigBootstrapService {
}
fn download_file(&self, url: &str, destination: &Path) -> Result<(), String> {
let attempts = DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS;
let mut last_error = None;
for attempt in 1..=attempts {
match self.download_file_once(url, destination) {
Ok(()) => return Ok(()),
Err(error) => {
last_error = Some(format!("第 {attempt}/{attempts} 次尝试失败：{error}"));
}
}
}
Err(last_error.unwrap_or_else(|| format!("curl 未执行就已失败：{url}")))
}
fn download_file_once(&self, url: &str, destination: &Path) -> Result<(), String> {
let output = Command::new(&self.curl_command)
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--output")
.arg(destination)
.arg("--url")
.arg(url)
.output()
.map_err(|error| {
format!(
"启动 curl 命令失败 {}：{error}",
self.curl_command.display()
)
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!("curl 拉取失败 {url}：{}", stderr.trim()));
}
Ok(())
run_curl_with_retry(
&self.curl_command,
url,
Some(destination),
DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS,
|| {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
.arg("--location")
.arg("--silent")
.arg("--show-error")
.arg("--output")
.arg(destination)
.arg("--url")
.arg(url);
command
},
)
.map(|_| ())
.map_err(|error| format!("curl 拉取失败 {url}：{error}"))
}
fn download_file_with_fallback(
@@ -168,7 +152,7 @@ impl OfficialGameMainConfigBootstrapService {
let backup_url = launcher_package_url(backup_cdn_root, relative_path)?;
self.download_file(&backup_url, destination).map_err(|backup_error| {
format!(
"下载官方游戏资源失败：主地址失败（{primary_error}），备用地址也失败（{backup_error}）"
"下载官方游戏资源失败：主 CDN 失败后已切换官方备用 CDN；主地址失败（{primary_error}），备用地址也失败（{backup_error}）"
)
})
}
@@ -391,3 +375,85 @@ fn find_resources_assets(root: &Path) -> Option<PathBuf> {
}
None
}
#[cfg(test)]
mod tests {
use super::*;
fn write_shell_script(path: &Path, script: &str) {
fs::write(path, script).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(path).unwrap().permissions();
permissions.set_mode(0o755);
fs::set_permissions(path, permissions).unwrap();
}
}
fn fake_primary_404_backup_ok_curl_script() -> &'static str {
r#"#!/bin/sh
set -eu
out=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
out="$2"
shift 2
;;
--url)
url="$2"
shift 2
;;
*)
shift
;;
esac
done
printf '%s\n' "$url" >> "$0.urls"
case "$url" in
https://launcher-pkg-ba-jp.yo-star.com/*)
printf 'curl: (22) The requested URL returned error: 404\n' >&2
exit 22
;;
https://launcher-pkg-ba-jp-bk.yo-star.com/*)
mkdir -p "$(dirname "$out")"
printf '%s' 'backup-ok' > "$out"
;;
*)
printf 'unexpected url: %s\n' "$url" >&2
exit 22
;;
esac
"#
}
#[test]
fn downloads_launcher_package_from_backup_after_primary_terminal_failure() {
let temp = TempDir::new().unwrap();
let curl_path = temp.path().join("curl");
write_shell_script(&curl_path, fake_primary_404_backup_ok_curl_script());
let service =
OfficialGameMainConfigBootstrapService::with_commands("1.17.0", &curl_path, "unzip");
let destination = temp.path().join("resources.assets");
let primary_url = "https://launcher-pkg-ba-jp.yo-star.com/prod/manifest/resources.assets";
service
.download_file_with_fallback(
primary_url,
"https://launcher-pkg-ba-jp-bk.yo-star.com",
"prod/manifest/resources.assets",
&destination,
)
.unwrap();
assert_eq!(fs::read_to_string(&destination).unwrap(), "backup-ok");
let urls = fs::read_to_string(curl_path.with_extension("urls")).unwrap();
assert!(
urls.contains("https://launcher-pkg-ba-jp.yo-star.com/prod/manifest/resources.assets")
);
assert!(urls
.contains("https://launcher-pkg-ba-jp-bk.yo-star.com/prod/manifest/resources.assets"));
}
}
infrastructure/src/official_launcher.rs
+19 -41
View File
@@ -5,9 +5,11 @@
//! discover the latest Windows client package and manifest, while resource
//! updates still come from the game client's server-info flow.
use crate::curl_transfer::run_curl_with_retry as run_curl_command_with_retry;
use bat_adapters::official::launcher::{YostarJpLauncherManifestFile, YOSTAR_JP_GAME_TAG};
use md5::{Digest, Md5};
use serde::Deserialize;
use std::path::Path;
use std::process::{Command, Output};
use std::time::{SystemTime, UNIX_EPOCH};
@@ -165,8 +167,8 @@ impl OfficialLauncherBootstrapService {
return Err(format!("拒绝拉取非官方启动器 manifest URL：{manifest_url}"));
}
let output = self.run_curl_with_retry(
|| {
let output = self
.run_curl_with_retry(manifest_url, || {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
@@ -176,15 +178,8 @@ impl OfficialLauncherBootstrapService {
.arg("--url")
.arg(manifest_url);
command
},
|output| {
let stderr = String::from_utf8_lossy(&output.stderr);
format!(
"curl 拉取启动器 manifest 失败 {manifest_url}：{}",
stderr.trim()
)
},
)?;
})
.map_err(|error| format!("curl 拉取启动器 manifest 失败 {manifest_url}：{error}"))?;
let manifest: YostarJpLauncherRemoteManifest = serde_json::from_slice(&output.stdout)
.map_err(|error| format!("解析官方启动器 manifest 失败：{error}"))?;
@@ -222,8 +217,8 @@ impl OfficialLauncherBootstrapService {
let url = launcher_api_url(path)?;
let authorization = launcher_authorization_header(&self.launcher_version, "", None)?;
let output = self.run_curl_with_retry(
|| {
let output = self
.run_curl_with_retry(&url, || {
let mut command = Command::new(&self.curl_command);
command
.arg("--fail")
@@ -237,12 +232,8 @@ impl OfficialLauncherBootstrapService {
.arg("--url")
.arg(&url);
command
},
|output| {
let stderr = String::from_utf8_lossy(&output.stderr);
format!("curl 拉取失败 {url}：{}", stderr.trim())
},
)?;
})
.map_err(|error| format!("curl 拉取失败 {url}：{error}"))?;
let envelope: LauncherEnvelope = serde_json::from_slice(&output.stdout)
.map_err(|error| format!("解析官方启动器 API 响应失败：{error}"))?;
@@ -264,30 +255,17 @@ impl OfficialLauncherBootstrapService {
fn run_curl_with_retry(
&self,
url: &str,
mut build_command: impl FnMut() -> Command,
failure_message: impl Fn(&Output) -> String,
) -> Result<Output, String> {
let attempts = self.retry_attempts.max(1);
let mut last_error = None;
for attempt in 1..=attempts {
match build_command().output() {
Ok(output) if output.status.success() => return Ok(output),
Ok(output) => {
last_error = Some(format!(
"第 {attempt}/{attempts} 次尝试失败：{}",
failure_message(&output)
));
}
Err(error) => {
last_error = Some(format!(
"第 {attempt}/{attempts} 次尝试失败：启动 curl 命令失败 {}：{error}",
self.curl_command
));
}
}
}
Err(last_error.unwrap_or_else(|| format!("curl 命令未执行：{}", self.curl_command)))
run_curl_command_with_retry(
Path::new(&self.curl_command),
url,
None,
self.retry_attempts,
&mut build_command,
)
.map_err(|error| error.to_string())
}
}
infrastructure/src/official_update.rs
+53 -1
View File
@@ -474,6 +474,16 @@ pub struct OfficialUpdateProgress {
pub download_bytes: Option<u64>,
/// Bytes transferred in this run for the URL when known.
pub download_transferred_bytes: Option<u64>,
/// Stable failure kind when a URL failed.
pub download_failure_kind: Option<String>,
/// HTTP status parsed from curl stderr when a URL failed.
pub download_failure_http_status: Option<u16>,
/// Whether the final failure was retryable.
pub download_failure_retryable: Option<bool>,
/// Number of transfer attempts executed before failure.
pub download_failure_attempts: Option<usize>,
/// Whether the URL was recorded in the quarantine manifest.
pub download_quarantined: Option<bool>,
}
impl OfficialUpdateProgress {
@@ -488,6 +498,11 @@ impl OfficialUpdateProgress {
download_status: None,
download_bytes: None,
download_transferred_bytes: None,
download_failure_kind: None,
download_failure_http_status: None,
download_failure_retryable: None,
download_failure_attempts: None,
download_quarantined: None,
}
}
@@ -495,9 +510,21 @@ impl OfficialUpdateProgress {
self.download_index = Some(event.index);
self.download_total = Some(event.total);
self.download_url = Some(event.url.clone());
self.download_status = event.status.map(|status| status.as_str().to_string());
self.download_status = event
.status
.map(|status| status.as_str().to_string())
.or_else(|| {
(event.kind == OfficialResourcePullProgressKind::Failed)
.then(|| "failed".to_string())
});
self.download_bytes = event.bytes;
self.download_transferred_bytes = event.transferred_bytes;
self.download_failure_kind = event.failure_kind.clone();
self.download_failure_http_status = event.failure_http_status;
self.download_failure_retryable = event.failure_retryable;
self.download_failure_attempts = event.failure_attempts;
self.download_quarantined =
(event.kind == OfficialResourcePullProgressKind::Failed).then_some(event.quarantined);
self
}
}
@@ -1391,6 +1418,31 @@ fn progress_from_pull_event(event: OfficialResourcePullProgress) -> OfficialUpda
)
.with_download_progress(&event)
}
OfficialResourcePullProgressKind::Failed => OfficialUpdateProgress::new(
"download",
format!(
"下载中断：总体 {}/{} ({:.1}%)；失败类型={} HTTP={} 可重试={} 尝试次数={} quarantine={}；本轮跳过该 URL 且不会发布不完整资源 URL={}",
event.index,
event.total,
download_progress_percent(event.index, event.total),
event.failure_kind.as_deref().unwrap_or("unknown"),
event
.failure_http_status
.map(|status| status.to_string())
.unwrap_or_else(|| "none".to_string()),
event
.failure_retryable
.map(|retryable| retryable.to_string())
.unwrap_or_else(|| "unknown".to_string()),
event
.failure_attempts
.map(|attempts| attempts.to_string())
.unwrap_or_else(|| "0".to_string()),
event.quarantined,
event.url
),
)
.with_download_progress(&event),
}
}