mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 07:24:55 +08:00
fix(sync): 补齐发布清理与校验进度
This commit is contained in:
@@ -11,14 +11,15 @@ use bat_infrastructure::{
|
|||||||
read_parse_cache_at, read_snapshot, read_textunit_index_at, read_version_state,
|
read_parse_cache_at, read_snapshot, read_textunit_index_at, read_version_state,
|
||||||
redact_proxy_url, resolve_curl_proxy, validate_output_root, validate_runtime_state_dir,
|
redact_proxy_url, resolve_curl_proxy, validate_output_root, validate_runtime_state_dir,
|
||||||
write_file_atomic, CurlProxyConfig, CurlProxyMode, OfficialEndpointMarkerRole,
|
write_file_atomic, CurlProxyConfig, CurlProxyMode, OfficialEndpointMarkerRole,
|
||||||
OfficialFailedVersionRecord, OfficialServerInfoSource, OfficialTextUnitQuery,
|
OfficialFailedVersionRecord, OfficialResourceHashVerification, OfficialResourceVerification,
|
||||||
OfficialUpdateConfig, OfficialUpdateProgress, OfficialUpdateReport, OfficialUpdateService,
|
OfficialServerInfoSource, OfficialTextUnitQuery, OfficialUpdateConfig, OfficialUpdateProgress,
|
||||||
OfficialUpdateSnapshot, OfficialUpdateStatus, OfficialVerificationSummary,
|
OfficialUpdateReport, OfficialUpdateService, OfficialUpdateSnapshot, OfficialUpdateStatus,
|
||||||
OfficialVersionRecord, OfficialVersionState, PatchApplyKind, PatchApplyParams,
|
OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState, PatchApplyKind,
|
||||||
PatchApplyReport, SqliteResourceRepository, UnityFsFieldPatchParams, UnityFsPatchReport,
|
PatchApplyParams, PatchApplyReport, SqliteResourceRepository, UnityFsFieldPatchParams,
|
||||||
UnityFsStringFieldPatchParams, UnityFsTextAssetPatchParams, LOCALIZED_CURRENT_LINK,
|
UnityFsPatchReport, UnityFsStringFieldPatchParams, UnityFsTextAssetPatchParams,
|
||||||
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_VERSIONS_DIR, LOCALIZED_VERSION_STATE_FILE,
|
LOCALIZED_CURRENT_LINK, LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_VERSIONS_DIR,
|
||||||
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE, PRIVATE_FILE_MODE,
|
LOCALIZED_VERSION_STATE_FILE, OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE,
|
||||||
|
PRIVATE_FILE_MODE,
|
||||||
};
|
};
|
||||||
use serde::de::DeserializeOwned;
|
use serde::de::DeserializeOwned;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -48,6 +49,7 @@ const DAEMON_LOG_FILE: &str = "bat-daemon.log";
|
|||||||
const DAEMON_STRUCTURED_LOG_FILE: &str = "bat-events.jsonl";
|
const DAEMON_STRUCTURED_LOG_FILE: &str = "bat-events.jsonl";
|
||||||
const DAEMON_SOCKET_FILE: &str = "bat.sock";
|
const DAEMON_SOCKET_FILE: &str = "bat.sock";
|
||||||
const DAEMON_CONTROL_LOCK_FILE: &str = "bat-control.lock";
|
const DAEMON_CONTROL_LOCK_FILE: &str = "bat-control.lock";
|
||||||
|
const SHUTDOWN_SIGNAL_POLL_MILLISECONDS: u64 = 100;
|
||||||
/// 后台进程保存代理凭据的专用文件,仅供 restart/reload 复用,永不序列化到状态输出。
|
/// 后台进程保存代理凭据的专用文件,仅供 restart/reload 复用,永不序列化到状态输出。
|
||||||
const DAEMON_PROXY_SECRET_FILE: &str = "bat-proxy.secret";
|
const DAEMON_PROXY_SECRET_FILE: &str = "bat-proxy.secret";
|
||||||
/// 代理凭据下传子进程使用的环境变量;避免凭据出现在子进程 argv(/proc/<pid>/cmdline)。
|
/// 代理凭据下传子进程使用的环境变量;避免凭据出现在子进程 argv(/proc/<pid>/cmdline)。
|
||||||
@@ -354,21 +356,22 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
|
|||||||
if options.daemon_child {
|
if options.daemon_child {
|
||||||
validate_runtime_state_dir(&options.state_dir).map_err(anyhow::Error::msg)?;
|
validate_runtime_state_dir(&options.state_dir).map_err(anyhow::Error::msg)?;
|
||||||
}
|
}
|
||||||
|
install_shutdown_signal_handlers()?;
|
||||||
|
clear_shutdown_signal_request();
|
||||||
let service = OfficialUpdateService::new();
|
let service = OfficialUpdateService::new();
|
||||||
let mut logger = ProgressLogger::new(options.progress);
|
let mut logger = ProgressLogger::new(options.progress);
|
||||||
let daemon_state_dir = options.state_dir.clone();
|
let daemon_state_dir = options.state_dir.clone();
|
||||||
if options.daemon_child {
|
if options.daemon_child {
|
||||||
logger.attach_structured_log(daemon_structured_log_path(&daemon_state_dir));
|
logger.attach_structured_log(daemon_structured_log_path(&daemon_state_dir));
|
||||||
}
|
}
|
||||||
let daemon_control = if options.daemon_child {
|
// 前台 watch 也使用控制状态,使信号停止和 daemon/RPC 停止共享同一条退出路径。
|
||||||
Some(new_daemon_control())
|
let daemon_control = Some(new_daemon_control());
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
// 进程内同步锁:watch 循环与任务 worker 在跑同步前都获取它,互相等待而非撞文件锁失败。
|
// 进程内同步锁:watch 循环与任务 worker 在跑同步前都获取它,互相等待而非撞文件锁失败。
|
||||||
let sync_lock = Arc::new(Mutex::new(()));
|
let sync_lock = Arc::new(Mutex::new(()));
|
||||||
let (_task_worker, _task_context, _rpc_server) = if let Some(control) = daemon_control.as_ref()
|
let (_task_worker, _task_context, _rpc_server) = if options.daemon_child {
|
||||||
{
|
let control = daemon_control
|
||||||
|
.as_ref()
|
||||||
|
.expect("daemon control must exist for daemon child");
|
||||||
// 任务历史持久化在 state dir(此前已通过 validate_runtime_state_dir 校验)。
|
// 任务历史持久化在 state dir(此前已通过 validate_runtime_state_dir 校验)。
|
||||||
let (registry, restore_summary) = TaskRegistry::with_persistence(&daemon_state_dir);
|
let (registry, restore_summary) = TaskRegistry::with_persistence(&daemon_state_dir);
|
||||||
logger.log_text("daemon", format!("任务历史:{restore_summary}"));
|
logger.log_text("daemon", format!("任务历史:{restore_summary}"));
|
||||||
@@ -403,7 +406,7 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
loop {
|
loop {
|
||||||
if daemon_control_stop_requested(daemon_control.as_ref()) {
|
if watch_stop_requested(daemon_control.as_ref()) {
|
||||||
logger.log_text("daemon", "收到停止请求,watch 循环准备退出");
|
logger.log_text("daemon", "收到停止请求,watch 循环准备退出");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -472,7 +475,7 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
|
|||||||
);
|
);
|
||||||
logger.log(event);
|
logger.log(event);
|
||||||
},
|
},
|
||||||
|| daemon_control_stop_requested(daemon_control.as_ref()),
|
|| watch_stop_requested(daemon_control.as_ref()),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
match run_result {
|
match run_result {
|
||||||
@@ -566,7 +569,7 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
|
|||||||
next_retry_seconds: Some(sleep_for.as_secs()),
|
next_retry_seconds: Some(sleep_for.as_secs()),
|
||||||
};
|
};
|
||||||
eprintln!("{}", serde_json::to_string_pretty(&payload)?);
|
eprintln!("{}", serde_json::to_string_pretty(&payload)?);
|
||||||
if daemon_control_stop_requested(daemon_control.as_ref()) {
|
if watch_stop_requested(daemon_control.as_ref()) {
|
||||||
logger.log_text("daemon", "停止请求已中断本轮同步,watch 循环准备退出");
|
logger.log_text("daemon", "停止请求已中断本轮同步,watch 循环准备退出");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -576,7 +579,7 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
|
|||||||
match wait_for_daemon_wake(daemon_control.as_ref(), sleep_for) {
|
match wait_for_daemon_wake(daemon_control.as_ref(), sleep_for) {
|
||||||
DaemonWake::Timeout => {}
|
DaemonWake::Timeout => {}
|
||||||
DaemonWake::Stop => {
|
DaemonWake::Stop => {
|
||||||
logger.log_text("daemon", "收到停止请求,watch 循环准备退出");
|
logger.log_text("daemon", "收到停止请求或进程信号,watch 循环准备退出");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
DaemonWake::Refresh { force } => {
|
DaemonWake::Refresh { force } => {
|
||||||
@@ -672,6 +675,10 @@ struct DaemonDownloadProgress {
|
|||||||
failure_attempts: Option<usize>,
|
failure_attempts: Option<usize>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
quarantined: Option<bool>,
|
quarantined: Option<bool>,
|
||||||
|
#[serde(default)]
|
||||||
|
verification: Option<OfficialResourceVerification>,
|
||||||
|
#[serde(default)]
|
||||||
|
official_hash: Option<OfficialResourceHashVerification>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -1737,10 +1744,68 @@ fn lexically_normalize_path(path: &Path) -> PathBuf {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
static SHUTDOWN_SIGNAL_REQUESTED: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
extern "C" fn handle_shutdown_signal(_signal: libc::c_int) {
|
||||||
|
// Signal handlers may only touch lock-free process state; the watch loop
|
||||||
|
// performs logging, cancellation, and cleanup after observing this flag.
|
||||||
|
SHUTDOWN_SIGNAL_REQUESTED.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn install_shutdown_signal_handlers() -> anyhow::Result<()> {
|
||||||
|
unsafe {
|
||||||
|
if libc::signal(
|
||||||
|
libc::SIGINT,
|
||||||
|
handle_shutdown_signal as *const () as libc::sighandler_t,
|
||||||
|
) == libc::SIG_ERR
|
||||||
|
{
|
||||||
|
return Err(anyhow::anyhow!("注册 SIGINT 优雅退出处理器失败"));
|
||||||
|
}
|
||||||
|
if libc::signal(
|
||||||
|
libc::SIGTERM,
|
||||||
|
handle_shutdown_signal as *const () as libc::sighandler_t,
|
||||||
|
) == libc::SIG_ERR
|
||||||
|
{
|
||||||
|
return Err(anyhow::anyhow!("注册 SIGTERM 优雅退出处理器失败"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn install_shutdown_signal_handlers() -> anyhow::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn clear_shutdown_signal_request() {
|
||||||
|
SHUTDOWN_SIGNAL_REQUESTED.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn clear_shutdown_signal_request() {}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn shutdown_signal_requested() -> bool {
|
||||||
|
SHUTDOWN_SIGNAL_REQUESTED.load(Ordering::SeqCst)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn shutdown_signal_requested() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
fn new_daemon_control() -> DaemonControl {
|
fn new_daemon_control() -> DaemonControl {
|
||||||
Arc::new((Mutex::new(DaemonControlState::default()), Condvar::new()))
|
Arc::new((Mutex::new(DaemonControlState::default()), Condvar::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn watch_stop_requested(control: Option<&DaemonControl>) -> bool {
|
||||||
|
shutdown_signal_requested() || daemon_control_stop_requested(control)
|
||||||
|
}
|
||||||
|
|
||||||
fn daemon_control_stop_requested(control: Option<&DaemonControl>) -> bool {
|
fn daemon_control_stop_requested(control: Option<&DaemonControl>) -> bool {
|
||||||
let Some(control) = control else {
|
let Some(control) = control else {
|
||||||
return false;
|
return false;
|
||||||
@@ -1781,21 +1846,42 @@ fn daemon_control_request_reload(control: &DaemonControl) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn wait_for_daemon_wake(control: Option<&DaemonControl>, timeout: Duration) -> DaemonWake {
|
fn wait_for_daemon_wake(control: Option<&DaemonControl>, timeout: Duration) -> DaemonWake {
|
||||||
|
let started_at = Instant::now();
|
||||||
let Some(control) = control else {
|
let Some(control) = control else {
|
||||||
thread::sleep(timeout);
|
loop {
|
||||||
|
if shutdown_signal_requested() {
|
||||||
|
return DaemonWake::Stop;
|
||||||
|
}
|
||||||
|
let elapsed = started_at.elapsed();
|
||||||
|
if elapsed >= timeout {
|
||||||
return DaemonWake::Timeout;
|
return DaemonWake::Timeout;
|
||||||
|
}
|
||||||
|
let remaining = timeout.saturating_sub(elapsed);
|
||||||
|
thread::sleep(remaining.min(Duration::from_millis(SHUTDOWN_SIGNAL_POLL_MILLISECONDS)));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let (lock, cvar) = &**control;
|
let (lock, cvar) = &**control;
|
||||||
let Ok(mut state) = lock.lock() else {
|
let Ok(mut state) = lock.lock() else {
|
||||||
return DaemonWake::Stop;
|
return DaemonWake::Stop;
|
||||||
};
|
};
|
||||||
|
loop {
|
||||||
|
if shutdown_signal_requested() {
|
||||||
|
return DaemonWake::Stop;
|
||||||
|
}
|
||||||
if let Some(wake) = take_daemon_wake(&mut state) {
|
if let Some(wake) = take_daemon_wake(&mut state) {
|
||||||
return wake;
|
return wake;
|
||||||
}
|
}
|
||||||
let Ok((mut state, _timeout)) = cvar.wait_timeout(state, timeout) else {
|
let elapsed = started_at.elapsed();
|
||||||
|
if elapsed >= timeout {
|
||||||
|
return DaemonWake::Timeout;
|
||||||
|
}
|
||||||
|
let remaining = timeout.saturating_sub(elapsed);
|
||||||
|
let wait_for = remaining.min(Duration::from_millis(SHUTDOWN_SIGNAL_POLL_MILLISECONDS));
|
||||||
|
let Ok((next_state, _timeout)) = cvar.wait_timeout(state, wait_for) else {
|
||||||
return DaemonWake::Stop;
|
return DaemonWake::Stop;
|
||||||
};
|
};
|
||||||
take_daemon_wake(&mut state).unwrap_or(DaemonWake::Timeout)
|
state = next_state;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn take_daemon_wake(state: &mut DaemonControlState) -> Option<DaemonWake> {
|
fn take_daemon_wake(state: &mut DaemonControlState) -> Option<DaemonWake> {
|
||||||
@@ -4499,6 +4585,18 @@ fn print_list(label: &str, values: &[String], limit: usize) {
|
|||||||
|
|
||||||
fn format_daemon_download_progress(progress: &DaemonDownloadProgress) -> String {
|
fn format_daemon_download_progress(progress: &DaemonDownloadProgress) -> String {
|
||||||
let status = progress.status.as_deref().unwrap_or("running");
|
let status = progress.status.as_deref().unwrap_or("running");
|
||||||
|
if let Some(hash) = progress.official_hash.as_ref() {
|
||||||
|
return format!(
|
||||||
|
"{}/{} official_hash algorithm={} expected={} actual={} data={} hash={}",
|
||||||
|
progress.index,
|
||||||
|
progress.total,
|
||||||
|
hash.algorithm.as_str(),
|
||||||
|
hash.expected,
|
||||||
|
hash.actual,
|
||||||
|
hash.data_url,
|
||||||
|
hash.hash_url
|
||||||
|
);
|
||||||
|
}
|
||||||
if status == "failed" {
|
if status == "failed" {
|
||||||
return format!(
|
return format!(
|
||||||
"{}/{} failed kind={} http={} retryable={} attempts={} quarantined={} {}",
|
"{}/{} failed kind={} http={} retryable={} attempts={} quarantined={} {}",
|
||||||
@@ -4524,6 +4622,19 @@ fn format_daemon_download_progress(progress: &DaemonDownloadProgress) -> String
|
|||||||
progress.url
|
progress.url
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if let Some(verification) = progress.verification.as_ref() {
|
||||||
|
return format!(
|
||||||
|
"{}/{} {} bytes={} blake3={} zip_checked={} zip_verified={} {}",
|
||||||
|
progress.index,
|
||||||
|
progress.total,
|
||||||
|
status,
|
||||||
|
verification.actual_bytes,
|
||||||
|
verification.actual_blake3,
|
||||||
|
verification.zip_checked,
|
||||||
|
verification.zip_structure_verified,
|
||||||
|
progress.url
|
||||||
|
);
|
||||||
|
}
|
||||||
format!(
|
format!(
|
||||||
"{}/{} {} {}",
|
"{}/{} {} {}",
|
||||||
progress.index, progress.total, status, progress.url
|
progress.index, progress.total, status, progress.url
|
||||||
@@ -5631,6 +5742,8 @@ fn update_daemon_progress(state_dir: &Path, event: &OfficialUpdateProgress) -> a
|
|||||||
failure_retryable: event.download_failure_retryable,
|
failure_retryable: event.download_failure_retryable,
|
||||||
failure_attempts: event.download_failure_attempts,
|
failure_attempts: event.download_failure_attempts,
|
||||||
quarantined: event.download_quarantined,
|
quarantined: event.download_quarantined,
|
||||||
|
verification: event.download_verification.clone(),
|
||||||
|
official_hash: event.official_hash_verification.clone(),
|
||||||
}),
|
}),
|
||||||
_ if event.stage != "download" => None,
|
_ if event.stage != "download" => None,
|
||||||
_ => status.download_progress,
|
_ => status.download_progress,
|
||||||
@@ -6169,6 +6282,8 @@ impl RotatingStructuredLogger {
|
|||||||
"failure_retryable": event.download_failure_retryable,
|
"failure_retryable": event.download_failure_retryable,
|
||||||
"failure_attempts": event.download_failure_attempts,
|
"failure_attempts": event.download_failure_attempts,
|
||||||
"quarantined": event.download_quarantined,
|
"quarantined": event.download_quarantined,
|
||||||
|
"verification": event.download_verification,
|
||||||
|
"official_hash": event.official_hash_verification,
|
||||||
})),
|
})),
|
||||||
});
|
});
|
||||||
let mut line = serde_json::to_vec(&payload)?;
|
let mut line = serde_json::to_vec(&payload)?;
|
||||||
@@ -9020,6 +9135,31 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn shutdown_signals_wake_watch_without_mutating_daemon_control() {
|
||||||
|
clear_shutdown_signal_request();
|
||||||
|
let control = new_daemon_control();
|
||||||
|
|
||||||
|
handle_shutdown_signal(libc::SIGINT);
|
||||||
|
assert!(watch_stop_requested(Some(&control)));
|
||||||
|
assert_eq!(
|
||||||
|
wait_for_daemon_wake(Some(&control), Duration::from_secs(60)),
|
||||||
|
DaemonWake::Stop
|
||||||
|
);
|
||||||
|
|
||||||
|
clear_shutdown_signal_request();
|
||||||
|
handle_shutdown_signal(libc::SIGTERM);
|
||||||
|
assert!(watch_stop_requested(Some(&control)));
|
||||||
|
assert_eq!(
|
||||||
|
wait_for_daemon_wake(Some(&control), Duration::from_secs(60)),
|
||||||
|
DaemonWake::Stop
|
||||||
|
);
|
||||||
|
|
||||||
|
clear_shutdown_signal_request();
|
||||||
|
assert!(!watch_stop_requested(Some(&control)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn daemon_control_lock_rejects_active_owner() {
|
fn daemon_control_lock_rejects_active_owner() {
|
||||||
let temp = tempfile::TempDir::new().unwrap();
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
@@ -9151,6 +9291,21 @@ mod tests {
|
|||||||
event.download_index = Some(1);
|
event.download_index = Some(1);
|
||||||
event.download_total = Some(2);
|
event.download_total = Some(2);
|
||||||
event.download_url = Some("https://prod-clientpatch.bluearchiveyostar.com/a.zip".into());
|
event.download_url = Some("https://prod-clientpatch.bluearchiveyostar.com/a.zip".into());
|
||||||
|
event.download_verification = Some(OfficialResourceVerification {
|
||||||
|
expected_bytes: Some(123),
|
||||||
|
actual_bytes: 123,
|
||||||
|
expected_blake3: Some("expected-blake3".to_string()),
|
||||||
|
actual_blake3: "expected-blake3".to_string(),
|
||||||
|
zip_checked: true,
|
||||||
|
zip_structure_verified: true,
|
||||||
|
});
|
||||||
|
event.official_hash_verification = Some(OfficialResourceHashVerification {
|
||||||
|
data_url: "https://prod-clientpatch.bluearchiveyostar.com/a.bytes".to_string(),
|
||||||
|
hash_url: "https://prod-clientpatch.bluearchiveyostar.com/a.hash".to_string(),
|
||||||
|
algorithm: bat_infrastructure::OfficialResourceHashAlgorithm::XxHash32Decimal,
|
||||||
|
expected: "2044170421".to_string(),
|
||||||
|
actual: "2044170421".to_string(),
|
||||||
|
});
|
||||||
|
|
||||||
for _ in 0..8 {
|
for _ in 0..8 {
|
||||||
logger
|
logger
|
||||||
@@ -9170,6 +9325,16 @@ mod tests {
|
|||||||
assert_eq!(value["stage"], "download");
|
assert_eq!(value["stage"], "download");
|
||||||
assert_eq!(value["download"]["index"], 1);
|
assert_eq!(value["download"]["index"], 1);
|
||||||
assert_eq!(value["download"]["total"], 2);
|
assert_eq!(value["download"]["total"], 2);
|
||||||
|
assert_eq!(value["download"]["verification"]["actual_bytes"], 123);
|
||||||
|
assert_eq!(
|
||||||
|
value["download"]["verification"]["zip_structure_verified"],
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
value["download"]["official_hash"]["algorithm"],
|
||||||
|
"xxhash32_decimal"
|
||||||
|
);
|
||||||
|
assert_eq!(value["download"]["official_hash"]["expected"], "2044170421");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
|
|||||||
@@ -59,9 +59,10 @@ pub use official_download::{
|
|||||||
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
|
read_download_manifest_at, DownloadError, OfficialDownloadManifest,
|
||||||
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
|
OfficialDownloadManifestEntry, OfficialLocalManifestAuditItem,
|
||||||
OfficialLocalManifestAuditReport, OfficialLocalManifestAuditStatus,
|
OfficialLocalManifestAuditReport, OfficialLocalManifestAuditStatus,
|
||||||
OfficialLocalVerificationReport, OfficialResourcePullItem, OfficialResourcePullProgress,
|
OfficialLocalVerificationReport, OfficialResourceHashAlgorithm,
|
||||||
|
OfficialResourceHashVerification, OfficialResourcePullItem, OfficialResourcePullProgress,
|
||||||
OfficialResourcePullProgressKind, OfficialResourcePullReport, OfficialResourcePullService,
|
OfficialResourcePullProgressKind, OfficialResourcePullReport, OfficialResourcePullService,
|
||||||
OfficialResourcePullStatus,
|
OfficialResourcePullStatus, OfficialResourceVerification,
|
||||||
};
|
};
|
||||||
pub use official_game_main_config::OfficialGameMainConfigBootstrapService;
|
pub use official_game_main_config::OfficialGameMainConfigBootstrapService;
|
||||||
pub use official_launcher::{
|
pub use official_launcher::{
|
||||||
|
|||||||
@@ -91,6 +91,8 @@ pub enum OfficialResourcePullProgressKind {
|
|||||||
Started,
|
Started,
|
||||||
/// A URL finished as skipped, resumed, or downloaded.
|
/// A URL finished as skipped, resumed, or downloaded.
|
||||||
Finished,
|
Finished,
|
||||||
|
/// An official sidecar hash pair passed verification.
|
||||||
|
Verification,
|
||||||
/// A URL failed after retry classification and was quarantined for this run.
|
/// A URL failed after retry classification and was quarantined for this run.
|
||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
@@ -101,11 +103,29 @@ impl OfficialResourcePullProgressKind {
|
|||||||
match self {
|
match self {
|
||||||
Self::Started => "started",
|
Self::Started => "started",
|
||||||
Self::Finished => "finished",
|
Self::Finished => "finished",
|
||||||
|
Self::Verification => "verification",
|
||||||
Self::Failed => "failed",
|
Self::Failed => "failed",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Local validation results for one completed resource.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct OfficialResourceVerification {
|
||||||
|
/// Expected byte size from the local manifest, when reusing a file.
|
||||||
|
pub expected_bytes: Option<u64>,
|
||||||
|
/// Actual byte size observed after the transfer.
|
||||||
|
pub actual_bytes: u64,
|
||||||
|
/// Expected BLAKE3 from the local manifest, when reusing a file.
|
||||||
|
pub expected_blake3: Option<String>,
|
||||||
|
/// Actual BLAKE3 computed from the completed file.
|
||||||
|
pub actual_blake3: String,
|
||||||
|
/// Whether ZIP structure validation was required for this resource.
|
||||||
|
pub zip_checked: bool,
|
||||||
|
/// Whether the required ZIP structure validation passed.
|
||||||
|
pub zip_structure_verified: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Progress for one URL in an official pull plan.
|
/// Progress for one URL in an official pull plan.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct OfficialResourcePullProgress {
|
pub struct OfficialResourcePullProgress {
|
||||||
@@ -128,6 +148,10 @@ pub struct OfficialResourcePullProgress {
|
|||||||
pub bytes: Option<u64>,
|
pub bytes: Option<u64>,
|
||||||
/// Bytes transferred during this run, when `kind` is `Finished`.
|
/// Bytes transferred during this run, when `kind` is `Finished`.
|
||||||
pub transferred_bytes: Option<u64>,
|
pub transferred_bytes: Option<u64>,
|
||||||
|
/// Local size/BLAKE3/ZIP validation for a completed URL.
|
||||||
|
pub verification: Option<OfficialResourceVerification>,
|
||||||
|
/// Official sidecar hash verification completed after this URL.
|
||||||
|
pub official_hash: Option<OfficialResourceHashVerification>,
|
||||||
/// Stable failure kind label, when `kind` is `Failed`.
|
/// Stable failure kind label, when `kind` is `Failed`.
|
||||||
pub failure_kind: Option<String>,
|
pub failure_kind: Option<String>,
|
||||||
/// HTTP status parsed from curl stderr, when available.
|
/// HTTP status parsed from curl stderr, when available.
|
||||||
@@ -150,6 +174,8 @@ impl OfficialResourcePullProgress {
|
|||||||
status: None,
|
status: None,
|
||||||
bytes: None,
|
bytes: None,
|
||||||
transferred_bytes: None,
|
transferred_bytes: None,
|
||||||
|
verification: None,
|
||||||
|
official_hash: None,
|
||||||
failure_kind: None,
|
failure_kind: None,
|
||||||
failure_http_status: None,
|
failure_http_status: None,
|
||||||
failure_retryable: None,
|
failure_retryable: None,
|
||||||
@@ -165,6 +191,7 @@ impl OfficialResourcePullProgress {
|
|||||||
status: OfficialResourcePullStatus,
|
status: OfficialResourcePullStatus,
|
||||||
bytes: u64,
|
bytes: u64,
|
||||||
transferred_bytes: u64,
|
transferred_bytes: u64,
|
||||||
|
verification: OfficialResourceVerification,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
kind: OfficialResourcePullProgressKind::Finished,
|
kind: OfficialResourcePullProgressKind::Finished,
|
||||||
@@ -174,6 +201,32 @@ impl OfficialResourcePullProgress {
|
|||||||
status: Some(status),
|
status: Some(status),
|
||||||
bytes: Some(bytes),
|
bytes: Some(bytes),
|
||||||
transferred_bytes: Some(transferred_bytes),
|
transferred_bytes: Some(transferred_bytes),
|
||||||
|
verification: Some(verification),
|
||||||
|
official_hash: None,
|
||||||
|
failure_kind: None,
|
||||||
|
failure_http_status: None,
|
||||||
|
failure_retryable: None,
|
||||||
|
failure_attempts: None,
|
||||||
|
quarantined: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verification(
|
||||||
|
index: usize,
|
||||||
|
total: usize,
|
||||||
|
url: String,
|
||||||
|
official_hash: OfficialResourceHashVerification,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
kind: OfficialResourcePullProgressKind::Verification,
|
||||||
|
index,
|
||||||
|
total,
|
||||||
|
url,
|
||||||
|
status: None,
|
||||||
|
bytes: None,
|
||||||
|
transferred_bytes: None,
|
||||||
|
verification: None,
|
||||||
|
official_hash: Some(official_hash),
|
||||||
failure_kind: None,
|
failure_kind: None,
|
||||||
failure_http_status: None,
|
failure_http_status: None,
|
||||||
failure_retryable: None,
|
failure_retryable: None,
|
||||||
@@ -191,6 +244,8 @@ impl OfficialResourcePullProgress {
|
|||||||
status: None,
|
status: None,
|
||||||
bytes: None,
|
bytes: None,
|
||||||
transferred_bytes: None,
|
transferred_bytes: None,
|
||||||
|
verification: None,
|
||||||
|
official_hash: None,
|
||||||
failure_kind: error.failure_kind().map(ToOwned::to_owned),
|
failure_kind: error.failure_kind().map(ToOwned::to_owned),
|
||||||
failure_http_status: error.http_status(),
|
failure_http_status: error.http_status(),
|
||||||
failure_retryable: error.retryable(),
|
failure_retryable: error.retryable(),
|
||||||
@@ -201,9 +256,10 @@ impl OfficialResourcePullProgress {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Official hash algorithm used by a verified resource sidecar.
|
/// Official hash algorithm used by a verified resource sidecar.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub enum OfficialResourceHashAlgorithm {
|
pub enum OfficialResourceHashAlgorithm {
|
||||||
/// Decimal text form of `xxHash32` with seed `0`.
|
/// Decimal text form of `xxHash32` with seed `0`.
|
||||||
|
#[serde(rename = "xxhash32_decimal")]
|
||||||
XxHash32Decimal,
|
XxHash32Decimal,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,7 +273,7 @@ impl OfficialResourceHashAlgorithm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Successful official hash verification for one downloaded resource.
|
/// Successful official hash verification for one downloaded resource.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct OfficialResourceHashVerification {
|
pub struct OfficialResourceHashVerification {
|
||||||
/// Downloaded data URL that was verified.
|
/// Downloaded data URL that was verified.
|
||||||
pub data_url: String,
|
pub data_url: String,
|
||||||
@@ -534,6 +590,40 @@ impl OfficialResourcePullService {
|
|||||||
self.output_root.join(DOWNLOAD_QUARANTINE_FILE)
|
self.output_root.join(DOWNLOAD_QUARANTINE_FILE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Removes resource files recorded by an older manifest but absent from
|
||||||
|
/// the current pull plan.
|
||||||
|
///
|
||||||
|
/// This is intentionally limited to destinations owned by the previous
|
||||||
|
/// download manifest. Management files such as snapshots, version state,
|
||||||
|
/// and quarantine records are left untouched.
|
||||||
|
pub fn prune_stale_manifest_entries(
|
||||||
|
&self,
|
||||||
|
plan: &OfficialResourcePullPlan,
|
||||||
|
) -> Result<usize, String> {
|
||||||
|
self.ensure_output_root_ready()?;
|
||||||
|
let expected_urls = plan.all_urls()?.into_iter().collect::<HashSet<_>>();
|
||||||
|
let mut manifest = self.read_download_manifest()?;
|
||||||
|
let stale_entries = manifest
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.filter(|(url, _)| !expected_urls.contains(*url))
|
||||||
|
.map(|(url, entry)| (url.clone(), entry.destination.clone()))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
for (url, relative_destination) in &stale_entries {
|
||||||
|
let destination = self.output_root.join(relative_destination);
|
||||||
|
self.remove_managed_destination(&destination, url)?;
|
||||||
|
self.remove_managed_destination(&partial_path_for(&destination), url)?;
|
||||||
|
manifest.entries.remove(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !stale_entries.is_empty() {
|
||||||
|
self.write_download_manifest(&manifest)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(stale_entries.len())
|
||||||
|
}
|
||||||
|
|
||||||
/// Executes the plan and downloads every official URL to disk.
|
/// Executes the plan and downloads every official URL to disk.
|
||||||
pub fn pull(
|
pub fn pull(
|
||||||
&self,
|
&self,
|
||||||
@@ -630,8 +720,8 @@ impl OfficialResourcePullService {
|
|||||||
|
|
||||||
let result = self.pull_one(&item.url, &item.destination);
|
let result = self.pull_one(&item.url, &item.destination);
|
||||||
match result {
|
match result {
|
||||||
Ok(pull_result) => {
|
Ok(mut pull_result) => {
|
||||||
if let Err(error) = self
|
let verification_result = self
|
||||||
.clear_quarantine_entry(&item.url)
|
.clear_quarantine_entry(&item.url)
|
||||||
.and_then(|_| {
|
.and_then(|_| {
|
||||||
self.record_download_manifest_entry(
|
self.record_download_manifest_entry(
|
||||||
@@ -640,9 +730,17 @@ impl OfficialResourcePullService {
|
|||||||
&item.destination,
|
&item.destination,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.and_then(|_| self.write_download_manifest(&manifest))
|
.and_then(|verification| {
|
||||||
{
|
self.write_download_manifest(&manifest)
|
||||||
let error = PullOneError::plain(format!("记录下载 manifest 失败:{error}"));
|
.map(|_| verification)
|
||||||
|
});
|
||||||
|
match verification_result {
|
||||||
|
Ok(verification) => {
|
||||||
|
pull_result.verification = verification;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let error =
|
||||||
|
PullOneError::plain(format!("记录下载 manifest 失败:{error}"));
|
||||||
self.record_quarantine_entry(&item.url, &item.destination, &error)?;
|
self.record_quarantine_entry(&item.url, &item.destination, &error)?;
|
||||||
progress(OfficialResourcePullProgress::failed(
|
progress(OfficialResourcePullProgress::failed(
|
||||||
completed_count,
|
completed_count,
|
||||||
@@ -660,6 +758,7 @@ impl OfficialResourcePullService {
|
|||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
completed_count += 1;
|
completed_count += 1;
|
||||||
progress(OfficialResourcePullProgress::finished(
|
progress(OfficialResourcePullProgress::finished(
|
||||||
@@ -669,6 +768,7 @@ impl OfficialResourcePullService {
|
|||||||
pull_result.status,
|
pull_result.status,
|
||||||
pull_result.bytes,
|
pull_result.bytes,
|
||||||
pull_result.transferred_bytes,
|
pull_result.transferred_bytes,
|
||||||
|
pull_result.verification.clone(),
|
||||||
));
|
));
|
||||||
download_results[plan_index] = Some(Ok(pull_result));
|
download_results[plan_index] = Some(Ok(pull_result));
|
||||||
}
|
}
|
||||||
@@ -719,8 +819,9 @@ impl OfficialResourcePullService {
|
|||||||
existing.status,
|
existing.status,
|
||||||
existing.bytes,
|
existing.bytes,
|
||||||
existing.transferred_bytes,
|
existing.transferred_bytes,
|
||||||
|
existing.verification.clone(),
|
||||||
));
|
));
|
||||||
*existing
|
existing.clone()
|
||||||
} else {
|
} else {
|
||||||
// Phase B 已保证需下载项此时均为 Ok(失败会在上面 fail-fast 返回)。
|
// Phase B 已保证需下载项此时均为 Ok(失败会在上面 fail-fast 返回)。
|
||||||
match download_results[plan_index].take() {
|
match download_results[plan_index].take() {
|
||||||
@@ -735,13 +836,21 @@ impl OfficialResourcePullService {
|
|||||||
};
|
};
|
||||||
|
|
||||||
processed_urls.insert(item.url.clone());
|
processed_urls.insert(item.url.clone());
|
||||||
self.verify_ready_official_hashes(
|
let newly_verified_hashes = self.verify_ready_official_hashes(
|
||||||
&official_hash_pairs,
|
&official_hash_pairs,
|
||||||
&processed_urls,
|
&processed_urls,
|
||||||
&mut verified_hash_urls,
|
&mut verified_hash_urls,
|
||||||
&mut verified_hashes,
|
&mut verified_hashes,
|
||||||
&mut manifest,
|
&mut manifest,
|
||||||
)?;
|
)?;
|
||||||
|
for verification in newly_verified_hashes {
|
||||||
|
progress(OfficialResourcePullProgress::verification(
|
||||||
|
completed_count,
|
||||||
|
total,
|
||||||
|
verification.data_url.clone(),
|
||||||
|
verification,
|
||||||
|
));
|
||||||
|
}
|
||||||
items.push(OfficialResourcePullItem {
|
items.push(OfficialResourcePullItem {
|
||||||
url: item.url.clone(),
|
url: item.url.clone(),
|
||||||
destination: item.destination.clone(),
|
destination: item.destination.clone(),
|
||||||
@@ -960,6 +1069,9 @@ impl OfficialResourcePullService {
|
|||||||
.map_err(PullOneError::plain)?
|
.map_err(PullOneError::plain)?
|
||||||
.ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))
|
.ok_or_else(|| format!("下载完成后目标文件缺失 {}", destination.display()))
|
||||||
.map_err(PullOneError::plain)?;
|
.map_err(PullOneError::plain)?;
|
||||||
|
let verification = self
|
||||||
|
.local_verification(url, destination, None, None)
|
||||||
|
.map_err(PullOneError::plain)?;
|
||||||
|
|
||||||
Ok(PullOneResult {
|
Ok(PullOneResult {
|
||||||
bytes,
|
bytes,
|
||||||
@@ -969,6 +1081,7 @@ impl OfficialResourcePullService {
|
|||||||
bytes
|
bytes
|
||||||
},
|
},
|
||||||
status,
|
status,
|
||||||
|
verification,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1010,7 +1123,8 @@ impl OfficialResourcePullService {
|
|||||||
verified_hash_urls: &mut HashSet<String>,
|
verified_hash_urls: &mut HashSet<String>,
|
||||||
verified_hashes: &mut Vec<OfficialResourceHashVerification>,
|
verified_hashes: &mut Vec<OfficialResourceHashVerification>,
|
||||||
manifest: &mut OfficialDownloadManifest,
|
manifest: &mut OfficialDownloadManifest,
|
||||||
) -> Result<(), String> {
|
) -> Result<Vec<OfficialResourceHashVerification>, String> {
|
||||||
|
let mut newly_verified = Vec::new();
|
||||||
for pair in pairs {
|
for pair in pairs {
|
||||||
if verified_hash_urls.contains(&pair.hash_url) {
|
if verified_hash_urls.contains(&pair.hash_url) {
|
||||||
continue;
|
continue;
|
||||||
@@ -1040,11 +1154,12 @@ impl OfficialResourcePullService {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
newly_verified.push(verification.clone());
|
||||||
verified_hashes.push(verification);
|
verified_hashes.push(verification);
|
||||||
verified_hash_urls.insert(pair.hash_url.clone());
|
verified_hash_urls.insert(pair.hash_url.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(newly_verified)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result<bool, String> {
|
fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result<bool, String> {
|
||||||
@@ -1124,6 +1239,41 @@ impl OfficialResourcePullService {
|
|||||||
Ok(manifest)
|
Ok(manifest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remove_managed_destination(&self, path: &Path, url: &str) -> Result<(), String> {
|
||||||
|
ensure_path_within_root(&self.output_root, path)?;
|
||||||
|
ensure_safe_file_target(&self.output_root, path, "旧官方资源清理目标")?;
|
||||||
|
|
||||||
|
match fs::symlink_metadata(path) {
|
||||||
|
Ok(metadata) if metadata.file_type().is_symlink() => Err(format!(
|
||||||
|
"旧官方资源清理目标不能是 symlink:url={url} path={}",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
Ok(metadata) if metadata.is_file() => {
|
||||||
|
fs::remove_file(path).map_err(|error| {
|
||||||
|
format!(
|
||||||
|
"清理旧官方资源失败:url={url} path={} error={error}",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
remove_empty_parent_directories(&self.output_root, path.parent())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Ok(metadata) if metadata.is_dir() => Err(format!(
|
||||||
|
"旧官方资源清理目标不能是目录:url={url} path={}",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
Ok(_) => Err(format!(
|
||||||
|
"旧官方资源清理目标不是普通文件:url={url} path={}",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||||
|
Err(error) => Err(format!(
|
||||||
|
"检查旧官方资源清理目标失败:url={url} path={} error={error}",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> {
|
fn write_download_manifest(&self, manifest: &OfficialDownloadManifest) -> Result<(), String> {
|
||||||
self.ensure_output_root_ready()?;
|
self.ensure_output_root_ready()?;
|
||||||
let path = self.download_manifest_path();
|
let path = self.download_manifest_path();
|
||||||
@@ -1253,23 +1403,52 @@ impl OfficialResourcePullService {
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let verification = self.local_verification(
|
||||||
|
url,
|
||||||
|
destination,
|
||||||
|
Some(entry.bytes),
|
||||||
|
Some(entry.blake3.clone()),
|
||||||
|
)?;
|
||||||
Ok(Some(PullOneResult {
|
Ok(Some(PullOneResult {
|
||||||
bytes,
|
bytes,
|
||||||
transferred_bytes: 0,
|
transferred_bytes: 0,
|
||||||
status: OfficialResourcePullStatus::SkippedExisting,
|
status: OfficialResourcePullStatus::SkippedExisting,
|
||||||
|
verification,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn local_verification(
|
||||||
|
&self,
|
||||||
|
url: &str,
|
||||||
|
destination: &Path,
|
||||||
|
expected_bytes: Option<u64>,
|
||||||
|
expected_blake3: Option<String>,
|
||||||
|
) -> Result<OfficialResourceVerification, String> {
|
||||||
|
let actual_bytes = file_len_if_exists(destination)?
|
||||||
|
.ok_or_else(|| format!("读取资源文件信息失败 {}", destination.display()))?;
|
||||||
|
let actual_blake3 = blake3_file_hex(destination)?;
|
||||||
|
let zip_checked = url_or_path_has_zip_extension(url) || path_has_zip_extension(destination);
|
||||||
|
if zip_checked {
|
||||||
|
self.validate_zip_if_needed(url, destination)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(OfficialResourceVerification {
|
||||||
|
expected_bytes,
|
||||||
|
actual_bytes,
|
||||||
|
expected_blake3,
|
||||||
|
actual_blake3,
|
||||||
|
zip_checked,
|
||||||
|
zip_structure_verified: zip_checked,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn record_download_manifest_entry(
|
fn record_download_manifest_entry(
|
||||||
&self,
|
&self,
|
||||||
manifest: &mut OfficialDownloadManifest,
|
manifest: &mut OfficialDownloadManifest,
|
||||||
url: &str,
|
url: &str,
|
||||||
destination: &Path,
|
destination: &Path,
|
||||||
) -> Result<(), String> {
|
) -> Result<OfficialResourceVerification, String> {
|
||||||
self.validate_zip_if_needed(url, destination)?;
|
let mut verification = self.local_verification(url, destination, None, None)?;
|
||||||
let bytes = file_len_if_exists(destination)?
|
|
||||||
.ok_or_else(|| format!("读取已下载文件信息失败 {}", destination.display()))?;
|
|
||||||
let digest = blake3_file_hex(destination)?;
|
|
||||||
let relative_destination = self.relative_destination(destination)?;
|
let relative_destination = self.relative_destination(destination)?;
|
||||||
|
|
||||||
manifest.entries.insert(
|
manifest.entries.insert(
|
||||||
@@ -1277,12 +1456,14 @@ impl OfficialResourcePullService {
|
|||||||
OfficialDownloadManifestEntry {
|
OfficialDownloadManifestEntry {
|
||||||
url: url.to_string(),
|
url: url.to_string(),
|
||||||
destination: relative_destination,
|
destination: relative_destination,
|
||||||
bytes,
|
bytes: verification.actual_bytes,
|
||||||
blake3: digest,
|
blake3: verification.actual_blake3.clone(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(())
|
verification.expected_bytes = Some(verification.actual_bytes);
|
||||||
|
verification.expected_blake3 = Some(verification.actual_blake3.clone());
|
||||||
|
Ok(verification)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_zip_if_needed(&self, url: &str, destination: &Path) -> Result<(), String> {
|
fn validate_zip_if_needed(&self, url: &str, destination: &Path) -> Result<(), String> {
|
||||||
@@ -1703,11 +1884,12 @@ struct PlannedDownload {
|
|||||||
existing: Option<PullOneResult>,
|
existing: Option<PullOneResult>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
struct PullOneResult {
|
struct PullOneResult {
|
||||||
bytes: u64,
|
bytes: u64,
|
||||||
transferred_bytes: u64,
|
transferred_bytes: u64,
|
||||||
status: OfficialResourcePullStatus,
|
status: OfficialResourcePullStatus,
|
||||||
|
verification: OfficialResourceVerification,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -1788,6 +1970,48 @@ fn partial_path_for(destination: &Path) -> PathBuf {
|
|||||||
PathBuf::from(partial)
|
PathBuf::from(partial)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remove_empty_parent_directories(root: &Path, parent: Option<&Path>) -> Result<(), String> {
|
||||||
|
let mut current = parent;
|
||||||
|
while let Some(path) = current {
|
||||||
|
if path == root {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let metadata = match fs::symlink_metadata(path) {
|
||||||
|
Ok(metadata) => metadata,
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||||
|
current = path.parent();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
return Err(format!(
|
||||||
|
"检查旧官方资源父目录失败 {}:{error}",
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(format!(
|
||||||
|
"旧官方资源父目录不能是 symlink:{}",
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !metadata.is_dir() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries = fs::read_dir(path)
|
||||||
|
.map_err(|error| format!("读取旧官方资源父目录失败 {}:{error}", path.display()))?;
|
||||||
|
if entries.next().is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
fs::remove_dir(path)
|
||||||
|
.map_err(|error| format!("清理旧官方资源空目录失败 {}:{error}", path.display()))?;
|
||||||
|
current = path.parent();
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn unix_seconds_now() -> u64 {
|
fn unix_seconds_now() -> u64 {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
@@ -2711,6 +2935,95 @@ exit 22
|
|||||||
assert!(manifest_state.has_any_resources());
|
assert!(manifest_state.has_any_resources());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prune_stale_manifest_entries_removes_old_resource_and_partial_only() {
|
||||||
|
let out_dir = TempDir::new().unwrap();
|
||||||
|
let service = OfficialResourcePullService::new(out_dir.path());
|
||||||
|
let plan = build_official_pull_plan(discovery_plan(), inventory());
|
||||||
|
let expected_url = plan.all_urls().unwrap().into_iter().next().unwrap();
|
||||||
|
let expected_destination = service.destination_for_url(&expected_url).unwrap();
|
||||||
|
fs::create_dir_all(expected_destination.parent().unwrap()).unwrap();
|
||||||
|
fs::write(&expected_destination, b"keep").unwrap();
|
||||||
|
|
||||||
|
let stale_url =
|
||||||
|
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle";
|
||||||
|
let stale_destination = service.destination_for_url(stale_url).unwrap();
|
||||||
|
fs::create_dir_all(stale_destination.parent().unwrap()).unwrap();
|
||||||
|
fs::write(&stale_destination, b"remove").unwrap();
|
||||||
|
fs::write(partial_path_for(&stale_destination), b"remove-partial").unwrap();
|
||||||
|
|
||||||
|
let mut manifest = OfficialDownloadManifest::default();
|
||||||
|
service
|
||||||
|
.record_download_manifest_entry(&mut manifest, &expected_url, &expected_destination)
|
||||||
|
.unwrap();
|
||||||
|
manifest.entries.insert(
|
||||||
|
stale_url.to_string(),
|
||||||
|
OfficialDownloadManifestEntry {
|
||||||
|
url: stale_url.to_string(),
|
||||||
|
destination: service.relative_destination(&stale_destination).unwrap(),
|
||||||
|
bytes: 6,
|
||||||
|
blake3: "stale".to_string(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
service.write_download_manifest(&manifest).unwrap();
|
||||||
|
|
||||||
|
let snapshot_path = out_dir.path().join("official-sync-snapshot.json");
|
||||||
|
let quarantine_path = out_dir.path().join(DOWNLOAD_QUARANTINE_FILE);
|
||||||
|
fs::write(&snapshot_path, b"snapshot").unwrap();
|
||||||
|
fs::write(&quarantine_path, b"quarantine").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(service.prune_stale_manifest_entries(&plan).unwrap(), 1);
|
||||||
|
assert!(!stale_destination.exists());
|
||||||
|
assert!(!partial_path_for(&stale_destination).exists());
|
||||||
|
assert!(!stale_destination.parent().unwrap().exists());
|
||||||
|
assert!(expected_destination.exists());
|
||||||
|
assert_eq!(fs::read(&snapshot_path).unwrap(), b"snapshot");
|
||||||
|
assert_eq!(fs::read(&quarantine_path).unwrap(), b"quarantine");
|
||||||
|
|
||||||
|
let pruned_manifest = service.read_download_manifest().unwrap();
|
||||||
|
assert!(pruned_manifest.entries.contains_key(&expected_url));
|
||||||
|
assert!(!pruned_manifest.entries.contains_key(stale_url));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn prune_stale_manifest_entries_rejects_symlink_destination() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let out_dir = TempDir::new().unwrap();
|
||||||
|
let service = OfficialResourcePullService::new(out_dir.path());
|
||||||
|
let plan = build_official_pull_plan(discovery_plan(), inventory());
|
||||||
|
let stale_url =
|
||||||
|
"https://prod-clientpatch.bluearchiveyostar.com/r93_token/obsolete/old.bundle";
|
||||||
|
let stale_destination = service.destination_for_url(stale_url).unwrap();
|
||||||
|
fs::create_dir_all(stale_destination.parent().unwrap()).unwrap();
|
||||||
|
let outside = out_dir.path().join("outside-resource");
|
||||||
|
fs::write(&outside, b"must-keep").unwrap();
|
||||||
|
symlink(&outside, &stale_destination).unwrap();
|
||||||
|
|
||||||
|
let mut manifest = OfficialDownloadManifest::default();
|
||||||
|
manifest.entries.insert(
|
||||||
|
stale_url.to_string(),
|
||||||
|
OfficialDownloadManifestEntry {
|
||||||
|
url: stale_url.to_string(),
|
||||||
|
destination: service.relative_destination(&stale_destination).unwrap(),
|
||||||
|
bytes: 9,
|
||||||
|
blake3: "stale".to_string(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
service.write_download_manifest(&manifest).unwrap();
|
||||||
|
|
||||||
|
let error = service.prune_stale_manifest_entries(&plan).unwrap_err();
|
||||||
|
assert!(error.contains("symlink"));
|
||||||
|
assert!(stale_destination.exists());
|
||||||
|
assert_eq!(fs::read(&outside).unwrap(), b"must-keep");
|
||||||
|
assert!(service
|
||||||
|
.read_download_manifest()
|
||||||
|
.unwrap()
|
||||||
|
.entries
|
||||||
|
.contains_key(stale_url));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn redownloads_existing_file_when_manifest_hash_mismatches() {
|
fn redownloads_existing_file_when_manifest_hash_mismatches() {
|
||||||
let out_dir = TempDir::new().unwrap();
|
let out_dir = TempDir::new().unwrap();
|
||||||
@@ -2974,8 +3287,6 @@ exit 22
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(report.items.len(), expected_urls);
|
assert_eq!(report.items.len(), expected_urls);
|
||||||
assert_eq!(events.len(), expected_urls * 2);
|
|
||||||
|
|
||||||
let started: Vec<_> = events
|
let started: Vec<_> = events
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|event| event.kind == OfficialResourcePullProgressKind::Started)
|
.filter(|event| event.kind == OfficialResourcePullProgressKind::Started)
|
||||||
@@ -2984,8 +3295,13 @@ exit 22
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
|
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
|
||||||
.collect();
|
.collect();
|
||||||
|
let verifications: Vec<_> = events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| event.kind == OfficialResourcePullProgressKind::Verification)
|
||||||
|
.collect();
|
||||||
assert_eq!(started.len(), expected_urls);
|
assert_eq!(started.len(), expected_urls);
|
||||||
assert_eq!(finished.len(), expected_urls);
|
assert_eq!(finished.len(), expected_urls);
|
||||||
|
assert_eq!(verifications.len(), report.verified_hashes.len());
|
||||||
|
|
||||||
// 每个 started 的 total 一致;index 表示已完成数量,不能超过总数。
|
// 每个 started 的 total 一致;index 表示已完成数量,不能超过总数。
|
||||||
assert!(started.iter().all(|event| event.index <= expected_urls));
|
assert!(started.iter().all(|event| event.index <= expected_urls));
|
||||||
@@ -3001,6 +3317,18 @@ exit 22
|
|||||||
assert!(finished
|
assert!(finished
|
||||||
.iter()
|
.iter()
|
||||||
.all(|event| event.status == Some(OfficialResourcePullStatus::Downloaded)));
|
.all(|event| event.status == Some(OfficialResourcePullStatus::Downloaded)));
|
||||||
|
assert!(finished.iter().all(|event| {
|
||||||
|
event.verification.as_ref().is_some_and(|verification| {
|
||||||
|
verification.actual_bytes > 0 && !verification.actual_blake3.is_empty()
|
||||||
|
})
|
||||||
|
}));
|
||||||
|
assert!(verifications.iter().all(|event| {
|
||||||
|
event.official_hash.as_ref().is_some_and(|hash| {
|
||||||
|
hash.algorithm == OfficialResourceHashAlgorithm::XxHash32Decimal
|
||||||
|
&& hash.expected == hash.actual
|
||||||
|
&& !hash.hash_url.is_empty()
|
||||||
|
})
|
||||||
|
}));
|
||||||
assert!(events
|
assert!(events
|
||||||
.iter()
|
.iter()
|
||||||
.any(|event| event.url.ends_with("/TableBundles/ExcelDB.db")));
|
.any(|event| event.url.ends_with("/TableBundles/ExcelDB.db")));
|
||||||
|
|||||||
@@ -32,9 +32,10 @@ use crate::{
|
|||||||
build_official_pull_plan_for_platform_inventory, build_official_sync_plan,
|
build_official_pull_plan_for_platform_inventory, build_official_sync_plan,
|
||||||
changed_endpoint_urls, default_official_platforms, DownloadError,
|
changed_endpoint_urls, default_official_platforms, DownloadError,
|
||||||
OfficialGameMainConfigBootstrapService, OfficialLauncherBootstrapService,
|
OfficialGameMainConfigBootstrapService, OfficialLauncherBootstrapService,
|
||||||
OfficialResourcePullPlan, OfficialResourcePullProgress, OfficialResourcePullProgressKind,
|
OfficialResourceHashVerification, OfficialResourcePullPlan, OfficialResourcePullProgress,
|
||||||
OfficialResourcePullService, YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig,
|
OfficialResourcePullProgressKind, OfficialResourcePullService, OfficialResourceVerification,
|
||||||
YostarJpLauncherManifestUrl, YostarJpLauncherRemoteManifest,
|
YostarJpLauncherCdnConfig, YostarJpLauncherGameConfig, YostarJpLauncherManifestUrl,
|
||||||
|
YostarJpLauncherRemoteManifest,
|
||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
read_parse_cache_at, OfficialParseCacheService, OfficialParseConfig, OfficialParseSummary,
|
read_parse_cache_at, OfficialParseCacheService, OfficialParseConfig, OfficialParseSummary,
|
||||||
@@ -865,6 +866,10 @@ pub struct OfficialUpdateProgress {
|
|||||||
pub download_failure_attempts: Option<usize>,
|
pub download_failure_attempts: Option<usize>,
|
||||||
/// Whether the URL was recorded in the quarantine manifest.
|
/// Whether the URL was recorded in the quarantine manifest.
|
||||||
pub download_quarantined: Option<bool>,
|
pub download_quarantined: Option<bool>,
|
||||||
|
/// Local size/BLAKE3/ZIP verification for a completed URL.
|
||||||
|
pub download_verification: Option<OfficialResourceVerification>,
|
||||||
|
/// Official `.hash` sidecar verification when a seed catalog pair passes.
|
||||||
|
pub official_hash_verification: Option<OfficialResourceHashVerification>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl OfficialUpdateProgress {
|
impl OfficialUpdateProgress {
|
||||||
@@ -884,6 +889,8 @@ impl OfficialUpdateProgress {
|
|||||||
download_failure_retryable: None,
|
download_failure_retryable: None,
|
||||||
download_failure_attempts: None,
|
download_failure_attempts: None,
|
||||||
download_quarantined: None,
|
download_quarantined: None,
|
||||||
|
download_verification: None,
|
||||||
|
official_hash_verification: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -906,6 +913,8 @@ impl OfficialUpdateProgress {
|
|||||||
self.download_failure_attempts = event.failure_attempts;
|
self.download_failure_attempts = event.failure_attempts;
|
||||||
self.download_quarantined =
|
self.download_quarantined =
|
||||||
(event.kind == OfficialResourcePullProgressKind::Failed).then_some(event.quarantined);
|
(event.kind == OfficialResourcePullProgressKind::Failed).then_some(event.quarantined);
|
||||||
|
self.download_verification = event.verification.clone();
|
||||||
|
self.official_hash_verification = event.official_hash.clone();
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1808,6 +1817,18 @@ impl OfficialUpdateService {
|
|||||||
&config.curl_command,
|
&config.curl_command,
|
||||||
)
|
)
|
||||||
.with_proxy_config(config.curl_proxy.clone());
|
.with_proxy_config(config.curl_proxy.clone());
|
||||||
|
let pruned_stale_resource_count = staging_fetcher
|
||||||
|
.prune_stale_manifest_entries(&pull_plan)
|
||||||
|
.map_err(anyhow::Error::msg)?;
|
||||||
|
if pruned_stale_resource_count > 0 {
|
||||||
|
progress(OfficialUpdateProgress::new(
|
||||||
|
"publish",
|
||||||
|
format!(
|
||||||
|
"已清理新 manifest 删除的旧官方资源:{} 项",
|
||||||
|
pruned_stale_resource_count
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
report.staging_path = Some(publish_plan.staging_path.clone());
|
report.staging_path = Some(publish_plan.staging_path.clone());
|
||||||
report.snapshot_path = staging_snapshot_path.clone();
|
report.snapshot_path = staging_snapshot_path.clone();
|
||||||
report.download_manifest = staging_fetcher.download_manifest_path();
|
report.download_manifest = staging_fetcher.download_manifest_path();
|
||||||
@@ -2701,6 +2722,21 @@ fn progress_from_pull_event(event: OfficialResourcePullProgress) -> OfficialUpda
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
.with_download_progress(&event),
|
.with_download_progress(&event),
|
||||||
|
OfficialResourcePullProgressKind::Verification => {
|
||||||
|
let hash = event.official_hash.as_ref();
|
||||||
|
OfficialUpdateProgress::new(
|
||||||
|
"verify",
|
||||||
|
format!(
|
||||||
|
"官方 hash 校验通过:算法={} 期望={} 实际={} 数据 URL={} hash URL={}",
|
||||||
|
hash.map(|value| value.algorithm.as_str()).unwrap_or("unknown"),
|
||||||
|
hash.map(|value| value.expected.as_str()).unwrap_or("unknown"),
|
||||||
|
hash.map(|value| value.actual.as_str()).unwrap_or("unknown"),
|
||||||
|
hash.map(|value| value.data_url.as_str()).unwrap_or(event.url.as_str()),
|
||||||
|
hash.map(|value| value.hash_url.as_str()).unwrap_or("unknown")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.with_download_progress(&event)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user