mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 12:45:17 +08:00
feat(download): 多线程下载 + 重试指数退避(issue #17)
下载主循环由串行改为「并发下载 + 串行簿记」三段式: - Phase A:无网络前置校验(官方性/目标路径/建目录)+ 判定跳过/需下载, 非官方 URL 在任何下载前 fail-fast - Phase B:need-download 项经 scoped 线程池并发下载(默认并发 4, 可配 1..=256)。worker 只做只读 &self 的 pull_one(各 URL 独立 .part/目标文件),经 mpsc 把结果送回主线程;manifest/quarantine 簿记与进度回调全在主线程串行执行,无需加锁。首个失败或 should_cancel 置 cancel 标志,其余 worker 在任务边界停止 - Phase C:按 plan 顺序串行收尾——seed .hash 校验(顺序相关、可 fail-fast)+ 构建有序结果 fail-fast 与「不发布不完整资源」不变量保留;进度事件按 URL 配对但 顺序不再单调(并发下天然如此)。 curl 重试加指数退避(网络类 200ms→400ms→800ms…上限 5s;ETXTBSY 仍走 极短退避),并发下对官方 CDN 更礼貌;退避基值 cfg(test) 下为 0 不拖慢 单测。 并发度经 OfficialUpdateConfig.download_concurrency 贯通,CLI --download-concurrency 与 BAT_DOWNLOAD_CONCURRENCY 可配,.env 模板 与 USERGUIDE/CURRENT_STATUS/CHANGELOG 同步。 验证:新增并发正确性测试(并发 8:每 URL 恰一次 started+finished、 全部落盘)、并发度钳制、退避时长计算、CLI/env 解析单测;progress 排序测试改为顺序无关不变量;workspace 全测试(20 套件) + fmt + clippy --all-targets -D warnings 全绿。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -427,6 +427,34 @@ impl std::fmt::Display for CurlRetryError {
|
||||
|
||||
impl std::error::Error for CurlRetryError {}
|
||||
|
||||
/// 网络类可重试失败的退避基值(毫秒)。生产 200ms、指数增长;测试下为 0
|
||||
/// 以免拖慢单测(单测仍验证退避时长的计算,只是不真正 sleep)。
|
||||
#[cfg(not(test))]
|
||||
const RETRY_BACKOFF_BASE_MS: u64 = 200;
|
||||
#[cfg(test)]
|
||||
const RETRY_BACKOFF_BASE_MS: u64 = 0;
|
||||
|
||||
/// 退避上限(毫秒)。
|
||||
const RETRY_BACKOFF_MAX_MS: u64 = 5_000;
|
||||
|
||||
/// 计算第 `attempt` 次失败后、下次重试前的退避时长。
|
||||
fn retry_backoff(busy: bool, attempt: usize) -> std::time::Duration {
|
||||
std::time::Duration::from_millis(backoff_delay_ms(RETRY_BACKOFF_BASE_MS, busy, attempt))
|
||||
}
|
||||
|
||||
/// 退避时长(毫秒)的纯计算,便于独立于 cfg 门控的基值做单测。
|
||||
///
|
||||
/// - `ETXTBSY`(fork/exec 竞态):极短固定退避,只为让兄弟进程完成 execve。
|
||||
/// - 其余网络类可重试失败:指数退避(`base·2^(attempt-1)`,上限 5s),并发下载
|
||||
/// 时对官方 CDN 更礼貌,避免 N 个连接失败后同时立即重发。
|
||||
fn backoff_delay_ms(base: u64, busy: bool, attempt: usize) -> u64 {
|
||||
if busy {
|
||||
return 5 * attempt as u64;
|
||||
}
|
||||
let shift = attempt.saturating_sub(1).min(5) as u32;
|
||||
base.saturating_mul(1u64 << shift).min(RETRY_BACKOFF_MAX_MS)
|
||||
}
|
||||
|
||||
pub(crate) fn run_curl_with_retry_with_proxy(
|
||||
curl_command: &Path,
|
||||
url: &str,
|
||||
@@ -448,8 +476,6 @@ pub(crate) fn run_curl_with_retry_with_proxy(
|
||||
Err(error) => CurlFailure::process_failed(url, destination, curl_command, error),
|
||||
};
|
||||
let retryable = failure.retryable();
|
||||
// ETXTBSY 是 fork/exec 竞态窗口造成的瞬时忙,立刻重试往往仍落在同一窗口内。
|
||||
// 让出 CPU 并做一次极短退避,使持有可写句柄的兄弟进程完成其 execve。
|
||||
let busy = failure.kind == CurlFailureKind::ProcessBusy;
|
||||
failures.push(CurlAttemptFailure {
|
||||
attempt,
|
||||
@@ -459,8 +485,8 @@ pub(crate) fn run_curl_with_retry_with_proxy(
|
||||
if !retryable {
|
||||
break;
|
||||
}
|
||||
if busy && attempt < attempts {
|
||||
std::thread::sleep(std::time::Duration::from_millis(5 * attempt as u64));
|
||||
if attempt < attempts {
|
||||
std::thread::sleep(retry_backoff(busy, attempt));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -838,4 +864,18 @@ mod tests {
|
||||
assert_eq!(command_env(&command, "ALL_PROXY"), Some(None));
|
||||
assert_eq!(command_env(&command, "HTTPS_PROXY"), Some(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_is_exponential_capped_and_busy_is_short() {
|
||||
// 网络类失败:base·2^(attempt-1),上限 5s。
|
||||
assert_eq!(backoff_delay_ms(200, false, 1), 200);
|
||||
assert_eq!(backoff_delay_ms(200, false, 2), 400);
|
||||
assert_eq!(backoff_delay_ms(200, false, 3), 800);
|
||||
assert_eq!(backoff_delay_ms(200, false, 4), 1600);
|
||||
// 高次方触顶 5s 上限。
|
||||
assert_eq!(backoff_delay_ms(200, false, 10), 5000);
|
||||
// ETXTBSY 走极短固定退避,与指数无关。
|
||||
assert_eq!(backoff_delay_ms(200, true, 1), 5);
|
||||
assert_eq!(backoff_delay_ms(200, true, 3), 15);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user