nyaKazuha/BlueArchiveToolkit
1
0
Fork 0
mirror of https://github.com/Yuyi-Oak/BlueArchiveToolkit.git synced 2026-09-18 10:04:55 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat(download): 多线程下载 + 重试指数退避（issue #17）

Browse Source 
下载主循环由串行改为「并发下载 + 串行簿记」三段式：
- Phase A：无网络前置校验（官方性/目标路径/建目录）+ 判定跳过/需下载，
  非官方 URL 在任何下载前 fail-fast
- Phase B：need-download 项经 scoped 线程池并发下载（默认并发 4，
  可配 1..=256）。worker 只做只读 &self 的 pull_one（各 URL 独立
  .part/目标文件），经 mpsc 把结果送回主线程；manifest/quarantine
  簿记与进度回调全在主线程串行执行，无需加锁。首个失败或 should_cancel
  置 cancel 标志，其余 worker 在任务边界停止
- Phase C：按 plan 顺序串行收尾——seed .hash 校验（顺序相关、可
  fail-fast）+ 构建有序结果

fail-fast 与「不发布不完整资源」不变量保留；进度事件按 URL 配对但
顺序不再单调（并发下天然如此）。

curl 重试加指数退避（网络类 200ms→400ms→800ms…上限 5s；ETXTBSY 仍走
极短退避），并发下对官方 CDN 更礼貌；退避基值 cfg(test) 下为 0 不拖慢
单测。

并发度经 OfficialUpdateConfig.download_concurrency 贯通，CLI
--download-concurrency 与 BAT_DOWNLOAD_CONCURRENCY 可配，.env 模板
与 USERGUIDE/CURRENT_STATUS/CHANGELOG 同步。

验证：新增并发正确性测试（并发 8：每 URL 恰一次 started+finished、
全部落盘）、并发度钳制、退避时长计算、CLI/env 解析单测；progress
排序测试改为顺序无关不变量；workspace 全测试(20 套件) + fmt +
clippy --all-targets -D warnings 全绿。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
nyaKazuha
2026-07-18 03:25:04 -07:00
co-authored by Claude Fable 5
parent 8efd8f36b4
commit 0ab3f3b953
7 changed files with 449 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infrastructure/src/official_download.rs
+318 -66
View File
@@ -16,6 +16,9 @@ use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc;
use std::thread;
use std::time::{SystemTime, UNIX_EPOCH};
/// 官方资源下载错误：携带统一错误码，便于 CLI/RPC 归类。
@@ -61,6 +64,10 @@ const DOWNLOAD_QUARANTINE_FILE: &str = "official-download-quarantine.json";
const DOWNLOAD_MANIFEST_VERSION: u32 = 1;
const DOWNLOAD_QUARANTINE_VERSION: u32 = 1;
const DEFAULT_RETRY_ATTEMPTS: usize = 3;
/// 默认下载并发度。保守取值，对官方 CDN 礼貌；可经配置调到 1..=256。
pub(crate) const DEFAULT_DOWNLOAD_CONCURRENCY: usize = 4;
/// 下载并发度上限。
const MAX_DOWNLOAD_CONCURRENCY: usize = 256;
/// Outcome for one official resource pull item.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -481,6 +488,7 @@ pub struct OfficialResourcePullService {
curl_command: PathBuf,
curl_proxy: CurlProxyConfig,
retry_attempts: usize,
download_concurrency: usize,
}
impl OfficialResourcePullService {
@@ -499,6 +507,7 @@ impl OfficialResourcePullService {
curl_command: curl_command.into(),
curl_proxy: CurlProxyConfig::default(),
retry_attempts: DEFAULT_RETRY_ATTEMPTS,
download_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY,
}
}
@@ -514,6 +523,17 @@ impl OfficialResourcePullService {
self
}
/// 设置下载并发度（并行执行的网络下载数），钳制到 `1..=256`。
pub fn with_download_concurrency(mut self, concurrency: usize) -> Self {
self.download_concurrency = concurrency.clamp(1, MAX_DOWNLOAD_CONCURRENCY);
self
}
/// 返回当前下载并发度。
pub fn download_concurrency(&self) -> usize {
self.download_concurrency
}
/// Returns the output root used for downloaded files.
pub fn output_root(&self) -> &Path {
&self.output_root
@@ -562,29 +582,21 @@ impl OfficialResourcePullService {
let mut manifest = self.read_download_manifest()?;
let urls = plan.all_urls()?;
let total = urls.len();
let mut items = Vec::new();
let mut verified_hashes = Vec::new();
let mut verified_hash_urls = HashSet::<String>::new();
let mut processed_urls = HashSet::<String>::new();
for (offset, url) in urls.into_iter().enumerate() {
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let index = offset + 1;
progress(OfficialResourcePullProgress::started(
index,
total,
url.clone(),
));
// Phase A：无网络的前置校验（fail-fast）。逐个校验官方性、算目标路径、
// 建目录，并判定该 URL 是「已验证可跳过」还是「需下载」。任何非官方
// URL 在发起任何下载前即拒绝。
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let mut planned: Vec<PlannedDownload> = Vec::with_capacity(total);
for url in urls {
if !is_official_yostar_jp_url(&url) {
return Err(DownloadError::new(
bat_core::ErrorCode::NON_OFFICIAL_URL,
format!("拒绝下载非官方 URL：{url}"),
));
}
let destination = self.destination_for_url(&url)?;
if let Some(parent) = destination.parent() {
ensure_safe_directory_path(parent, "下载目标目录")?;
@@ -596,50 +608,193 @@ impl OfficialResourcePullService {
ensure_safe_file_target(&self.output_root, &destination, "下载目标文件")?;
let force_refresh = force_refresh_urls.contains(&url);
let result = if force_refresh {
let existing = if force_refresh {
None
} else {
self.validated_existing_file(&url, &destination, &manifest)?
};
let result = if let Some(result) = result {
self.clear_quarantine_entry(&url)?;
result
} else {
let result = match self.pull_one(&url, &destination) {
Ok(result) => result,
Err(error) => {
self.record_quarantine_entry(&url, &destination, &error)?;
progress(OfficialResourcePullProgress::failed(
index,
total,
url.clone(),
&error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={url} quarantine={}；{}",
self.download_quarantine_path().display(),
error.message
),
));
planned.push(PlannedDownload {
url,
destination,
existing,
});
}
// Phase B：并发下载 need-download 项（各 URL 目标/`.part` 相互独立，
// 天然可并行）。worker 只做只读 `&self` 的网络下载，经 mpsc 把结果送回
// 主线程；manifest/quarantine 簿记与进度回调全部在主线程串行完成，无需
// 加锁。首个失败或 `should_cancel` 会置 cancel 标志，其余 worker 在下一
// 个任务边界停止，保持 fail-fast 与「不发布不完整资源」不变量。
let download_indices: Vec<usize> = planned
.iter()
.enumerate()
.filter(|(_, item)| item.existing.is_none())
.map(|(index, _)| index)
.collect();
let mut download_results: Vec<Option<Result<PullOneResult, PullOneError>>> =
(0..planned.len()).map(|_| None).collect();
if !download_indices.is_empty() {
let concurrency = self.download_concurrency.clamp(1, MAX_DOWNLOAD_CONCURRENCY);
let cursor = AtomicUsize::new(0);
let cancel = AtomicBool::new(false);
let (sender, receiver) = mpsc::channel::<WorkerMessage>();
thread::scope(|scope| {
for _ in 0..concurrency.min(download_indices.len()) {
let sender = sender.clone();
let cursor = &cursor;
let cancel = &cancel;
let download_indices = &download_indices;
let planned = &planned;
let service = &*self;
scope.spawn(move || loop {
if cancel.load(Ordering::Relaxed) {
break;
}
let slot = cursor.fetch_add(1, Ordering::Relaxed);
let Some(&plan_index) = download_indices.get(slot) else {
break;
};
let item = &planned[plan_index];
if sender.send(WorkerMessage::Started { plan_index }).is_err() {
break;
}
let result = service.pull_one(&item.url, &item.destination);
if result.is_err() {
cancel.store(true, Ordering::Relaxed);
}
if sender
.send(WorkerMessage::Done { plan_index, result })
.is_err()
{
break;
}
});
}
// 主线程持有的 sender 副本必须丢弃，否则 receiver 永不结束。
drop(sender);
for message in receiver {
match message {
WorkerMessage::Started { plan_index } => {
let item = &planned[plan_index];
progress(OfficialResourcePullProgress::started(
plan_index + 1,
total,
item.url.clone(),
));
// 停止请求：置 cancel，让 worker 在下个任务边界退出。
if should_cancel() {
cancel.store(true, Ordering::Relaxed);
}
}
WorkerMessage::Done { plan_index, result } => {
if let Ok(pull_result) = &result {
let item = &planned[plan_index];
// manifest/quarantine 簿记在主线程串行执行。
if let Err(error) = self
.clear_quarantine_entry(&item.url)
.and_then(|_| {
self.record_download_manifest_entry(
&mut manifest,
&item.url,
&item.destination,
)
})
.and_then(|_| self.write_download_manifest(&manifest))
{
// 簿记失败：记为该项错误并触发 fail-fast。
cancel.store(true, Ordering::Relaxed);
download_results[plan_index] = Some(Err(PullOneError::plain(
format!("记录下载 manifest 失败：{error}"),
)));
continue;
}
progress(OfficialResourcePullProgress::finished(
plan_index + 1,
total,
item.url.clone(),
pull_result.status,
pull_result.bytes,
pull_result.transferred_bytes,
));
}
download_results[plan_index] = Some(result);
}
}
};
self.clear_quarantine_entry(&url)?;
self.record_download_manifest_entry(&mut manifest, &url, &destination)?;
self.write_download_manifest(&manifest)?;
result
}
});
if should_cancel() && download_results.iter().any(Option::is_none) {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
// fail-fast：按 plan 顺序取首个失败项，记 quarantine、发 failed 进度并中止。
for &plan_index in &download_indices {
if let Some(Some(Err(error))) = download_results.get(plan_index) {
let item = &planned[plan_index];
self.record_quarantine_entry(&item.url, &item.destination, error)?;
progress(OfficialResourcePullProgress::failed(
plan_index + 1,
total,
item.url.clone(),
error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
}
}
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
// Phase C：按 plan 顺序串行收尾——跳过项清 quarantine 并补发进度，
// 逐项做官方 seed `.hash` 校验（顺序相关、可 fail-fast），构建有序结果。
let mut items = Vec::with_capacity(planned.len());
let mut verified_hashes = Vec::new();
let mut verified_hash_urls = HashSet::<String>::new();
let mut processed_urls = HashSet::<String>::new();
for (plan_index, item) in planned.iter().enumerate() {
let result = if let Some(existing) = &item.existing {
self.clear_quarantine_entry(&item.url)?;
progress(OfficialResourcePullProgress::started(
plan_index + 1,
total,
item.url.clone(),
));
progress(OfficialResourcePullProgress::finished(
plan_index + 1,
total,
item.url.clone(),
existing.status,
existing.bytes,
existing.transferred_bytes,
));
*existing
} else {
// Phase B 已保证需下载项此时均为 Ok（失败会在上面 fail-fast 返回）。
match download_results[plan_index].take() {
Some(Ok(result)) => result,
_ => {
return Err(DownloadError::new(
bat_core::ErrorCode::INTERNAL,
format!("内部错误：下载结果缺失 url={}", item.url),
))
}
}
};
progress(OfficialResourcePullProgress::finished(
index,
total,
url.clone(),
result.status,
result.bytes,
result.transferred_bytes,
));
processed_urls.insert(url.clone());
processed_urls.insert(item.url.clone());
self.verify_ready_official_hashes(
&official_hash_pairs,
&processed_urls,
@@ -648,17 +803,14 @@ impl OfficialResourcePullService {
&mut manifest,
)?;
items.push(OfficialResourcePullItem {
url,
destination,
url: item.url.clone(),
destination: item.destination.clone(),
bytes: result.bytes,
transferred_bytes: result.transferred_bytes,
status: result.status,
});
}
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?;
Ok(OfficialResourcePullReport {
@@ -1603,6 +1755,25 @@ fn default_download_quarantine_version() -> u32 {
DOWNLOAD_QUARANTINE_VERSION
}
/// Phase A 产出的单个下载计划项：URL、目标路径，以及若命中本地 manifest
/// 校验则带上「已验证可跳过」的结果（`existing`）。
struct PlannedDownload {
url: String,
destination: PathBuf,
existing: Option<PullOneResult>,
}
/// worker 线程经 mpsc 送回主线程的消息。
enum WorkerMessage {
/// worker 已领取某计划项、即将下载（主线程据此发 started 进度）。
Started { plan_index: usize },
/// 某计划项下载结束（成功或失败）。
Done {
plan_index: usize,
result: Result<PullOneResult, PullOneError>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct PullOneResult {
bytes: u64,
@@ -2853,6 +3024,8 @@ exit 22
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
// 并发下事件顺序不固定，此处只断言与顺序无关的不变量；并发正确性另有
// downloads_run_concurrently_and_each_url_reports_once 专测。
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
@@ -2868,19 +3041,98 @@ exit 22
assert_eq!(report.items.len(), expected_urls);
assert_eq!(events.len(), expected_urls * 2);
assert_eq!(events[0].kind, OfficialResourcePullProgressKind::Started);
assert_eq!(events[0].index, 1);
assert_eq!(events[0].total, expected_urls);
assert_eq!(events[1].kind, OfficialResourcePullProgressKind::Finished);
assert_eq!(
events[1].status,
Some(OfficialResourcePullStatus::Downloaded)
);
let started: Vec<_> = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Started)
.collect();
let finished: Vec<_> = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
.collect();
assert_eq!(started.len(), expected_urls);
assert_eq!(finished.len(), expected_urls);
// 每个 started 的 total 一致；index 覆盖 1..=N 且不重复。
let mut indices: Vec<usize> = started.iter().map(|event| event.index).collect();
indices.sort_unstable();
assert_eq!(indices, (1..=expected_urls).collect::<Vec<_>>());
assert!(started.iter().all(|event| event.total == expected_urls));
// started 与 finished 覆盖同一组 URL；finished 均为已下载。
let started_urls: HashSet<_> = started.iter().map(|event| event.url.clone()).collect();
let finished_urls: HashSet<_> = finished.iter().map(|event| event.url.clone()).collect();
assert_eq!(started_urls, finished_urls);
assert!(finished
.iter()
.all(|event| event.status == Some(OfficialResourcePullStatus::Downloaded)));
assert!(events
.iter()
.any(|event| event.url.ends_with("/TableBundles/ExcelDB.db")));
}
#[test]
fn downloads_run_concurrently_and_each_url_reports_once() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
// 并发度 8：每个 URL 恰好一次 started + 一次 finished，全部文件落盘。
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_download_concurrency(8);
assert_eq!(service.download_concurrency(), 8);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let all_urls = plan.all_urls().unwrap();
let mut events = Vec::new();
let report = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap();
assert_eq!(report.items.len(), all_urls.len());
for url in &all_urls {
let started = events
.iter()
.filter(|event| {
&event.url == url && event.kind == OfficialResourcePullProgressKind::Started
})
.count();
let finished = events
.iter()
.filter(|event| {
&event.url == url && event.kind == OfficialResourcePullProgressKind::Finished
})
.count();
assert_eq!(started, 1, "url {url} 的 started 次数");
assert_eq!(finished, 1, "url {url} 的 finished 次数");
}
// 所有下载都记入 manifest。
let manifest = service.read_download_manifest().unwrap();
assert_eq!(manifest.entries.len(), all_urls.len());
}
#[test]
fn download_concurrency_is_clamped() {
let service = OfficialResourcePullService::with_curl_command("/tmp/unused", "curl");
assert_eq!(service.download_concurrency(), DEFAULT_DOWNLOAD_CONCURRENCY);
assert_eq!(
OfficialResourcePullService::with_curl_command("/tmp/unused", "curl")
.with_download_concurrency(0)
.download_concurrency(),
1
);
assert_eq!(
OfficialResourcePullService::with_curl_command("/tmp/unused", "curl")
.with_download_concurrency(9999)
.download_concurrency(),
MAX_DOWNLOAD_CONCURRENCY
);
}
#[test]
fn retries_transient_download_failures() {
let out_dir = TempDir::new().unwrap();