nyaKazuha/BlueArchiveToolkit
1
0
Fork 0
mirror of https://github.com/Yuyi-Oak/BlueArchiveToolkit.git synced 2026-09-18 10:04:55 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: 完成下载并发与翻译交接链路

Browse Source
This commit is contained in:
nyaKazuha
2026-08-02 22:34:23 +08:00
parent d533c88108
commit 8b64cc94f3
19 changed files with 2923 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infrastructure/src/official_download.rs
+317 -53
View File
@@ -1,6 +1,10 @@
//! Official JP resource download execution.
use crate::curl_transfer::{run_curl_with_retry_with_proxy, CurlProxyConfig, CurlRetryError};
use crate::downloader::{
DownloadScheduler, DownloaderBackend, DEFAULT_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY,
MIN_DOWNLOAD_CONCURRENCY,
};
use crate::official_pull::OfficialResourcePullPlan;
use crate::path_security::{
ensure_path_within_root, ensure_safe_directory_path, ensure_safe_file_target,
@@ -9,7 +13,10 @@ use crate::path_security::{
use crate::zip_validation::{
path_has_zip_extension, url_or_path_has_zip_extension, validate_zip_structure,
};
use bat_adapters::official::yostar_jp::{is_official_yostar_jp_url, YostarJpResourceEndpointKind};
use bat_adapters::official::yostar_jp::YostarJpResourceEndpointKind;
use bat_adapters::official::{
destination_under_root, DownloadUrlMapper, OfficialResourceBackend, YostarJpBackend,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashSet};
use std::fs::{self, File};
@@ -539,9 +546,11 @@ impl OfficialLocalResourceState {
#[derive(Debug, Clone)]
pub struct OfficialResourcePullService {
output_root: PathBuf,
backend: YostarJpBackend,
curl_command: PathBuf,
curl_proxy: CurlProxyConfig,
retry_attempts: usize,
max_concurrency: usize,
}
impl OfficialResourcePullService {
@@ -557,9 +566,11 @@ impl OfficialResourcePullService {
) -> Self {
Self {
output_root: output_root.into(),
backend: YostarJpBackend,
curl_command: curl_command.into(),
curl_proxy: CurlProxyConfig::default(),
retry_attempts: DEFAULT_RETRY_ATTEMPTS,
max_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY,
}
}
@@ -575,6 +586,22 @@ impl OfficialResourcePullService {
self
}
/// Sets the bounded number of concurrent downloads.
///
/// The default is [`DEFAULT_DOWNLOAD_CONCURRENCY`]. Values are kept
/// within the supported `1..=256` range; the public update configuration
/// validates input before constructing this service.
pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self {
self.max_concurrency =
max_concurrency.clamp(MIN_DOWNLOAD_CONCURRENCY, MAX_DOWNLOAD_CONCURRENCY);
self
}
/// Returns the configured download concurrency.
pub fn max_concurrency(&self) -> usize {
self.max_concurrency
}
/// Returns the output root used for downloaded files.
pub fn output_root(&self) -> &Path {
&self.output_root
@@ -666,7 +693,7 @@ impl OfficialResourcePullService {
}
let mut planned: Vec<PlannedDownload> = Vec::with_capacity(total);
for url in urls {
if !is_official_yostar_jp_url(&url) {
if !self.backend.is_official_url(&url) {
return Err(DownloadError::new(
bat_core::ErrorCode::NON_OFFICIAL_URL,
format!("拒绝下载非官方 URL：{url}"),
@@ -695,6 +722,17 @@ impl OfficialResourcePullService {
});
}
if self.max_concurrency > 1 {
return self.pull_planned_concurrently(
planned,
manifest,
official_hash_pairs,
total,
&mut progress,
&mut should_cancel,
);
}
// Phase B：按 plan 顺序处理每个 URL。下载或复用完成并写入 manifest 后，
// 立即尝试校验已经到齐的官方 `.bytes/.hash` pair，避免把文件级问题延后到整轮末尾。
let mut completed_count = 0usize;
@@ -824,6 +862,183 @@ impl OfficialResourcePullService {
})
}
fn pull_planned_concurrently(
&self,
planned: Vec<PlannedDownload>,
mut manifest: OfficialDownloadManifest,
official_hash_pairs: Vec<OfficialSeedHashPair>,
total: usize,
progress: &mut impl FnMut(OfficialResourcePullProgress),
should_cancel: &mut impl FnMut() -> bool,
) -> Result<OfficialResourcePullReport, DownloadError> {
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
for item in &planned {
progress(OfficialResourcePullProgress::started(
0,
total,
item.url.clone(),
));
}
let backend = CurlDownloadBackend { service: self };
let mut completed_count = 0usize;
let mut items_by_plan_index: Vec<Option<OfficialResourcePullItem>> =
(0..total).map(|_| None).collect();
let mut verified_hashes = Vec::new();
let mut verified_hash_urls = HashSet::<String>::new();
let mut processed_urls = HashSet::<String>::new();
DownloadScheduler::new(self.max_concurrency).execute_with_observer(
&backend,
planned.clone(),
|plan_index, result| {
let item = &planned[plan_index];
if should_cancel() {
return Err("官方资源拉取已被停止请求中断".to_string().into());
}
let needs_manifest = item.existing.is_none();
let verification = match result {
Ok(pull_result) => {
let verification_result = self
.clear_quarantine_entry(&item.url)
.and_then(|_| {
if needs_manifest {
self.record_download_manifest_entry(
&mut manifest,
&item.url,
&item.destination,
)
} else {
Ok(pull_result.verification.clone())
}
})
.and_then(|verification| {
if needs_manifest {
self.write_download_manifest(&manifest)
.map(|_| verification)
} else {
Ok(verification)
}
});
match verification_result {
Ok(verification) => verification,
Err(error) => {
let error = PullOneError::plain(format!(
"记录下载 manifest 失败：{error}"
));
self.record_quarantine_entry(
&item.url,
&item.destination,
&error,
)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
&error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
}
}
Err(error) => {
self.record_quarantine_entry(&item.url, &item.destination, error)?;
progress(OfficialResourcePullProgress::failed(
completed_count,
total,
item.url.clone(),
error,
));
return Err(DownloadError::new(
error.error_code(),
format!(
"官方资源下载失败：URL 已进入 quarantine，中止本轮同步、不发布不完整资源；url={} quarantine={}；{}",
item.url,
self.download_quarantine_path().display(),
error.message
),
));
}
};
completed_count += 1;
progress(OfficialResourcePullProgress::finished(
completed_count,
total,
item.url.clone(),
result
.as_ref()
.expect("successful result handled above")
.status,
result
.as_ref()
.expect("successful result handled above")
.bytes,
result
.as_ref()
.expect("successful result handled above")
.transferred_bytes,
verification,
));
processed_urls.insert(item.url.clone());
let newly_verified_hashes = self.verify_ready_official_hashes(
&official_hash_pairs,
&processed_urls,
&mut verified_hash_urls,
&mut verified_hashes,
&mut manifest,
)?;
for verification in newly_verified_hashes {
progress(OfficialResourcePullProgress::verification(
completed_count,
total,
verification.data_url.clone(),
verification,
));
}
let pull_result = result
.as_ref()
.expect("successful result handled above");
items_by_plan_index[plan_index] = Some(OfficialResourcePullItem {
url: item.url.clone(),
destination: item.destination.clone(),
bytes: pull_result.bytes,
transferred_bytes: pull_result.transferred_bytes,
status: pull_result.status,
});
Ok(())
},
)?;
self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?;
let items = items_by_plan_index
.into_iter()
.enumerate()
.map(|(index, item)| {
item.ok_or_else(|| {
DownloadError::from(format!(
"并发下载结果缺少 plan index={index}，拒绝发布不完整资源"
))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(OfficialResourcePullReport {
items,
verified_hashes,
})
}
/// Audits every URL in a pull plan against the local download manifest.
///
/// This performs no network I/O. It checks that each URL has a manifest
@@ -914,7 +1129,7 @@ impl OfficialResourcePullService {
/// `TableCatalog.bytes`, `BundlePackingInfo.bytes`, and
/// `MediaCatalog.bytes`.
pub fn fetch_bytes(&self, url: &str) -> Result<Vec<u8>, DownloadError> {
if !is_official_yostar_jp_url(url) {
if !self.backend.is_official_url(url) {
return Err(DownloadError::new(
bat_core::ErrorCode::NON_OFFICIAL_URL,
format!("拒绝拉取非官方 URL：{url}"),
@@ -1581,36 +1796,12 @@ impl OfficialResourcePullService {
fn destination_for_url(&self, url: &str) -> Result<PathBuf, String> {
self.ensure_output_root_safe()?;
if !is_official_yostar_jp_url(url) {
if !self.backend.is_official_url(url) {
return Err(format!("URL 不是官方 JP host：{url}"));
}
let rest = url
.strip_prefix("https://")
.ok_or_else(|| format!("官方 URL 必须使用 https：{url}"))?;
let (host, path) = rest
.split_once('/')
.ok_or_else(|| format!("官方 URL 缺少路径：{url}"))?;
let mut relative_destination = PathBuf::from(sanitize_segment(host));
for segment in path.split('/') {
if segment.is_empty() {
continue;
}
if segment == "." || segment == ".." {
return Err(format!("官方 URL 包含不安全路径片段：{url}"));
}
// 官方资源 URL 不携带 query/fragment。若出现则直接拒绝，而非静默剥除——
// 否则仅 query 不同的两个 URL 会映射到同一目标文件而相互覆盖，
// 并导致每轮 hash 复用校验失配、反复重下。
if segment.contains('?') || segment.contains('#') {
return Err(format!("官方资源 URL 不允许包含 query 或 fragment：{url}"));
}
relative_destination.push(sanitize_segment(segment));
}
let destination = self.output_root.join(relative_destination);
let relative_destination = self.backend.relative_destination(url)?;
let destination = destination_under_root(&self.output_root, &relative_destination)?;
ensure_path_within_root(&self.output_root, &destination)?;
Ok(destination)
}
@@ -1834,6 +2025,7 @@ fn default_download_quarantine_version() -> u32 {
/// Phase A 产出的单个下载计划项：URL、目标路径，以及若命中本地 manifest
/// 校验则带上「已验证可跳过」的结果（`existing`）。
#[derive(Debug, Clone)]
struct PlannedDownload {
url: String,
destination: PathBuf,
@@ -1854,6 +2046,21 @@ struct PullOneError {
retry_error: Option<CurlRetryError>,
}
struct CurlDownloadBackend<'a> {
service: &'a OfficialResourcePullService,
}
impl DownloaderBackend<PlannedDownload> for CurlDownloadBackend<'_> {
type Output = PullOneResult;
type Error = PullOneError;
fn download(&self, task: PlannedDownload) -> Result<Self::Output, Self::Error> {
task.existing
.map(Ok)
.unwrap_or_else(|| self.service.pull_one(&task.url, &task.destination))
}
}
impl PullOneError {
fn plain(message: String) -> Self {
Self {
@@ -2088,20 +2295,6 @@ fn read_u32_le(bytes: &[u8], offset: usize) -> u32 {
])
}
fn sanitize_segment(segment: &str) -> String {
segment
.chars()
.map(|ch| {
if ch.is_control() || matches!(ch, '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|')
{
'_'
} else {
ch
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
@@ -3263,21 +3456,26 @@ exit 22
.official_hash
.as_ref()
.expect("verification event must carry official hash detail");
let hash_finished_index = events
.iter()
.position(|event| {
event.kind == OfficialResourcePullProgressKind::Finished
&& event.url == hash.hash_url
let pair_finished_indices = [hash.data_url.as_str(), hash.hash_url.as_str()]
.into_iter()
.map(|url| {
events
.iter()
.position(|event| {
event.kind == OfficialResourcePullProgressKind::Finished
&& event.url == url
})
.expect("hash pair member must finish before verification")
})
.expect("hash sidecar must finish before verification");
.collect::<Vec<_>>();
let verification_index = events
.iter()
.position(|event| std::ptr::eq(event, *verification_event))
.expect("verification event must be present in event stream");
assert_eq!(
verification_index,
hash_finished_index + 1,
"official hash verification must run immediately after sidecar is complete"
pair_finished_indices.into_iter().max().unwrap() + 1,
"official hash verification must run immediately after the hash pair is complete"
);
}
@@ -3320,7 +3518,8 @@ exit 22
write_fake_curl(&curl_path);
// 顺序下载：每个 URL 恰好一次 started + 一次 finished，全部文件落盘。
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_max_concurrency(1);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
@@ -3354,6 +3553,71 @@ exit 22
assert_eq!(manifest.entries.len(), all_urls.len());
}
#[test]
fn downloads_run_with_bounded_concurrency_and_keep_report_order() {
let out_dir = TempDir::new().unwrap();
let bin_dir = TempDir::new().unwrap();
let curl_path = bin_dir.path().join("curl");
write_fake_curl(&curl_path);
let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
.with_max_concurrency(3);
let plan = build_official_pull_plan_for_platforms(
discovery_plan(),
inventory(),
&[PatchPlatform::Windows],
);
let all_urls = plan.all_urls().unwrap();
let mut events = Vec::new();
let report = service
.pull_with_progress(&plan, |event| events.push(event))
.unwrap();
assert_eq!(service.max_concurrency(), 3);
assert_eq!(report.items.len(), all_urls.len());
assert_eq!(
report
.items
.iter()
.map(|item| &item.url)
.collect::<Vec<_>>(),
all_urls.iter().collect::<Vec<_>>()
);
for url in &all_urls {
assert_eq!(
events
.iter()
.filter(|event| {
event.url == *url && event.kind == OfficialResourcePullProgressKind::Started
})
.count(),
1,
"url {url} 的 started 次数"
);
assert_eq!(
events
.iter()
.filter(|event| {
event.url == *url
&& event.kind == OfficialResourcePullProgressKind::Finished
})
.count(),
1,
"url {url} 的 finished 次数"
);
}
let finished_indices = events
.iter()
.filter(|event| event.kind == OfficialResourcePullProgressKind::Finished)
.map(|event| event.index)
.collect::<Vec<_>>();
assert_eq!(finished_indices, (1..=all_urls.len()).collect::<Vec<_>>());
assert_eq!(
service.read_download_manifest().unwrap().entries.len(),
all_urls.len()
);
}
#[test]
fn retries_transient_download_failures() {
let out_dir = TempDir::new().unwrap();