fix(sync): 修复官方资源并发下载与媒体路径
bat-rust / Build and test Rust (push) Successful in 3m47s

daemon 子进程现会透传下载并发配置,下载进度按已完成数量单调上报。MediaCatalog 改为使用官方相对路径生成全局媒体 URL,覆盖 GameData、Prologue 等目录并补齐 jpg 资源,避免叶子文件名误拼媒体根目录导致 403。同步更新测试、smoke 断言和运行文档。
This commit is contained in:
2026-07-23 11:56:34 +08:00
parent d694100d6c
commit 3f5d2a8da7
16 changed files with 182 additions and 74 deletions
+111 -23
View File
@@ -10,7 +10,7 @@ pub struct YostarJpDownloadInventory {
pub bundle_patch_pack_names: Vec<String>,
/// Table file names from `TableCatalog.bytes`.
pub table_file_names: Vec<String>,
/// Media file names from `MediaCatalog.bytes`.
/// Media file relative paths from `MediaCatalog.bytes`.
pub media_file_names: Vec<String>,
}
@@ -21,7 +21,7 @@ pub struct YostarJpPlatformCatalogInventory {
pub platform: PatchPlatform,
/// Patch-pack zip names from this platform's `BundlePackingInfo.bytes`.
pub bundle_patch_pack_names: Vec<String>,
/// Media file names from this platform's `MediaCatalog.bytes`.
/// Media file relative paths from this platform's `MediaCatalog.bytes`.
pub media_file_names: Vec<String>,
}
@@ -35,10 +35,7 @@ impl YostarJpPlatformCatalogInventory {
Self {
platform,
bundle_patch_pack_names: extract_full_patch_pack_names(bundle_packing_info),
media_file_names: extract_file_names(
media_catalog,
&["zip", "mp4", "png", "ogg", "wav"],
),
media_file_names: extract_media_file_paths(media_catalog),
}
}
@@ -172,10 +169,7 @@ impl YostarJpDownloadInventory {
Self {
bundle_patch_pack_names: extract_full_patch_pack_names(bundle_packing_info),
table_file_names: extract_table_file_names(table_catalog),
media_file_names: extract_file_names(
media_catalog,
&["zip", "mp4", "png", "ogg", "wav"],
),
media_file_names: extract_media_file_paths(media_catalog),
}
}
@@ -327,6 +321,20 @@ fn extract_file_names(data: &[u8], extensions: &[&str]) -> Vec<String> {
names.into_iter().collect()
}
fn extract_media_file_paths(data: &[u8]) -> Vec<String> {
let mut paths = BTreeSet::new();
for string in extract_printable_strings(data, 4) {
for path in
candidate_relative_paths(&string, &["zip", "mp4", "png", "jpg", "jpeg", "ogg", "wav"])
{
paths.insert(path);
}
}
paths.into_iter().collect()
}
fn extract_printable_strings(data: &[u8], min_len: usize) -> Vec<String> {
let mut strings = Vec::new();
let mut current = Vec::new();
@@ -375,6 +383,32 @@ fn candidate_file_names(value: &str, extensions: &[&str]) -> Vec<String> {
names
}
fn candidate_relative_paths(value: &str, extensions: &[&str]) -> Vec<String> {
let mut paths = Vec::new();
let bytes = value.as_bytes();
for extension in extensions {
let suffix = format!(".{extension}");
let mut search_from = 0;
while let Some(relative_index) = value[search_from..].find(&suffix) {
let extension_start = search_from + relative_index;
let start = filename_start(bytes, extension_start);
let end = extension_start + suffix.len();
let candidate = &value[start..end];
let candidate = candidate.replace('\\', "/");
if is_plausible_relative_path(&candidate) {
paths.push(candidate);
}
search_from = end;
}
}
paths
}
fn filename_start(bytes: &[u8], mut index: usize) -> usize {
while index > 0 {
let byte = bytes[index - 1];
@@ -409,6 +443,28 @@ fn is_plausible_file_name(name: &str) -> bool {
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
}
fn is_plausible_relative_path(path: &str) -> bool {
if path.is_empty()
|| path.starts_with('/')
|| path.starts_with('.')
|| path.contains("..")
|| path.contains(':')
|| path.contains('=')
|| !path.contains('/')
{
return false;
}
path.split('/').all(|segment| {
!segment.is_empty()
&& segment != "."
&& segment != ".."
&& segment
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
})
}
fn unique_platforms(platforms: &[PatchPlatform]) -> Vec<PatchPlatform> {
platforms
.iter()
@@ -441,7 +497,7 @@ mod tests {
let inventory = YostarJpDownloadInventory::from_catalog_bytes(
b"prefix FullPatch_000.zip noise FullPatch_114.zip suffix",
b"GameData\\Table\\ExcelDB.db\0ExcelDB.db\0rawdata/table/excel/ignored.bytes\0Battle.zip\0Battle.zip8",
b"audio/voc_jp/jp_airi/jp_airi\0GameData\\Audio\\VOC_JP\\JP_Airi.zip8\0JP_Akane.zip",
b"audio/voc_jp/jp_airi/jp_airi\0GameData\\Audio\\VOC_JP\\JP_Airi.zip8\0audio/voc_jp/jp_akane/jp_akane\0GameData\\Audio\\VOC_JP\\JP_Akane.zip",
);
assert_eq!(
@@ -457,7 +513,10 @@ mod tests {
);
assert_eq!(
inventory.media_file_names,
vec!["JP_Airi.zip".to_string(), "JP_Akane.zip".to_string(),]
vec![
"GameData/Audio/VOC_JP/JP_Airi.zip".to_string(),
"GameData/Audio/VOC_JP/JP_Akane.zip".to_string(),
]
);
}
@@ -482,7 +541,7 @@ mod tests {
let inventory = YostarJpDownloadInventory::from_catalog_bytes(
b"FullPatch_000.zip FullPatch_001.zip",
b"ExcelDB.db ExcelDB.db Battle.zip Battle.zip",
b"JP_Airi.zip JP_Akane.zip",
b"GameData\\Audio\\VOC_JP\\JP_Airi.zip GameData\\Audio\\VOC_JP\\JP_Akane.zip",
);
let root = YostarJpResourceRoot::from_root_token(ROOT).unwrap();
@@ -496,7 +555,7 @@ mod tests {
.any(|url| url.ends_with("/TableBundles/ExcelDB.db")));
assert!(urls
.iter()
.any(|url| url.ends_with("/MediaResources-Windows/JP_Airi.zip")));
.any(|url| url.ends_with("/MediaResources-Windows/GameData/Audio/VOC_JP/JP_Airi.zip")));
}
#[test]
@@ -504,7 +563,7 @@ mod tests {
let inventory = YostarJpDownloadInventory::from_catalog_bytes(
b"FullPatch_000.zip",
b"ExcelDB.db ExcelDB.db",
b"JP_Airi.zip",
b"GameData\\Audio\\VOC_JP\\JP_Airi.zip",
);
let root = YostarJpResourceRoot::from_root_token(ROOT).unwrap();
@@ -524,10 +583,10 @@ mod tests {
.any(|url| url.ends_with("/TableBundles/ExcelDB.db")));
assert!(urls
.iter()
.any(|url| url.ends_with("/MediaResources-Windows/JP_Airi.zip")));
.any(|url| url.ends_with("/MediaResources-Windows/GameData/Audio/VOC_JP/JP_Airi.zip")));
assert!(urls
.iter()
.any(|url| url.ends_with("/MediaResources/JP_Airi.zip")));
.any(|url| url.ends_with("/MediaResources/GameData/Audio/VOC_JP/JP_Airi.zip")));
}
#[test]
@@ -538,12 +597,12 @@ mod tests {
YostarJpPlatformCatalogInventory::from_catalog_bytes(
PatchPlatform::Windows,
b"FullPatch_000.zip",
b"JP_Airi_Win.zip",
b"GameData\\Audio\\VOC_JP\\JP_Airi_Win.zip",
),
YostarJpPlatformCatalogInventory::from_catalog_bytes(
PatchPlatform::Android,
b"FullPatch_001.zip",
b"JP_Airi_Android.zip",
b"GameData\\Audio\\VOC_JP\\JP_Airi_Android.zip",
),
],
);
@@ -561,10 +620,11 @@ mod tests {
.any(|url| url.ends_with("/Android_PatchPack/FullPatch_001.zip")));
assert!(urls
.iter()
.any(|url| url.ends_with("/MediaResources-Windows/JP_Airi_Win.zip")));
.any(|url| url
.ends_with("/MediaResources-Windows/GameData/Audio/VOC_JP/JP_Airi_Win.zip")));
assert!(urls
.iter()
.any(|url| url.ends_with("/MediaResources/JP_Airi_Android.zip")));
.any(|url| url.ends_with("/MediaResources/GameData/Audio/VOC_JP/JP_Airi_Android.zip")));
assert!(!urls
.iter()
.any(|url| url.ends_with("/Android_PatchPack/FullPatch_000.zip")));
@@ -573,6 +633,30 @@ mod tests {
.any(|url| url.ends_with("/Windows_PatchPack/FullPatch_001.zip")));
}
#[test]
fn media_catalog_uses_download_relative_path_not_leaf_name() {
let inventory = YostarJpPlatformDownloadInventory::from_catalog_bytes(
b"ExcelDB.db ExcelDB.db",
vec![YostarJpPlatformCatalogInventory::from_catalog_bytes(
PatchPlatform::Windows,
b"FullPatch_000.zip",
b"scenario/event/10000_title_sound\0Prologue\\Scenario\\Event\\10000_Title_Sound.ogg\0 10000_Title_Sound.ogg",
)],
);
let root = YostarJpResourceRoot::from_root_token(ROOT).unwrap();
let urls = inventory
.direct_download_urls_for_platforms(&root, &[PatchPlatform::Windows])
.unwrap();
assert!(urls.iter().any(|url| {
url.ends_with("/MediaResources-Windows/Prologue/Scenario/Event/10000_Title_Sound.ogg")
}));
assert!(!urls
.iter()
.any(|url| url.ends_with("/MediaResources-Windows/10000_Title_Sound.ogg")));
}
#[test]
#[ignore = "requires BAT_REAL_OFFICIAL_BUNDLE_PACKING_INFO, BAT_REAL_OFFICIAL_TABLE_CATALOG, BAT_REAL_OFFICIAL_MEDIA_CATALOG"]
fn extracts_realistic_counts_from_official_shape() {
@@ -591,7 +675,7 @@ mod tests {
assert_eq!(inventory.bundle_patch_pack_names.len(), 142);
assert!(inventory.table_file_names.len() < 1000);
assert_eq!(inventory.media_file_names.len(), 1887);
assert!(inventory.media_file_names.len() >= 4000);
assert!(inventory
.bundle_patch_pack_names
.iter()
@@ -603,6 +687,10 @@ mod tests {
assert!(inventory
.media_file_names
.iter()
.any(|name| name == "JP_Airi.zip"));
.any(|name| name == "GameData/Audio/VOC_JP/JP_Airi.zip"));
assert!(inventory
.media_file_names
.iter()
.any(|name| name.ends_with(".jpg")));
}
}
+6 -5
View File
@@ -646,8 +646,8 @@ impl YostarJpResourceRoot {
/// Returns an official media archive URL.
///
/// The argument is the `Media.FileName` field from `MediaCatalog.bytes`,
/// for example `JP_Airi.zip`.
/// The argument is the downloadable relative path from `MediaCatalog.bytes`,
/// for example `GameData/Audio/VOC_JP/JP_Airi.zip`.
pub fn media_file(&self, platform: PatchPlatform, file_name: &str) -> Result<String, String> {
validate_relative_path(file_name, "media file")?;
Ok(format!(
@@ -1134,8 +1134,9 @@ mod tests {
"https://prod-clientpatch.bluearchiveyostar.com/r93_dctuo3tcd029wwxnvb55/MediaResources-Windows/Catalog/MediaCatalog.bytes"
);
assert_eq!(
root.media_file(PatchPlatform::Windows, "JP_Airi.zip").unwrap(),
"https://prod-clientpatch.bluearchiveyostar.com/r93_dctuo3tcd029wwxnvb55/MediaResources-Windows/JP_Airi.zip"
root.media_file(PatchPlatform::Windows, "GameData/Audio/VOC_JP/JP_Airi.zip")
.unwrap(),
"https://prod-clientpatch.bluearchiveyostar.com/r93_dctuo3tcd029wwxnvb55/MediaResources-Windows/GameData/Audio/VOC_JP/JP_Airi.zip"
);
}
@@ -1211,7 +1212,7 @@ mod tests {
);
assert!(root.table_bundle("text=jp/ExcelDB.db").is_err());
assert!(root
.media_file(PatchPlatform::Windows, "/JP_Airi.zip")
.media_file(PatchPlatform::Windows, "/GameData/Audio/VOC_JP/JP_Airi.zip")
.is_err());
assert!(root
.bundle_patch_pack(PatchPlatform::Windows, "../FullPatch_000.zip")