feat: track official versions and import resources

Persist official version-state, extend CAS/ResourceRepository import classification, and add offline catalog and failure regression fixtures.

Fixes #13

Fixes #12

Fixes #8
This commit is contained in:
2026-07-15 19:53:10 +08:00
parent 4192d7ee4c
commit 90307a3243
30 changed files with 1269 additions and 157 deletions
+235 -28
View File
@@ -4,6 +4,7 @@ use bat_adapters::manifest::GenericManifest;
use bat_adapters::unity::{RawAssetBundle, UnityAdapterRegistry};
use bat_core::domain::{Resource, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
/// 待导入的 AssetBundle 数据。
@@ -32,12 +33,47 @@ pub struct ImportedResource {
pub id: String,
/// Manifest 中的资源路径。
pub source_path: String,
/// Manifest 或路径推断出的资源类型。
pub resource_type: ResourceType,
/// 导入链路使用的稳定分类标签。
pub category: ResourceImportCategory,
/// CAS 返回的对象 ID。
pub object_id: String,
/// 写入 CAS 的字节数。
pub bytes: u64,
/// UnityFS 解析摘要。
pub unityfs: UnityFsImportSummary,
/// UnityFS 解析摘要,仅 AssetBundle 资源会填充
pub unityfs: Option<UnityFsImportSummary>,
}
/// Stable resource category produced by the import pipeline.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ResourceImportCategory {
/// UnityFS AssetBundle resource.
AssetBundle,
/// Unity TextAsset or text-like payload.
TextAsset,
/// Table bundle/database resource.
Table,
/// Media payload such as audio, video, texture, or media archive.
Media,
/// Manifest/catalog sidecar.
Manifest,
/// Resource that does not match a known category yet.
Other,
}
impl ResourceImportCategory {
/// Returns a stable lowercase label for reports and golden tests.
pub fn as_str(self) -> &'static str {
match self {
Self::AssetBundle => "asset_bundle",
Self::TextAsset => "text_asset",
Self::Table => "table",
Self::Media => "media",
Self::Manifest => "manifest",
Self::Other => "other",
}
}
}
/// 导入时解析到的 UnityFS 基础结构摘要。
@@ -58,8 +94,10 @@ pub struct UnityFsImportSummary {
pub struct ResourceImportReport {
/// 成功导入的资源。
pub imported: Vec<ImportedResource>,
/// 被跳过的非 AssetBundle 资源路径。
/// 被跳过的资源路径。
pub skipped: Vec<String>,
/// 按稳定分类标签统计的成功导入数量。
pub category_counts: BTreeMap<String, usize>,
}
/// 将解析后的资源清单写入 CAS 与资源仓储的应用服务。
@@ -88,10 +126,12 @@ impl<'a> ResourceImportService<'a> {
}
}
/// 导入 manifest 中的 AssetBundle 资源
/// 导入 manifest 中可用的资源数据
///
/// AssetBundle 条目会记录到 `skipped`。AssetBundle 条目必须能在
/// `bundles` 中按完整路径或文件名找到对应数据,否则返回错误。
/// AssetBundle 条目必须能在 `bundles` 中按完整路径或文件名找到对应
/// 数据并成功解析 UnityFS,否则返回错误。TextAsset/Table/Media 等
/// 非 AssetBundle 条目在有数据时写入 CAS 和 `ResourceRepository`;缺少
/// 数据时记录到 `skipped`,用于渐进式导入官方下载结果。
pub async fn import_manifest_bundles(
&self,
manifest: &GenericManifest,
@@ -100,18 +140,23 @@ impl<'a> ResourceImportService<'a> {
let mut report = ResourceImportReport::default();
for entry in &manifest.resources {
if entry.resource_type != ResourceType::AssetBundle {
let category = classify_import_category(entry.resource_type, &entry.path);
let Some(bundle) = find_bundle(bundles, &entry.path) else {
if entry.resource_type == ResourceType::AssetBundle {
return Err(bat_core::Error::InvalidArgument(format!(
"Missing bundle data for manifest resource: {}",
entry.path
)));
}
report.skipped.push(entry.path.clone());
continue;
}
};
let bundle = find_bundle(bundles, &entry.path).ok_or_else(|| {
bat_core::Error::InvalidArgument(format!(
"Missing bundle data for manifest resource: {}",
entry.path
))
})?;
let unityfs = self.parse_unityfs_summary(&entry.path, bundle).await?;
let unityfs = if entry.resource_type == ResourceType::AssetBundle {
Some(self.parse_unityfs_summary(&entry.path, bundle).await?)
} else {
None
};
let object_id = self.cas.store(&bundle.data).await?;
let mut stored_entry = entry.clone();
@@ -128,10 +173,16 @@ impl<'a> ResourceImportService<'a> {
report.imported.push(ImportedResource {
id,
source_path: entry.path.clone(),
resource_type: entry.resource_type,
category,
object_id,
bytes: bundle.data.len() as u64,
unityfs,
});
*report
.category_counts
.entry(category.as_str().to_string())
.or_insert(0) += 1;
}
Ok(report)
@@ -196,6 +247,43 @@ fn file_name(path: &str) -> Option<&str> {
Path::new(path).file_name()?.to_str()
}
fn classify_import_category(resource_type: ResourceType, path: &str) -> ResourceImportCategory {
match resource_type {
ResourceType::AssetBundle => ResourceImportCategory::AssetBundle,
ResourceType::TextAsset => ResourceImportCategory::TextAsset,
ResourceType::TableBundle => ResourceImportCategory::Table,
ResourceType::Media => ResourceImportCategory::Media,
ResourceType::Manifest => ResourceImportCategory::Manifest,
ResourceType::Other => classify_import_category_from_path(path),
}
}
fn classify_import_category_from_path(path: &str) -> ResourceImportCategory {
let normalized = path.replace('\\', "/").to_ascii_lowercase();
if normalized.contains("tablebundles/") {
ResourceImportCategory::Table
} else if normalized.contains("mediaresources/")
|| normalized.contains("mediaresources-")
|| matches!(
normalized.rsplit('.').next(),
Some("mp3" | "mp4" | "ogg" | "wav" | "png" | "jpg" | "jpeg" | "webp" | "acb" | "awb")
)
{
ResourceImportCategory::Media
} else if normalized.contains("textassets/")
|| matches!(
normalized.rsplit('.').next(),
Some("txt" | "csv" | "xml" | "yaml" | "yml")
)
{
ResourceImportCategory::TextAsset
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
ResourceImportCategory::Manifest
} else {
ResourceImportCategory::Other
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -287,6 +375,30 @@ mod tests {
address: None,
dependencies: Vec::new(),
},
ResourceEntry {
path: "TextAssets/dialogue.csv".to_string(),
hash: "synthetic-text-hash".to_string(),
size: 1,
resource_type: ResourceType::TextAsset,
address: Some("dialogue".to_string()),
dependencies: Vec::new(),
},
ResourceEntry {
path: "TableBundles/ExcelDB.db".to_string(),
hash: "synthetic-table-hash".to_string(),
size: 1,
resource_type: ResourceType::TableBundle,
address: None,
dependencies: Vec::new(),
},
ResourceEntry {
path: "MediaResources-Windows/voice/title.acb".to_string(),
hash: "synthetic-media-hash".to_string(),
size: 1,
resource_type: ResourceType::Media,
address: None,
dependencies: Vec::new(),
},
],
metadata: ManifestMetadata {
locator_id: None,
@@ -306,32 +418,78 @@ mod tests {
let report = service
.import_manifest_bundles(
&synthetic_manifest(),
&[BundleSource::new(
"minimal.bundle",
synthetic_minimal_unityfs_bundle(),
)],
&[
BundleSource::new("minimal.bundle", synthetic_minimal_unityfs_bundle()),
BundleSource::new("dialogue.csv", b"id,text\n1,hello".to_vec()),
BundleSource::new("ExcelDB.db", b"table-fixture".to_vec()),
BundleSource::new("title.acb", b"media-fixture".to_vec()),
],
)
.await
.unwrap();
assert_eq!(report.imported.len(), 1);
assert_eq!(report.imported.len(), 4);
assert_eq!(report.skipped, vec!["synthetic/catalog.json".to_string()]);
assert!(cas.exists(&report.imported[0].object_id).await);
assert_eq!(report.imported[0].unityfs.unity_version, "2021.3.56f2");
assert_eq!(report.imported[0].unityfs.block_count, 1);
assert_eq!(report.imported[0].unityfs.directory_count, 1);
assert_eq!(report.imported[0].resource_type, ResourceType::AssetBundle);
assert_eq!(
report.imported[0].unityfs.directories,
vec!["SYNTHETIC-CAB".to_string()]
report.imported[0].category,
ResourceImportCategory::AssetBundle
);
let unityfs = report.imported[0].unityfs.as_ref().unwrap();
assert_eq!(unityfs.unity_version, "2021.3.56f2");
assert_eq!(unityfs.block_count, 1);
assert_eq!(unityfs.directory_count, 1);
assert_eq!(unityfs.directories, vec!["SYNTHETIC-CAB".to_string()]);
assert_eq!(
report.imported[1].category,
ResourceImportCategory::TextAsset
);
assert_eq!(report.imported[2].category, ResourceImportCategory::Table);
assert_eq!(report.imported[3].category, ResourceImportCategory::Media);
assert_eq!(report.category_counts.get("asset_bundle"), Some(&1));
assert_eq!(report.category_counts.get("text_asset"), Some(&1));
assert_eq!(report.category_counts.get("table"), Some(&1));
assert_eq!(report.category_counts.get("media"), Some(&1));
let indexed = resources.list(ResourceQuery::all()).await.unwrap();
assert_eq!(indexed.len(), 1);
assert_eq!(indexed[0].entry.hash, report.imported[0].object_id);
assert_eq!(indexed.len(), 4);
let indexed_bundle = indexed
.iter()
.find(|resource| resource.entry.path == "synthetic/minimal.bundle")
.unwrap();
assert_eq!(indexed_bundle.entry.hash, report.imported[0].object_id);
assert_eq!(
indexed[0].entry.size,
indexed_bundle.entry.size,
synthetic_minimal_unityfs_bundle().len() as u64
);
assert_eq!(
indexed
.iter()
.find(|resource| resource.entry.path.ends_with("title.acb"))
.unwrap()
.entry
.resource_type,
ResourceType::Media
);
assert_eq!(
indexed
.iter()
.find(|resource| resource.entry.path.ends_with("ExcelDB.db"))
.unwrap()
.entry
.resource_type,
ResourceType::TableBundle
);
assert_eq!(
indexed
.iter()
.find(|resource| resource.entry.path.ends_with("dialogue.csv"))
.unwrap()
.entry
.resource_type,
ResourceType::TextAsset
);
}
#[tokio::test]
@@ -370,4 +528,53 @@ mod tests {
assert!(matches!(error, bat_core::Error::InvalidArgument(_)));
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
}
#[tokio::test]
async fn indexes_non_bundle_resources_without_unityfs_summary() {
let temp_dir = TempDir::new().unwrap();
let cas = FileSystemCasRepository::new(temp_dir.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = ResourceImportService::new(&cas, &resources);
let mut manifest = synthetic_manifest();
manifest
.resources
.retain(|entry| entry.resource_type != ResourceType::AssetBundle);
let report = service
.import_manifest_bundles(
&manifest,
&[
BundleSource::new("dialogue.csv", b"id,text\n1,hello".to_vec()),
BundleSource::new("ExcelDB.db", b"table-fixture".to_vec()),
BundleSource::new("title.acb", b"media-fixture".to_vec()),
],
)
.await
.unwrap();
assert_eq!(report.imported.len(), 3);
assert!(report.imported.iter().all(|item| item.unityfs.is_none()));
assert_eq!(report.skipped, vec!["synthetic/catalog.json".to_string()]);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TextAsset))
.await
.unwrap(),
1
);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TableBundle))
.await
.unwrap(),
1
);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::Media))
.await
.unwrap(),
1
);
}
}