feat(sync): 接入解析缓存与汉化发布前置

补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
2026-07-31 00:38:45 +08:00
parent f4880a71bd
commit 2079c6a307
25 changed files with 17287 additions and 225 deletions
+543
View File
@@ -0,0 +1,543 @@
//! Import of a verified official release into CAS and ResourceRepository.
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus,
OfficialParseSummary,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink,
};
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// Configuration for importing one already-published official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialReleaseImportConfig {
/// Published release root containing `official-download-manifest.json`.
pub release_root: PathBuf,
/// Official release ID associated with this root, when known.
pub official_release_id: Option<String>,
}
impl OfficialReleaseImportConfig {
/// Creates an import configuration.
pub fn new(release_root: impl Into<PathBuf>) -> Self {
Self {
release_root: release_root.into(),
official_release_id: None,
}
}
/// Attaches the official release ID that should be stored in resource metadata.
pub fn with_official_release_id(mut self, release_id: impl Into<String>) -> Self {
self.official_release_id = Some(release_id.into());
self
}
}
/// Summary returned by an official release repository import.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialReleaseImportReport {
/// Number of verified manifest entries.
pub manifest_entry_count: usize,
/// Number of new or changed repository rows.
pub imported_count: usize,
/// Number of rows already pointing at the same CAS object.
pub unchanged_count: usize,
/// Number of unchanged rows whose metadata was refreshed from parse cache.
#[serde(default)]
pub metadata_updated_count: usize,
/// Number of resources classified as AssetBundle.
pub asset_bundle_count: usize,
/// Number of resources classified as text-like payloads.
pub text_asset_count: usize,
/// Number of resources classified as tables.
pub table_count: usize,
/// Number of resources classified as media.
pub media_count: usize,
/// Parse-cache summary associated with this release, when available.
pub parse_summary: Option<OfficialParseSummary>,
/// Non-fatal cleanup warnings, such as an old CAS reference that could not
/// be decremented after a successful row replacement.
pub warnings: Vec<String>,
}
/// Imports verified official resources into CAS and the resource index.
pub struct OfficialReleaseImportService<'a> {
cas: &'a dyn CasRepository,
resources: &'a dyn ResourceRepository,
}
impl<'a> OfficialReleaseImportService<'a> {
/// Creates an import service.
pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self {
Self { cas, resources }
}
/// Imports every entry in one published release manifest.
///
/// The source files remain untouched. Every file is checked against the
/// verified download manifest before it can enter CAS. Repository IDs are
/// stable by destination, making repeated imports idempotent.
pub async fn import_release(
&self,
config: &OfficialReleaseImportConfig,
) -> bat_core::Result<OfficialReleaseImportReport> {
let manifest = read_download_manifest_at(&config.release_root)
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| {
bat_core::Error::NotFound(format!(
"官方下载 manifest 不存在:{}",
config.release_root.display()
))
})?;
let parse_cache =
read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?;
let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone());
let parse_entries_by_destination = parse_cache
.as_ref()
.map(parse_entries_by_destination)
.unwrap_or_default();
let release_id = config
.official_release_id
.clone()
.or_else(|| release_id_from_root(&config.release_root));
let mut report = OfficialReleaseImportReport {
manifest_entry_count: manifest.entries.len(),
imported_count: 0,
unchanged_count: 0,
metadata_updated_count: 0,
asset_bundle_count: 0,
text_asset_count: 0,
table_count: 0,
media_count: 0,
parse_summary,
warnings: Vec::new(),
};
for entry in manifest.entries.values() {
let parse_entries = parse_entries_by_destination
.get(&entry.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
self.import_entry(
&config.release_root,
release_id.as_deref(),
entry,
parse_entries,
&mut report,
)
.await?;
}
Ok(report)
}
async fn import_entry(
&self,
release_root: &Path,
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
report: &mut OfficialReleaseImportReport,
) -> bat_core::Result<()> {
let path = release_root.join(Path::new(&manifest_entry.destination));
ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?;
ensure_safe_file_target(release_root, &path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?;
let bytes = read_file_no_symlink(&path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?;
if bytes.len() as u64 != manifest_entry.bytes {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 size 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination,
manifest_entry.bytes,
bytes.len()
)));
}
let actual_hash = blake3::hash(&bytes).to_hex().to_string();
if actual_hash != manifest_entry.blake3 {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination, manifest_entry.blake3, actual_hash
)));
}
let resource_type = resource_type_for_path(&manifest_entry.destination);
let metadata =
metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries);
let resource_id = resource_id_for_destination(&manifest_entry.destination);
let previous = match self.resources.find_by_id(&resource_id).await {
Ok(resource) => Some(resource),
Err(bat_core::Error::NotFound(_)) => None,
Err(error) => return Err(error),
};
if previous
.as_ref()
.is_some_and(|resource| resource.entry.hash == actual_hash)
{
if let Some(mut resource) = previous {
let should_refresh_metadata = resource.metadata != metadata
|| resource.entry.resource_type != resource_type
|| resource.entry.size != manifest_entry.bytes
|| resource.local_path.as_path() != Path::new(&manifest_entry.destination);
if should_refresh_metadata {
resource.local_path = PathBuf::from(&manifest_entry.destination);
resource.entry.size = manifest_entry.bytes;
resource.entry.resource_type = resource_type;
resource.metadata = metadata;
self.resources.update(resource).await?;
report.metadata_updated_count += 1;
}
}
report.unchanged_count += 1;
count_resource_type(report, resource_type);
return Ok(());
}
let object_id = self.cas.store(&bytes).await?;
let resource = Resource {
id: resource_id,
local_path: PathBuf::from(&manifest_entry.destination),
entry: ResourceEntry {
path: manifest_entry.destination.clone(),
hash: object_id.clone(),
size: manifest_entry.bytes,
resource_type,
address: None,
dependencies: Vec::new(),
crc: None,
},
metadata,
};
if let Err(error) = self.resources.add(resource).await {
let _ = self.cas.remove_reference(&object_id).await;
return Err(error);
}
if let Some(previous) = previous {
if previous.entry.hash != object_id {
match self.cas.remove_reference(&previous.entry.hash).await {
Ok(_) => {}
Err(error) => report.warnings.push(format!(
"旧 CAS 引用清理失败 {}{}",
previous.entry.hash, error
)),
}
}
}
report.imported_count += 1;
count_resource_type(report, resource_type);
Ok(())
}
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn metadata_for_manifest_entry(
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
) -> ResourceMetadata {
let mut metadata = ResourceMetadata {
official_release_id: official_release_id.map(ToOwned::to_owned),
platform: platform_for_destination(&manifest_entry.destination),
bundle_path: Some(manifest_entry.destination.clone()),
..ResourceMetadata::default()
};
let mut archive_entries = BTreeSet::new();
let mut parse_statuses = BTreeSet::new();
let mut unity_versions = BTreeSet::new();
let mut text_assets = BTreeSet::new();
let mut text_unit_formats = BTreeSet::new();
for entry in parse_entries {
if let Some(archive_entry) = entry.archive_entry.as_ref() {
archive_entries.insert(archive_entry.clone());
}
parse_statuses.insert(parse_status_label(entry.status).to_string());
if let Some(unity_version) = entry.unity_version.as_ref() {
unity_versions.insert(unity_version.clone());
}
metadata.unityfs_file_count += entry.file_count as u64;
metadata.serialized_file_count += entry.serialized_file_count as u64;
metadata.text_asset_count += entry.text_asset_count as u64;
metadata.text_unit_count += entry.text_unit_count as u64;
metadata.text_unit_error_count += entry.text_unit_error_count as u64;
text_assets.extend(entry.text_assets.iter().cloned());
text_unit_formats.extend(entry.text_unit_formats.iter().cloned());
}
metadata.archive_entries = archive_entries.into_iter().collect();
metadata.parse_statuses = parse_statuses.into_iter().collect();
metadata.unity_versions = unity_versions.into_iter().collect();
metadata.text_assets = text_assets.into_iter().collect();
metadata.text_unit_formats = text_unit_formats.into_iter().collect();
metadata
}
fn parse_status_label(status: OfficialParseStatus) -> &'static str {
match status {
OfficialParseStatus::Parsed => "parsed",
OfficialParseStatus::SkippedUnsupported => "skipped_unsupported",
OfficialParseStatus::Failed => "failed",
}
}
fn platform_for_destination(destination: &str) -> Option<String> {
let normalized = destination.replace('\\', "/").to_ascii_lowercase();
if normalized.contains("windows") || normalized.contains("/win/") {
Some("windows".to_string())
} else if normalized.contains("android") {
Some("android".to_string())
} else {
None
}
}
fn release_id_from_root(release_root: &Path) -> Option<String> {
release_root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty() && *name != "current")
.map(ToOwned::to_owned)
}
fn resource_id_for_destination(destination: &str) -> String {
format!("official/{}", destination.replace('\\', "/"))
}
fn resource_type_for_path(path: &str) -> ResourceType {
let normalized = path.replace('\\', "/").to_ascii_lowercase();
if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") {
ResourceType::AssetBundle
} else if normalized.contains("tablebundles/") {
ResourceType::TableBundle
} else if normalized.contains("textassets/")
|| matches!(
normalized.rsplit('.').next(),
Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml")
)
{
ResourceType::TextAsset
} else if normalized.contains("mediaresources/")
|| matches!(
normalized.rsplit('.').next(),
Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp")
)
{
ResourceType::Media
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
ResourceType::Manifest
} else {
ResourceType::Other
}
}
fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) {
match resource_type {
ResourceType::AssetBundle => report.asset_bundle_count += 1,
ResourceType::TextAsset => report.text_asset_count += 1,
ResourceType::TableBundle => report.table_count += 1,
ResourceType::Media => report.media_count += 1,
ResourceType::Manifest | ResourceType::Other => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{FileSystemCasRepository, InMemoryResourceRepository};
use crate::{
OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION,
};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use std::collections::BTreeMap;
use tempfile::TempDir;
fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) {
let path = root.join(destination);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, bytes).unwrap();
let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialDownloadManifestEntry {
url: url.clone(),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
};
let manifest = serde_json::json!({
"version": 1,
"entries": BTreeMap::from([(url, entry)]),
});
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
}
fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) {
let source_url =
format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialParseCacheEntry {
key: format!("direct:{source_url}"),
source_url: source_url.clone(),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url,
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: 1,
text_assets: vec!["Scenario".to_string()],
serialized_parse_error_count: 0,
text_unit_count: 2,
text_unit_formats: vec!["json".to_string(), "plain".to_string()],
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
error: None,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 123,
summary: OfficialParseSummary {
manifest_entry_count: 1,
cache_entry_count: 1,
candidate_file_count: 1,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 0,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
},
entries: BTreeMap::from([(entry.key.clone(), entry)]),
};
std::fs::write(
root.join("official-parse-cache.json"),
serde_json::to_vec(&cache).unwrap(),
)
.unwrap();
}
#[tokio::test]
async fn imports_verified_release_idempotently() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog");
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let config = OfficialReleaseImportConfig::new(temp.path());
let first = service.import_release(&config).await.unwrap();
let second = service.import_release(&config).await.unwrap();
assert_eq!(first.imported_count, 1);
assert_eq!(second.imported_count, 0);
assert_eq!(second.unchanged_count, 1);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TableBundle))
.await
.unwrap(),
1
);
let id = resource_id_for_destination("TableBundles/TableCatalog.bytes");
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
cas.get_reference_count(&resource.entry.hash).await.unwrap(),
1
);
}
#[tokio::test]
async fn imports_parse_cache_metadata_into_resource_index() {
let temp = TempDir::new().unwrap();
let destination = "Windows/Bundles/scenario.bundle";
let bytes = b"bundle-bytes";
write_manifest(temp.path(), destination, bytes);
write_parse_cache(temp.path(), destination, bytes);
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let report = service
.import_release(
&OfficialReleaseImportConfig::new(temp.path())
.with_official_release_id("release-current"),
)
.await
.unwrap();
assert_eq!(report.imported_count, 1);
assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2);
let id = resource_id_for_destination(destination);
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
resource.metadata.official_release_id.as_deref(),
Some("release-current")
);
assert_eq!(resource.metadata.platform.as_deref(), Some("windows"));
assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination));
assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]);
assert_eq!(
resource.metadata.unity_versions,
vec!["2021.3.56f2".to_string()]
);
assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(resource.metadata.text_unit_count, 2);
assert_eq!(
resource.metadata.text_unit_formats,
vec!["json".to_string(), "plain".to_string()]
);
assert_eq!(resource.metadata.text_unit_error_count, 1);
}
#[tokio::test]
async fn rejects_manifest_hash_mismatch_before_cas_write() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original");
let path = temp.path().join("TextAssets/dialogue.txt");
std::fs::write(&path, b"tampered").unwrap();
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let error = service
.import_release(&OfficialReleaseImportConfig::new(temp.path()))
.await
.unwrap_err();
assert!(error.to_string().contains("BLAKE3"));
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
}
}