//! Import of a verified official release into CAS and ResourceRepository. use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry}; use crate::official_parse::{ read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus, OfficialParseSummary, }; use crate::path_security::{ ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, }; use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType}; use bat_core::repositories::{CasRepository, ResourceRepository}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; /// Configuration for importing one already-published official release. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfficialReleaseImportConfig { /// Published release root containing `official-download-manifest.json`. pub release_root: PathBuf, /// Official release ID associated with this root, when known. pub official_release_id: Option, } impl OfficialReleaseImportConfig { /// Creates an import configuration. pub fn new(release_root: impl Into) -> Self { Self { release_root: release_root.into(), official_release_id: None, } } /// Attaches the official release ID that should be stored in resource metadata. pub fn with_official_release_id(mut self, release_id: impl Into) -> Self { self.official_release_id = Some(release_id.into()); self } } /// Summary returned by an official release repository import. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialReleaseImportReport { /// Number of verified manifest entries. pub manifest_entry_count: usize, /// Number of new or changed repository rows. pub imported_count: usize, /// Number of rows already pointing at the same CAS object. pub unchanged_count: usize, /// Number of unchanged rows whose metadata was refreshed from parse cache. #[serde(default)] pub metadata_updated_count: usize, /// Number of resources classified as AssetBundle. pub asset_bundle_count: usize, /// Number of resources classified as text-like payloads. pub text_asset_count: usize, /// Number of resources classified as tables. pub table_count: usize, /// Number of resources classified as media. pub media_count: usize, /// Parse-cache summary associated with this release, when available. pub parse_summary: Option, /// Non-fatal cleanup warnings, such as an old CAS reference that could not /// be decremented after a successful row replacement. pub warnings: Vec, } /// Imports verified official resources into CAS and the resource index. pub struct OfficialReleaseImportService<'a> { cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository, } impl<'a> OfficialReleaseImportService<'a> { /// Creates an import service. pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self { Self { cas, resources } } /// Imports every entry in one published release manifest. /// /// The source files remain untouched. Every file is checked against the /// verified download manifest before it can enter CAS. Repository IDs are /// stable by destination, making repeated imports idempotent. pub async fn import_release( &self, config: &OfficialReleaseImportConfig, ) -> bat_core::Result { let manifest = read_download_manifest_at(&config.release_root) .map_err(bat_core::Error::InvalidArgument)? .ok_or_else(|| { bat_core::Error::NotFound(format!( "官方下载 manifest 不存在:{}", config.release_root.display() )) })?; let parse_cache = read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?; let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone()); let parse_entries_by_destination = parse_cache .as_ref() .map(parse_entries_by_destination) .unwrap_or_default(); let release_id = config .official_release_id .clone() .or_else(|| release_id_from_root(&config.release_root)); let mut report = OfficialReleaseImportReport { manifest_entry_count: manifest.entries.len(), imported_count: 0, unchanged_count: 0, metadata_updated_count: 0, asset_bundle_count: 0, text_asset_count: 0, table_count: 0, media_count: 0, parse_summary, warnings: Vec::new(), }; for entry in manifest.entries.values() { let parse_entries = parse_entries_by_destination .get(&entry.destination) .map(Vec::as_slice) .unwrap_or(&[]); self.import_entry( &config.release_root, release_id.as_deref(), entry, parse_entries, &mut report, ) .await?; } Ok(report) } async fn import_entry( &self, release_root: &Path, official_release_id: Option<&str>, manifest_entry: &OfficialDownloadManifestEntry, parse_entries: &[&OfficialParseCacheEntry], report: &mut OfficialReleaseImportReport, ) -> bat_core::Result<()> { let path = release_root.join(Path::new(&manifest_entry.destination)); ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?; ensure_safe_file_target(release_root, &path, "官方资源导入输入") .map_err(bat_core::Error::InvalidArgument)?; let bytes = read_file_no_symlink(&path, "官方资源导入输入") .map_err(bat_core::Error::InvalidArgument)? .ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?; if bytes.len() as u64 != manifest_entry.bytes { return Err(bat_core::Error::InvalidArgument(format!( "官方资源导入 size 校验失败 {}:期望 {},实际 {}", manifest_entry.destination, manifest_entry.bytes, bytes.len() ))); } let actual_hash = blake3::hash(&bytes).to_hex().to_string(); if actual_hash != manifest_entry.blake3 { return Err(bat_core::Error::InvalidArgument(format!( "官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}", manifest_entry.destination, manifest_entry.blake3, actual_hash ))); } let resource_type = resource_type_for_path(&manifest_entry.destination); let metadata = metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries); let resource_id = resource_id_for_destination(&manifest_entry.destination); let previous = match self.resources.find_by_id(&resource_id).await { Ok(resource) => Some(resource), Err(bat_core::Error::NotFound(_)) => None, Err(error) => return Err(error), }; if previous .as_ref() .is_some_and(|resource| resource.entry.hash == actual_hash) { if let Some(mut resource) = previous { let should_refresh_metadata = resource.metadata != metadata || resource.entry.resource_type != resource_type || resource.entry.size != manifest_entry.bytes || resource.local_path.as_path() != Path::new(&manifest_entry.destination); if should_refresh_metadata { resource.local_path = PathBuf::from(&manifest_entry.destination); resource.entry.size = manifest_entry.bytes; resource.entry.resource_type = resource_type; resource.metadata = metadata; self.resources.update(resource).await?; report.metadata_updated_count += 1; } } report.unchanged_count += 1; count_resource_type(report, resource_type); return Ok(()); } let object_id = self.cas.store(&bytes).await?; let resource = Resource { id: resource_id, local_path: PathBuf::from(&manifest_entry.destination), entry: ResourceEntry { path: manifest_entry.destination.clone(), hash: object_id.clone(), size: manifest_entry.bytes, resource_type, address: None, dependencies: Vec::new(), crc: None, }, metadata, }; if let Err(error) = self.resources.add(resource).await { let _ = self.cas.remove_reference(&object_id).await; return Err(error); } if let Some(previous) = previous { if previous.entry.hash != object_id { match self.cas.remove_reference(&previous.entry.hash).await { Ok(_) => {} Err(error) => report.warnings.push(format!( "旧 CAS 引用清理失败 {}:{}", previous.entry.hash, error )), } } } report.imported_count += 1; count_resource_type(report, resource_type); Ok(()) } } fn parse_entries_by_destination( cache: &OfficialParseCache, ) -> BTreeMap> { let mut by_destination: BTreeMap> = BTreeMap::new(); for entry in cache.entries.values() { by_destination .entry(entry.destination.clone()) .or_default() .push(entry); } by_destination } fn metadata_for_manifest_entry( official_release_id: Option<&str>, manifest_entry: &OfficialDownloadManifestEntry, parse_entries: &[&OfficialParseCacheEntry], ) -> ResourceMetadata { let mut metadata = ResourceMetadata { official_release_id: official_release_id.map(ToOwned::to_owned), platform: platform_for_destination(&manifest_entry.destination), bundle_path: Some(manifest_entry.destination.clone()), ..ResourceMetadata::default() }; let mut archive_entries = BTreeSet::new(); let mut parse_statuses = BTreeSet::new(); let mut unity_versions = BTreeSet::new(); let mut text_assets = BTreeSet::new(); let mut text_unit_formats = BTreeSet::new(); for entry in parse_entries { if let Some(archive_entry) = entry.archive_entry.as_ref() { archive_entries.insert(archive_entry.clone()); } parse_statuses.insert(parse_status_label(entry.status).to_string()); if let Some(unity_version) = entry.unity_version.as_ref() { unity_versions.insert(unity_version.clone()); } metadata.unityfs_file_count += entry.file_count as u64; metadata.serialized_file_count += entry.serialized_file_count as u64; metadata.text_asset_count += entry.text_asset_count as u64; metadata.text_unit_count += entry.text_unit_count as u64; metadata.text_unit_error_count += entry.text_unit_error_count as u64; text_assets.extend(entry.text_assets.iter().cloned()); text_unit_formats.extend(entry.text_unit_formats.iter().cloned()); } metadata.archive_entries = archive_entries.into_iter().collect(); metadata.parse_statuses = parse_statuses.into_iter().collect(); metadata.unity_versions = unity_versions.into_iter().collect(); metadata.text_assets = text_assets.into_iter().collect(); metadata.text_unit_formats = text_unit_formats.into_iter().collect(); metadata } fn parse_status_label(status: OfficialParseStatus) -> &'static str { match status { OfficialParseStatus::Parsed => "parsed", OfficialParseStatus::SkippedUnsupported => "skipped_unsupported", OfficialParseStatus::Failed => "failed", } } fn platform_for_destination(destination: &str) -> Option { let normalized = destination.replace('\\', "/").to_ascii_lowercase(); if normalized.contains("windows") || normalized.contains("/win/") { Some("windows".to_string()) } else if normalized.contains("android") { Some("android".to_string()) } else { None } } fn release_id_from_root(release_root: &Path) -> Option { release_root .file_name() .and_then(|name| name.to_str()) .filter(|name| !name.is_empty() && *name != "current") .map(ToOwned::to_owned) } fn resource_id_for_destination(destination: &str) -> String { format!("official/{}", destination.replace('\\', "/")) } fn resource_type_for_path(path: &str) -> ResourceType { let normalized = path.replace('\\', "/").to_ascii_lowercase(); if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") { ResourceType::AssetBundle } else if normalized.contains("tablebundles/") { ResourceType::TableBundle } else if normalized.contains("textassets/") || matches!( normalized.rsplit('.').next(), Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml") ) { ResourceType::TextAsset } else if normalized.contains("mediaresources/") || matches!( normalized.rsplit('.').next(), Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp") ) { ResourceType::Media } else if normalized.contains("catalog") || normalized.ends_with(".hash") { ResourceType::Manifest } else { ResourceType::Other } } fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) { match resource_type { ResourceType::AssetBundle => report.asset_bundle_count += 1, ResourceType::TextAsset => report.text_asset_count += 1, ResourceType::TableBundle => report.table_count += 1, ResourceType::Media => report.media_count += 1, ResourceType::Manifest | ResourceType::Other => {} } } #[cfg(test)] mod tests { use super::*; use crate::{FileSystemCasRepository, InMemoryResourceRepository}; use crate::{ OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION, }; use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository}; use std::collections::BTreeMap; use tempfile::TempDir; fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) { let path = root.join(destination); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(&path, bytes).unwrap(); let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}"); let entry = OfficialDownloadManifestEntry { url: url.clone(), destination: destination.to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }; let manifest = serde_json::json!({ "version": 1, "entries": BTreeMap::from([(url, entry)]), }); std::fs::write( root.join("official-download-manifest.json"), serde_json::to_vec(&manifest).unwrap(), ) .unwrap(); } fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) { let source_url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}"); let entry = OfficialParseCacheEntry { key: format!("direct:{source_url}"), source_url: source_url.clone(), destination: destination.to_string(), archive_entry: None, source_kind: OfficialParseSourceKind::DirectBundle, fingerprint: OfficialParseSourceFingerprint { source_url, destination: destination.to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }, status: OfficialParseStatus::Parsed, reused_from_previous_cache: false, unity_version: Some("2021.3.56f2".to_string()), file_count: 1, serialized_file_count: 1, text_asset_count: 1, text_assets: vec!["Scenario".to_string()], serialized_parse_error_count: 0, text_unit_count: 2, text_unit_formats: vec!["json".to_string(), "plain".to_string()], skipped_binary_text_asset_count: 0, text_unit_error_count: 1, error: None, }; let cache = OfficialParseCache { version: OFFICIAL_PARSE_CACHE_VERSION, generated_unix_seconds: 123, summary: OfficialParseSummary { manifest_entry_count: 1, cache_entry_count: 1, candidate_file_count: 1, zip_entry_count: 0, skipped_unchanged_count: 0, parsed_bundle_count: 1, unsupported_count: 0, failed_count: 0, text_asset_count: 1, text_unit_count: 2, skipped_binary_text_asset_count: 0, text_unit_error_count: 1, }, entries: BTreeMap::from([(entry.key.clone(), entry)]), }; std::fs::write( root.join("official-parse-cache.json"), serde_json::to_vec(&cache).unwrap(), ) .unwrap(); } #[tokio::test] async fn imports_verified_release_idempotently() { let temp = TempDir::new().unwrap(); write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog"); let cas = FileSystemCasRepository::new(temp.path().join("cas")); let resources = InMemoryResourceRepository::new(); let service = OfficialReleaseImportService::new(&cas, &resources); let config = OfficialReleaseImportConfig::new(temp.path()); let first = service.import_release(&config).await.unwrap(); let second = service.import_release(&config).await.unwrap(); assert_eq!(first.imported_count, 1); assert_eq!(second.imported_count, 0); assert_eq!(second.unchanged_count, 1); assert_eq!( resources .count(ResourceQuery::by_type(ResourceType::TableBundle)) .await .unwrap(), 1 ); let id = resource_id_for_destination("TableBundles/TableCatalog.bytes"); let resource = resources.find_by_id(&id).await.unwrap(); assert_eq!( cas.get_reference_count(&resource.entry.hash).await.unwrap(), 1 ); } #[tokio::test] async fn imports_parse_cache_metadata_into_resource_index() { let temp = TempDir::new().unwrap(); let destination = "Windows/Bundles/scenario.bundle"; let bytes = b"bundle-bytes"; write_manifest(temp.path(), destination, bytes); write_parse_cache(temp.path(), destination, bytes); let cas = FileSystemCasRepository::new(temp.path().join("cas")); let resources = InMemoryResourceRepository::new(); let service = OfficialReleaseImportService::new(&cas, &resources); let report = service .import_release( &OfficialReleaseImportConfig::new(temp.path()) .with_official_release_id("release-current"), ) .await .unwrap(); assert_eq!(report.imported_count, 1); assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2); let id = resource_id_for_destination(destination); let resource = resources.find_by_id(&id).await.unwrap(); assert_eq!( resource.metadata.official_release_id.as_deref(), Some("release-current") ); assert_eq!(resource.metadata.platform.as_deref(), Some("windows")); assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination)); assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]); assert_eq!( resource.metadata.unity_versions, vec!["2021.3.56f2".to_string()] ); assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]); assert_eq!(resource.metadata.text_unit_count, 2); assert_eq!( resource.metadata.text_unit_formats, vec!["json".to_string(), "plain".to_string()] ); assert_eq!(resource.metadata.text_unit_error_count, 1); } #[tokio::test] async fn rejects_manifest_hash_mismatch_before_cas_write() { let temp = TempDir::new().unwrap(); write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original"); let path = temp.path().join("TextAssets/dialogue.txt"); std::fs::write(&path, b"tampered").unwrap(); let cas = FileSystemCasRepository::new(temp.path().join("cas")); let resources = InMemoryResourceRepository::new(); let service = OfficialReleaseImportService::new(&cas, &resources); let error = service .import_release(&OfficialReleaseImportConfig::new(temp.path())) .await .unwrap_err(); assert!(error.to_string().contains("BLAKE3")); assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0); } }