Files
BlueArchiveToolkit/infrastructure/src/official_repository.rs
T
nyaKazuha 2079c6a307 feat(sync): 接入解析缓存与汉化发布前置
补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
2026-07-31 00:38:45 +08:00

544 lines
22 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Import of a verified official release into CAS and ResourceRepository.
use crate::official_download::{read_download_manifest_at, OfficialDownloadManifestEntry};
use crate::official_parse::{
read_parse_cache_at, OfficialParseCache, OfficialParseCacheEntry, OfficialParseStatus,
OfficialParseSummary,
};
use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink,
};
use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType};
use bat_core::repositories::{CasRepository, ResourceRepository};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// Configuration for importing one already-published official release.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfficialReleaseImportConfig {
/// Published release root containing `official-download-manifest.json`.
pub release_root: PathBuf,
/// Official release ID associated with this root, when known.
pub official_release_id: Option<String>,
}
impl OfficialReleaseImportConfig {
/// Creates an import configuration.
pub fn new(release_root: impl Into<PathBuf>) -> Self {
Self {
release_root: release_root.into(),
official_release_id: None,
}
}
/// Attaches the official release ID that should be stored in resource metadata.
pub fn with_official_release_id(mut self, release_id: impl Into<String>) -> Self {
self.official_release_id = Some(release_id.into());
self
}
}
/// Summary returned by an official release repository import.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialReleaseImportReport {
/// Number of verified manifest entries.
pub manifest_entry_count: usize,
/// Number of new or changed repository rows.
pub imported_count: usize,
/// Number of rows already pointing at the same CAS object.
pub unchanged_count: usize,
/// Number of unchanged rows whose metadata was refreshed from parse cache.
#[serde(default)]
pub metadata_updated_count: usize,
/// Number of resources classified as AssetBundle.
pub asset_bundle_count: usize,
/// Number of resources classified as text-like payloads.
pub text_asset_count: usize,
/// Number of resources classified as tables.
pub table_count: usize,
/// Number of resources classified as media.
pub media_count: usize,
/// Parse-cache summary associated with this release, when available.
pub parse_summary: Option<OfficialParseSummary>,
/// Non-fatal cleanup warnings, such as an old CAS reference that could not
/// be decremented after a successful row replacement.
pub warnings: Vec<String>,
}
/// Imports verified official resources into CAS and the resource index.
pub struct OfficialReleaseImportService<'a> {
cas: &'a dyn CasRepository,
resources: &'a dyn ResourceRepository,
}
impl<'a> OfficialReleaseImportService<'a> {
/// Creates an import service.
pub fn new(cas: &'a dyn CasRepository, resources: &'a dyn ResourceRepository) -> Self {
Self { cas, resources }
}
/// Imports every entry in one published release manifest.
///
/// The source files remain untouched. Every file is checked against the
/// verified download manifest before it can enter CAS. Repository IDs are
/// stable by destination, making repeated imports idempotent.
pub async fn import_release(
&self,
config: &OfficialReleaseImportConfig,
) -> bat_core::Result<OfficialReleaseImportReport> {
let manifest = read_download_manifest_at(&config.release_root)
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| {
bat_core::Error::NotFound(format!(
"官方下载 manifest 不存在:{}",
config.release_root.display()
))
})?;
let parse_cache =
read_parse_cache_at(&config.release_root).map_err(bat_core::Error::InvalidArgument)?;
let parse_summary = parse_cache.as_ref().map(|cache| cache.summary.clone());
let parse_entries_by_destination = parse_cache
.as_ref()
.map(parse_entries_by_destination)
.unwrap_or_default();
let release_id = config
.official_release_id
.clone()
.or_else(|| release_id_from_root(&config.release_root));
let mut report = OfficialReleaseImportReport {
manifest_entry_count: manifest.entries.len(),
imported_count: 0,
unchanged_count: 0,
metadata_updated_count: 0,
asset_bundle_count: 0,
text_asset_count: 0,
table_count: 0,
media_count: 0,
parse_summary,
warnings: Vec::new(),
};
for entry in manifest.entries.values() {
let parse_entries = parse_entries_by_destination
.get(&entry.destination)
.map(Vec::as_slice)
.unwrap_or(&[]);
self.import_entry(
&config.release_root,
release_id.as_deref(),
entry,
parse_entries,
&mut report,
)
.await?;
}
Ok(report)
}
async fn import_entry(
&self,
release_root: &Path,
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
report: &mut OfficialReleaseImportReport,
) -> bat_core::Result<()> {
let path = release_root.join(Path::new(&manifest_entry.destination));
ensure_path_within_root(release_root, &path).map_err(bat_core::Error::InvalidArgument)?;
ensure_safe_file_target(release_root, &path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?;
let bytes = read_file_no_symlink(&path, "官方资源导入输入")
.map_err(bat_core::Error::InvalidArgument)?
.ok_or_else(|| bat_core::Error::NotFound(path.display().to_string()))?;
if bytes.len() as u64 != manifest_entry.bytes {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 size 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination,
manifest_entry.bytes,
bytes.len()
)));
}
let actual_hash = blake3::hash(&bytes).to_hex().to_string();
if actual_hash != manifest_entry.blake3 {
return Err(bat_core::Error::InvalidArgument(format!(
"官方资源导入 BLAKE3 校验失败 {}:期望 {},实际 {}",
manifest_entry.destination, manifest_entry.blake3, actual_hash
)));
}
let resource_type = resource_type_for_path(&manifest_entry.destination);
let metadata =
metadata_for_manifest_entry(official_release_id, manifest_entry, parse_entries);
let resource_id = resource_id_for_destination(&manifest_entry.destination);
let previous = match self.resources.find_by_id(&resource_id).await {
Ok(resource) => Some(resource),
Err(bat_core::Error::NotFound(_)) => None,
Err(error) => return Err(error),
};
if previous
.as_ref()
.is_some_and(|resource| resource.entry.hash == actual_hash)
{
if let Some(mut resource) = previous {
let should_refresh_metadata = resource.metadata != metadata
|| resource.entry.resource_type != resource_type
|| resource.entry.size != manifest_entry.bytes
|| resource.local_path.as_path() != Path::new(&manifest_entry.destination);
if should_refresh_metadata {
resource.local_path = PathBuf::from(&manifest_entry.destination);
resource.entry.size = manifest_entry.bytes;
resource.entry.resource_type = resource_type;
resource.metadata = metadata;
self.resources.update(resource).await?;
report.metadata_updated_count += 1;
}
}
report.unchanged_count += 1;
count_resource_type(report, resource_type);
return Ok(());
}
let object_id = self.cas.store(&bytes).await?;
let resource = Resource {
id: resource_id,
local_path: PathBuf::from(&manifest_entry.destination),
entry: ResourceEntry {
path: manifest_entry.destination.clone(),
hash: object_id.clone(),
size: manifest_entry.bytes,
resource_type,
address: None,
dependencies: Vec::new(),
crc: None,
},
metadata,
};
if let Err(error) = self.resources.add(resource).await {
let _ = self.cas.remove_reference(&object_id).await;
return Err(error);
}
if let Some(previous) = previous {
if previous.entry.hash != object_id {
match self.cas.remove_reference(&previous.entry.hash).await {
Ok(_) => {}
Err(error) => report.warnings.push(format!(
"旧 CAS 引用清理失败 {}{}",
previous.entry.hash, error
)),
}
}
}
report.imported_count += 1;
count_resource_type(report, resource_type);
Ok(())
}
}
fn parse_entries_by_destination(
cache: &OfficialParseCache,
) -> BTreeMap<String, Vec<&OfficialParseCacheEntry>> {
let mut by_destination: BTreeMap<String, Vec<&OfficialParseCacheEntry>> = BTreeMap::new();
for entry in cache.entries.values() {
by_destination
.entry(entry.destination.clone())
.or_default()
.push(entry);
}
by_destination
}
fn metadata_for_manifest_entry(
official_release_id: Option<&str>,
manifest_entry: &OfficialDownloadManifestEntry,
parse_entries: &[&OfficialParseCacheEntry],
) -> ResourceMetadata {
let mut metadata = ResourceMetadata {
official_release_id: official_release_id.map(ToOwned::to_owned),
platform: platform_for_destination(&manifest_entry.destination),
bundle_path: Some(manifest_entry.destination.clone()),
..ResourceMetadata::default()
};
let mut archive_entries = BTreeSet::new();
let mut parse_statuses = BTreeSet::new();
let mut unity_versions = BTreeSet::new();
let mut text_assets = BTreeSet::new();
let mut text_unit_formats = BTreeSet::new();
for entry in parse_entries {
if let Some(archive_entry) = entry.archive_entry.as_ref() {
archive_entries.insert(archive_entry.clone());
}
parse_statuses.insert(parse_status_label(entry.status).to_string());
if let Some(unity_version) = entry.unity_version.as_ref() {
unity_versions.insert(unity_version.clone());
}
metadata.unityfs_file_count += entry.file_count as u64;
metadata.serialized_file_count += entry.serialized_file_count as u64;
metadata.text_asset_count += entry.text_asset_count as u64;
metadata.text_unit_count += entry.text_unit_count as u64;
metadata.text_unit_error_count += entry.text_unit_error_count as u64;
text_assets.extend(entry.text_assets.iter().cloned());
text_unit_formats.extend(entry.text_unit_formats.iter().cloned());
}
metadata.archive_entries = archive_entries.into_iter().collect();
metadata.parse_statuses = parse_statuses.into_iter().collect();
metadata.unity_versions = unity_versions.into_iter().collect();
metadata.text_assets = text_assets.into_iter().collect();
metadata.text_unit_formats = text_unit_formats.into_iter().collect();
metadata
}
fn parse_status_label(status: OfficialParseStatus) -> &'static str {
match status {
OfficialParseStatus::Parsed => "parsed",
OfficialParseStatus::SkippedUnsupported => "skipped_unsupported",
OfficialParseStatus::Failed => "failed",
}
}
fn platform_for_destination(destination: &str) -> Option<String> {
let normalized = destination.replace('\\', "/").to_ascii_lowercase();
if normalized.contains("windows") || normalized.contains("/win/") {
Some("windows".to_string())
} else if normalized.contains("android") {
Some("android".to_string())
} else {
None
}
}
fn release_id_from_root(release_root: &Path) -> Option<String> {
release_root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty() && *name != "current")
.map(ToOwned::to_owned)
}
fn resource_id_for_destination(destination: &str) -> String {
format!("official/{}", destination.replace('\\', "/"))
}
fn resource_type_for_path(path: &str) -> ResourceType {
let normalized = path.replace('\\', "/").to_ascii_lowercase();
if normalized.ends_with(".bundle") || normalized.ends_with(".unity3d") {
ResourceType::AssetBundle
} else if normalized.contains("tablebundles/") {
ResourceType::TableBundle
} else if normalized.contains("textassets/")
|| matches!(
normalized.rsplit('.').next(),
Some("txt" | "csv" | "json" | "xml" | "yaml" | "yml")
)
{
ResourceType::TextAsset
} else if normalized.contains("mediaresources/")
|| matches!(
normalized.rsplit('.').next(),
Some("acb" | "awb" | "jpg" | "jpeg" | "mp3" | "mp4" | "ogg" | "png" | "wav" | "webp")
)
{
ResourceType::Media
} else if normalized.contains("catalog") || normalized.ends_with(".hash") {
ResourceType::Manifest
} else {
ResourceType::Other
}
}
fn count_resource_type(report: &mut OfficialReleaseImportReport, resource_type: ResourceType) {
match resource_type {
ResourceType::AssetBundle => report.asset_bundle_count += 1,
ResourceType::TextAsset => report.text_asset_count += 1,
ResourceType::TableBundle => report.table_count += 1,
ResourceType::Media => report.media_count += 1,
ResourceType::Manifest | ResourceType::Other => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{FileSystemCasRepository, InMemoryResourceRepository};
use crate::{
OfficialParseSourceFingerprint, OfficialParseSourceKind, OFFICIAL_PARSE_CACHE_VERSION,
};
use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository};
use std::collections::BTreeMap;
use tempfile::TempDir;
fn write_manifest(root: &Path, destination: &str, bytes: &[u8]) {
let path = root.join(destination);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, bytes).unwrap();
let url = format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialDownloadManifestEntry {
url: url.clone(),
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
};
let manifest = serde_json::json!({
"version": 1,
"entries": BTreeMap::from([(url, entry)]),
});
std::fs::write(
root.join("official-download-manifest.json"),
serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
}
fn write_parse_cache(root: &Path, destination: &str, bytes: &[u8]) {
let source_url =
format!("https://prod-clientpatch.bluearchiveyostar.com/r93/{destination}");
let entry = OfficialParseCacheEntry {
key: format!("direct:{source_url}"),
source_url: source_url.clone(),
destination: destination.to_string(),
archive_entry: None,
source_kind: OfficialParseSourceKind::DirectBundle,
fingerprint: OfficialParseSourceFingerprint {
source_url,
destination: destination.to_string(),
bytes: bytes.len() as u64,
blake3: blake3::hash(bytes).to_hex().to_string(),
},
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some("2021.3.56f2".to_string()),
file_count: 1,
serialized_file_count: 1,
text_asset_count: 1,
text_assets: vec!["Scenario".to_string()],
serialized_parse_error_count: 0,
text_unit_count: 2,
text_unit_formats: vec!["json".to_string(), "plain".to_string()],
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
error: None,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: 123,
summary: OfficialParseSummary {
manifest_entry_count: 1,
cache_entry_count: 1,
candidate_file_count: 1,
zip_entry_count: 0,
skipped_unchanged_count: 0,
parsed_bundle_count: 1,
unsupported_count: 0,
failed_count: 0,
text_asset_count: 1,
text_unit_count: 2,
skipped_binary_text_asset_count: 0,
text_unit_error_count: 1,
},
entries: BTreeMap::from([(entry.key.clone(), entry)]),
};
std::fs::write(
root.join("official-parse-cache.json"),
serde_json::to_vec(&cache).unwrap(),
)
.unwrap();
}
#[tokio::test]
async fn imports_verified_release_idempotently() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TableBundles/TableCatalog.bytes", b"catalog");
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let config = OfficialReleaseImportConfig::new(temp.path());
let first = service.import_release(&config).await.unwrap();
let second = service.import_release(&config).await.unwrap();
assert_eq!(first.imported_count, 1);
assert_eq!(second.imported_count, 0);
assert_eq!(second.unchanged_count, 1);
assert_eq!(
resources
.count(ResourceQuery::by_type(ResourceType::TableBundle))
.await
.unwrap(),
1
);
let id = resource_id_for_destination("TableBundles/TableCatalog.bytes");
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
cas.get_reference_count(&resource.entry.hash).await.unwrap(),
1
);
}
#[tokio::test]
async fn imports_parse_cache_metadata_into_resource_index() {
let temp = TempDir::new().unwrap();
let destination = "Windows/Bundles/scenario.bundle";
let bytes = b"bundle-bytes";
write_manifest(temp.path(), destination, bytes);
write_parse_cache(temp.path(), destination, bytes);
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let report = service
.import_release(
&OfficialReleaseImportConfig::new(temp.path())
.with_official_release_id("release-current"),
)
.await
.unwrap();
assert_eq!(report.imported_count, 1);
assert_eq!(report.parse_summary.as_ref().unwrap().text_unit_count, 2);
let id = resource_id_for_destination(destination);
let resource = resources.find_by_id(&id).await.unwrap();
assert_eq!(
resource.metadata.official_release_id.as_deref(),
Some("release-current")
);
assert_eq!(resource.metadata.platform.as_deref(), Some("windows"));
assert_eq!(resource.metadata.bundle_path.as_deref(), Some(destination));
assert_eq!(resource.metadata.parse_statuses, vec!["parsed".to_string()]);
assert_eq!(
resource.metadata.unity_versions,
vec!["2021.3.56f2".to_string()]
);
assert_eq!(resource.metadata.text_assets, vec!["Scenario".to_string()]);
assert_eq!(resource.metadata.text_unit_count, 2);
assert_eq!(
resource.metadata.text_unit_formats,
vec!["json".to_string(), "plain".to_string()]
);
assert_eq!(resource.metadata.text_unit_error_count, 1);
}
#[tokio::test]
async fn rejects_manifest_hash_mismatch_before_cas_write() {
let temp = TempDir::new().unwrap();
write_manifest(temp.path(), "TextAssets/dialogue.txt", b"original");
let path = temp.path().join("TextAssets/dialogue.txt");
std::fs::write(&path, b"tampered").unwrap();
let cas = FileSystemCasRepository::new(temp.path().join("cas"));
let resources = InMemoryResourceRepository::new();
let service = OfficialReleaseImportService::new(&cas, &resources);
let error = service
.import_release(&OfficialReleaseImportConfig::new(temp.path()))
.await
.unwrap_err();
assert!(error.to_string().contains("BLAKE3"));
assert_eq!(resources.count(ResourceQuery::all()).await.unwrap(), 0);
}
}