//! Official resource change sets and translation handoff files. //! //! This module is intentionally file-based. Official update generates the //! durable change set after a new release has been fully downloaded and //! verified; parser and translation modules can then consume the same immutable //! handoff without depending on daemon internals. use crate::official_download::{ read_download_manifest_at, OfficialDownloadManifest, OfficialDownloadManifestEntry, }; use crate::path_security::{ ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic, STATE_FILE_MODE, }; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; /// Current resource-change-set schema version. pub const OFFICIAL_RESOURCE_CHANGES_VERSION: u32 = 1; /// File name stored under a published official release root. pub const OFFICIAL_RESOURCE_CHANGES_FILE: &str = "official-resource-changes.json"; /// Current Crowdin handoff schema version. pub const CROWDIN_TRANSLATION_HANDOFF_VERSION: u32 = 1; /// File name stored under a published official release root. pub const CROWDIN_TRANSLATION_HANDOFF_FILE: &str = "crowdin-translation-handoff.json"; /// Change kind for one official resource destination. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OfficialResourceChangeKind { /// Destination did not exist in the previous complete release. Added, /// Destination exists in both releases, but verified size or BLAKE3 changed. Modified, /// Destination existed in the previous release but is absent from the new one. Removed, } impl OfficialResourceChangeKind { /// Returns the stable JSON/RPC label for the change kind. pub fn as_str(self) -> &'static str { match self { Self::Added => "added", Self::Modified => "modified", Self::Removed => "removed", } } /// Returns true when the changed resource should be offered to parser and /// translation modules. pub fn is_incremental_candidate(self) -> bool { matches!(self, Self::Added | Self::Modified) } } /// Verified manifest attributes for one official resource. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceDescriptor { /// Official URL from the download manifest. pub url: String, /// Relative path under the official release root. pub destination: String, /// Verified byte count from the download manifest. pub bytes: u64, /// Verified BLAKE3 digest from the download manifest. pub blake3: String, } impl From<&OfficialDownloadManifestEntry> for OfficialResourceDescriptor { fn from(entry: &OfficialDownloadManifestEntry) -> Self { Self { url: entry.url.clone(), destination: entry.destination.clone(), bytes: entry.bytes, blake3: entry.blake3.clone(), } } } /// One resource-level change between two complete official releases. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceChange { /// Relative path used as the stable comparison key. pub destination: String, /// Change kind for this destination. pub kind: OfficialResourceChangeKind, /// Previous release descriptor, when the destination existed before. pub previous: Option, /// Current release descriptor, when the destination exists now. pub current: Option, /// Whether parser modules should inspect this resource in incremental mode. pub parse_candidate: bool, /// Whether translation modules should enqueue this resource in incremental mode. pub translation_candidate: bool, } impl OfficialResourceChange { fn new( destination: String, kind: OfficialResourceChangeKind, previous: Option, current: Option, ) -> Self { let is_candidate = kind.is_incremental_candidate(); Self { destination, kind, previous, current, parse_candidate: is_candidate, translation_candidate: is_candidate, } } } /// Aggregate counters for one official resource change set. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceChangeSummary { /// Whether a previous complete release manifest was available. pub previous_manifest_present: bool, /// Number of entries in the previous release manifest. pub previous_manifest_entry_count: usize, /// Number of entries in the current release manifest. pub current_manifest_entry_count: usize, /// Number of newly added destinations. pub added_count: usize, /// Number of destinations whose verified bytes or BLAKE3 changed. pub modified_count: usize, /// Number of destinations removed from the current release. pub removed_count: usize, /// Number of resources to offer to parser modules. pub parse_candidate_count: usize, /// Number of resources to offer to translation modules. pub translation_candidate_count: usize, } /// Durable comparison result between a previous complete official release and /// the newly published official release. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceChangeSet { /// Change-set schema version. #[serde(default = "default_resource_changes_version")] pub change_set_version: u32, /// Current official release ID. pub official_release_id: String, /// Previous official release ID, when known. pub previous_release_id: Option, /// Generation time as Unix seconds. pub generated_unix_seconds: u64, /// Previous complete release root, when available. pub previous_resource_root: Option, /// Current complete release root. pub current_resource_root: PathBuf, /// Aggregate counters. pub summary: OfficialResourceChangeSummary, /// Stable, destination-sorted list of changed resources. pub changes: Vec, } impl OfficialResourceChangeSet { /// Builds a change set from already loaded manifests. pub fn from_manifests( official_release_id: impl Into, previous_release_id: Option, previous_resource_root: Option, current_resource_root: PathBuf, previous_manifest: Option<&OfficialDownloadManifest>, current_manifest: &OfficialDownloadManifest, ) -> Self { let previous_by_destination = previous_manifest .map(entries_by_destination) .unwrap_or_default(); let current_by_destination = entries_by_destination(current_manifest); let destinations = previous_by_destination .keys() .chain(current_by_destination.keys()) .cloned() .collect::>(); let mut changes = Vec::new(); let mut summary = OfficialResourceChangeSummary { previous_manifest_present: previous_manifest.is_some(), previous_manifest_entry_count: previous_manifest .map(|manifest| manifest.entries.len()) .unwrap_or(0), current_manifest_entry_count: current_manifest.entries.len(), ..OfficialResourceChangeSummary::default() }; for destination in destinations { match ( previous_by_destination.get(&destination), current_by_destination.get(&destination), ) { (None, Some(current)) => { summary.added_count += 1; changes.push(OfficialResourceChange::new( destination, OfficialResourceChangeKind::Added, None, Some((*current).into()), )); } (Some(previous), Some(current)) if content_changed(previous, current) => { summary.modified_count += 1; changes.push(OfficialResourceChange::new( destination, OfficialResourceChangeKind::Modified, Some((*previous).into()), Some((*current).into()), )); } (Some(previous), None) => { summary.removed_count += 1; changes.push(OfficialResourceChange::new( destination, OfficialResourceChangeKind::Removed, Some((*previous).into()), None, )); } _ => {} } } summary.parse_candidate_count = changes .iter() .filter(|change| change.parse_candidate) .count(); summary.translation_candidate_count = changes .iter() .filter(|change| change.translation_candidate) .count(); Self { change_set_version: OFFICIAL_RESOURCE_CHANGES_VERSION, official_release_id: official_release_id.into(), previous_release_id, generated_unix_seconds: unix_seconds_now(), previous_resource_root, current_resource_root, summary, changes, } } /// Returns the resources that parser modules should inspect for /// incremental work. pub fn parse_candidates(&self) -> Vec<&OfficialResourceChange> { self.changes .iter() .filter(|change| change.parse_candidate) .collect() } /// Returns the resources that translation modules should enqueue. pub fn translation_candidates(&self) -> Vec<&OfficialResourceChange> { self.changes .iter() .filter(|change| change.translation_candidate) .collect() } } /// Provider reserved for translation handoff consumers. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TranslationHandoffProvider { /// Crowdin provider. The handoff file does not make a network request. Crowdin, } impl TranslationHandoffProvider { /// Returns the stable provider label. pub fn as_str(self) -> &'static str { match self { Self::Crowdin => "crowdin", } } } /// Status of a generated translation handoff. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TranslationHandoffStatus { /// The handoff was written locally and is waiting for a translation worker. QueuedOffline, } impl TranslationHandoffStatus { /// Returns the stable status label. pub fn as_str(self) -> &'static str { match self { Self::QueuedOffline => "queued_offline", } } } /// One resource entry queued for translation-provider processing. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TranslationHandoffResource { /// Relative path under the official release root. pub destination: String, /// Change kind that caused this resource to be queued. pub kind: OfficialResourceChangeKind, /// Current official URL. pub url: String, /// Verified byte count. pub bytes: u64, /// Verified BLAKE3 digest. pub blake3: String, } /// Crowdin-ready local queue file for added or modified official resources. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CrowdinTranslationHandoff { /// Handoff schema version. #[serde(default = "default_crowdin_handoff_version")] pub handoff_version: u32, /// Translation provider reserved for this queue. pub provider: TranslationHandoffProvider, /// Current queue status. pub status: TranslationHandoffStatus, /// Current official release ID. pub official_release_id: String, /// Previous official release ID, when known. pub previous_release_id: Option, /// Generation time as Unix seconds. pub generated_unix_seconds: u64, /// Number of queued resources. pub resource_count: usize, /// Added or modified resources to pass into parsing/translation workers. pub resources: Vec, } impl CrowdinTranslationHandoff { /// Builds a local Crowdin handoff from a resource change set. pub fn from_change_set(change_set: &OfficialResourceChangeSet) -> Self { let resources = change_set .translation_candidates() .into_iter() .filter_map(|change| { let current = change.current.as_ref()?; Some(TranslationHandoffResource { destination: change.destination.clone(), kind: change.kind, url: current.url.clone(), bytes: current.bytes, blake3: current.blake3.clone(), }) }) .collect::>(); Self { handoff_version: CROWDIN_TRANSLATION_HANDOFF_VERSION, provider: TranslationHandoffProvider::Crowdin, status: TranslationHandoffStatus::QueuedOffline, official_release_id: change_set.official_release_id.clone(), previous_release_id: change_set.previous_release_id.clone(), generated_unix_seconds: unix_seconds_now(), resource_count: resources.len(), resources, } } } /// Generates a change set for two complete release roots and writes both the /// change set and the Crowdin handoff under the current release root. pub fn write_official_resource_change_handoff( previous_resource_root: Option<&Path>, current_resource_root: &Path, official_release_id: &str, previous_release_id: Option, ) -> Result { let current_manifest = read_download_manifest_at(current_resource_root)?.ok_or_else(|| { format!( "缺少当前官方下载 manifest,无法生成资源变更集:{}", current_resource_root.display() ) })?; let previous_manifest = match previous_resource_root { Some(root) => read_download_manifest_at(root)?, None => None, }; let previous_manifest_root = previous_manifest .as_ref() .and(previous_resource_root) .map(Path::to_path_buf); let previous_release_id = previous_manifest.as_ref().and(previous_release_id); let change_set = OfficialResourceChangeSet::from_manifests( official_release_id, previous_release_id, previous_manifest_root, current_resource_root.to_path_buf(), previous_manifest.as_ref(), ¤t_manifest, ); write_resource_change_set_at(current_resource_root, &change_set)?; let handoff = CrowdinTranslationHandoff::from_change_set(&change_set); write_crowdin_translation_handoff_at(current_resource_root, &handoff)?; Ok(OfficialResourceChangeHandoffReport { change_set_path: current_resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE), crowdin_handoff_path: current_resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE), summary: change_set.summary, }) } /// Paths and summary produced after writing a resource-change handoff. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OfficialResourceChangeHandoffReport { /// Path to `official-resource-changes.json`. pub change_set_path: PathBuf, /// Path to `crowdin-translation-handoff.json`. pub crowdin_handoff_path: PathBuf, /// Aggregate change counters. pub summary: OfficialResourceChangeSummary, } /// Reads a generated resource change set from a release root. pub fn read_resource_change_set_at( resource_root: &Path, ) -> Result, String> { let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE); let Some(bytes) = read_file_no_symlink(&path, "官方资源变更集")? else { return Ok(None); }; let change_set: OfficialResourceChangeSet = serde_json::from_slice(&bytes) .map_err(|error| format!("解析官方资源变更集失败 {}:{error}", path.display()))?; if change_set.change_set_version != OFFICIAL_RESOURCE_CHANGES_VERSION { return Err(format!( "不支持的官方资源变更集版本 {},文件 {}", change_set.change_set_version, path.display() )); } Ok(Some(change_set)) } /// Writes a generated resource change set under a release root. pub fn write_resource_change_set_at( resource_root: &Path, change_set: &OfficialResourceChangeSet, ) -> Result<(), String> { let path = resource_root.join(OFFICIAL_RESOURCE_CHANGES_FILE); ensure_path_within_root(resource_root, &path)?; ensure_safe_file_target(resource_root, &path, "官方资源变更集")?; let bytes = serde_json::to_vec_pretty(change_set) .map_err(|error| format!("序列化官方资源变更集失败:{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方资源变更集") } /// Writes a generated Crowdin handoff under a release root. pub fn write_crowdin_translation_handoff_at( resource_root: &Path, handoff: &CrowdinTranslationHandoff, ) -> Result<(), String> { let path = resource_root.join(CROWDIN_TRANSLATION_HANDOFF_FILE); ensure_path_within_root(resource_root, &path)?; ensure_safe_file_target(resource_root, &path, "Crowdin 翻译 handoff")?; let bytes = serde_json::to_vec_pretty(handoff) .map_err(|error| format!("序列化 Crowdin 翻译 handoff 失败:{error}"))?; write_file_atomic(&path, &bytes, STATE_FILE_MODE, "Crowdin 翻译 handoff") } fn entries_by_destination( manifest: &OfficialDownloadManifest, ) -> BTreeMap { manifest .entries .values() .map(|entry| (entry.destination.clone(), entry)) .collect() } fn content_changed( previous: &OfficialDownloadManifestEntry, current: &OfficialDownloadManifestEntry, ) -> bool { previous.bytes != current.bytes || previous.blake3 != current.blake3 } fn unix_seconds_now() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs() } fn default_resource_changes_version() -> u32 { OFFICIAL_RESOURCE_CHANGES_VERSION } fn default_crowdin_handoff_version() -> u32 { CROWDIN_TRANSLATION_HANDOFF_VERSION } #[cfg(test)] mod tests { use super::*; fn manifest(entries: &[(&str, &str, &[u8])]) -> OfficialDownloadManifest { let mut manifest = OfficialDownloadManifest::default(); for (url, destination, bytes) in entries { manifest.entries.insert( (*url).to_string(), OfficialDownloadManifestEntry { url: (*url).to_string(), destination: (*destination).to_string(), bytes: bytes.len() as u64, blake3: blake3::hash(bytes).to_hex().to_string(), }, ); } manifest } fn write_manifest(root: &Path, manifest: &OfficialDownloadManifest) { std::fs::create_dir_all(root).unwrap(); std::fs::write( root.join("official-download-manifest.json"), serde_json::to_vec(manifest).unwrap(), ) .unwrap(); } #[test] fn change_set_classifies_added_modified_and_removed_resources() { let previous = manifest(&[ ("https://old/a", "TableBundles/a.bytes", b"old-a"), ("https://old/b", "TableBundles/b.bytes", b"same"), ("https://old/c", "TableBundles/c.bytes", b"removed"), ( "https://old/u", "TableBundles/url-only.bytes", b"same-url-only", ), ]); let current = manifest(&[ ("https://new/a", "TableBundles/a.bytes", b"new-a"), ("https://new/b", "TableBundles/b.bytes", b"same"), ("https://new/d", "TableBundles/d.bytes", b"added"), ( "https://changed-host/u", "TableBundles/url-only.bytes", b"same-url-only", ), ]); let change_set = OfficialResourceChangeSet::from_manifests( "release-new", Some("release-old".to_string()), Some(PathBuf::from("/previous")), PathBuf::from("/current"), Some(&previous), ¤t, ); assert_eq!(change_set.summary.added_count, 1); assert_eq!(change_set.summary.modified_count, 1); assert_eq!(change_set.summary.removed_count, 1); assert_eq!(change_set.summary.parse_candidate_count, 2); assert_eq!(change_set.summary.translation_candidate_count, 2); let destinations = change_set .translation_candidates() .into_iter() .map(|change| change.destination.as_str()) .collect::>(); assert_eq!( destinations, vec!["TableBundles/a.bytes", "TableBundles/d.bytes"] ); } #[test] fn first_release_treats_all_current_resources_as_added() { let current = manifest(&[ ("https://new/a", "a.bundle", b"a"), ("https://new/b", "b.bundle", b"b"), ]); let change_set = OfficialResourceChangeSet::from_manifests( "release-new", None, None, PathBuf::from("/current"), None, ¤t, ); assert!(!change_set.summary.previous_manifest_present); assert_eq!(change_set.summary.added_count, 2); assert_eq!(change_set.summary.translation_candidate_count, 2); assert_eq!(change_set.changes.len(), 2); } #[test] fn crowdin_handoff_excludes_removed_resources() { let previous = manifest(&[("https://old/a", "a.bundle", b"a")]); let current = manifest(&[("https://new/b", "b.bundle", b"b")]); let change_set = OfficialResourceChangeSet::from_manifests( "release-new", Some("release-old".to_string()), None, PathBuf::from("/current"), Some(&previous), ¤t, ); let handoff = CrowdinTranslationHandoff::from_change_set(&change_set); assert_eq!(handoff.provider, TranslationHandoffProvider::Crowdin); assert_eq!(handoff.status, TranslationHandoffStatus::QueuedOffline); assert_eq!(handoff.resource_count, 1); assert_eq!(handoff.resources[0].destination, "b.bundle"); } #[test] fn write_handoff_persists_change_set_and_crowdin_queue() { let temp = tempfile::TempDir::new().unwrap(); let previous_root = temp.path().join("previous"); let current_root = temp.path().join("current"); write_manifest( &previous_root, &manifest(&[("https://old/a", "a.bundle", b"old")]), ); write_manifest( ¤t_root, &manifest(&[ ("https://new/a", "a.bundle", b"new"), ("https://new/b", "b.bundle", b"added"), ]), ); let report = write_official_resource_change_handoff( Some(&previous_root), ¤t_root, "release-new", Some("release-old".to_string()), ) .unwrap(); assert_eq!(report.summary.modified_count, 1); assert_eq!(report.summary.added_count, 1); assert!(report.change_set_path.exists()); assert!(report.crowdin_handoff_path.exists()); let change_set = read_resource_change_set_at(¤t_root).unwrap().unwrap(); assert_eq!(change_set.summary.translation_candidate_count, 2); } }