fix(release): 完善发布事务和状态隔离
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s

This commit is contained in:
2026-09-12 17:50:09 +08:00
parent 30d1cd77e8
commit f2c20367a6
4 changed files with 208 additions and 23 deletions
@@ -242,7 +242,7 @@ trusted 和 release/TextUnit/provider/run provenance`translation.tasks` 优
11. 下载、manifest、本地 BLAKE3、ZIP 和官方 `.hash` 校验完成后写入新的 snapshot,并在 staging 中写入 `official-launcher-bootstrap.json`(若本轮启用 `--auto-discover`)。 11. 下载、manifest、本地 BLAKE3、ZIP 和官方 `.hash` 校验完成后写入新的 snapshot,并在 staging 中写入 `official-launcher-bootstrap.json`(若本轮启用 `--auto-discover`)。
12. 将 staging rename 为 `<output>/versions/<id>`,再原子替换 `<output>/current` symlink 指向该 versioned 目录。 12. 将 staging rename 为 `<output>/versions/<id>`,再原子替换 `<output>/current` symlink 指向该 versioned 目录。
13. 发布完成后先对比上一完整 release 和当前 release 的 `official-download-manifest.json`,写出 `official-resource-changes.json``crowdin-translation-handoff.json`。同一 destination 只有 size 或 BLAKE3 变化才算 modified;新增+变更资源进入解析/翻译 handoff,删除资源只进入差异记录。当前只预留 Crowdin 本地 handoff,不发外部 API 请求。 13. 发布完成后先对比上一完整 release 和当前 release 的 `official-download-manifest.json`,写出 `official-resource-changes.json``crowdin-translation-handoff.json`。同一 destination 只有 size 或 BLAKE3 变化才算 modified;新增+变更资源进入解析/翻译 handoff,删除资源只进入差异记录。当前只预留 Crowdin 本地 handoff,不发外部 API 请求。
14. 随后刷新 active release 下的 `official-parse-cache.json``official-textunit-index.json`,并从 Added/Modified 资源、parse cache 与 TextUnit 明细索引派生 `official-textunit-tasks.json``crowdin-textunit-queue.json` 和版本化的 `translation-tasks.sqlite`up-to-date 轮询在已有有效解析缓存、TextUnit 明细索引和 TextUnit 队列时只读取摘要,不重复解析,重新同步队列时保留已有 worker 状态。 14. 随后刷新 active release 下的 `official-parse-cache.json``official-textunit-index.json`,并从 Added/Modified 资源、parse cache 与 TextUnit 明细索引派生 `official-textunit-tasks.json``crowdin-textunit-queue.json` 和版本化的 `translation-tasks.sqlite`up-to-date 轮询在已有有效解析缓存、TextUnit 明细索引和 TextUnit 队列时只读取摘要,不重复解析,重新同步队列时保留已有 worker 状态。历史 release 复用只允许不可变资源 payload/sidecar 硬链接;download manifest、snapshot、parse/textunit cache、queue、handoff、bootstrap、CAS reuse references 以及 `translation-tasks.sqlite`、WAL/SHM 都必须独立复制,不能共享可变 inode。
15. 若启用 `--import-repository`,已校验 release 会被导入 CAS + `ResourceRepository`,并可经 `resource.index` 查询。历史 release 候选失效时,已有 CAS 对象会先经过完整性和元数据校验,再增加 release 引用并原子物化;当前 release 在 `official-cas-reuse-references.json` 中记录引用,staging/release 清理时递减,失败则回退网络并保留诊断。 15. 若启用 `--import-repository`,已校验 release 会被导入 CAS + `ResourceRepository`,并可经 `resource.index` 查询。历史 release 候选失效时,已有 CAS 对象会先经过完整性和元数据校验,再增加 release 引用并原子物化;当前 release 在 `official-cas-reuse-references.json` 中记录引用,staging/release 清理时递减,失败则回退网络并保留诊断。
16. 官方同步报告默认给出 `localized_release_status=not_localized`,表示原版资源已发布、汉化资源未发布;generic manifest 驱动的 Binary/JSON/Text 以及当前支持的 UnityFS TextAsset、TypeTree string field 和 managed-reference string field patch 发布成功并通过 `localized-patch-manifest.json`、current symlink、release ID 及 ZIP 内层最终重解析校验后,`localized.status` 才返回 `localized`,表示原版和汉化两套资源都已发布。`localized.status` 分开返回 `patch_manifest_contract_status``artifact_integrity_status`state/current/identity 存在但文件被截断或手工修改时返回 `localized.degraded`,只读检查不回滚、不删除、不修复。`translation.proofread` 只会把 workflow 标记成 `manual_proofreading` / `translation.manual_proofreading`,不会回退已发布汉化 release 的发布状态。 16. 官方同步报告默认给出 `localized_release_status=not_localized`,表示原版资源已发布、汉化资源未发布;generic manifest 驱动的 Binary/JSON/Text 以及当前支持的 UnityFS TextAsset、TypeTree string field 和 managed-reference string field patch 发布成功并通过 `localized-patch-manifest.json`、current symlink、release ID 及 ZIP 内层最终重解析校验后,`localized.status` 才返回 `localized`,表示原版和汉化两套资源都已发布。`localized.status` 分开返回 `patch_manifest_contract_status``artifact_integrity_status`state/current/identity 存在但文件被截断或手工修改时返回 `localized.degraded`,只读检查不回滚、不删除、不修复。`translation.proofread` 只会把 workflow 标记成 `manual_proofreading` / `translation.manual_proofreading`,不会回退已发布汉化 release 的发布状态。
+154 -2
View File
@@ -43,7 +43,7 @@ pub const LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING: &str = "manual_proof
pub const LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING_LABEL: &str = "人工校对中"; pub const LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING_LABEL: &str = "人工校对中";
#[derive(Debug)] #[derive(Debug)]
struct LocalizedOutputLock { pub(crate) struct LocalizedOutputLock {
file: std::fs::File, file: std::fs::File,
} }
@@ -84,6 +84,10 @@ impl LocalizedOutputLock {
} }
} }
pub(crate) fn acquire_localized_output_lock(root: &Path) -> anyhow::Result<LocalizedOutputLock> {
LocalizedOutputLock::acquire(root)
}
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
struct LocalizedReleaseTransaction { struct LocalizedReleaseTransaction {
version: u32, version: u32,
@@ -834,6 +838,7 @@ impl LocalizedPatchService {
) )
})?; })?;
verify_localized_release_files(&version_path, &manifest)?; verify_localized_release_files(&version_path, &manifest)?;
verify_localized_distribution_manifest_at(&version_path, &current_release_id)?;
if manifest.localized_release_id != current_release_id { if manifest.localized_release_id != current_release_id {
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
"manifest release={} 与当前状态 release={} 不一致", "manifest release={} 与当前状态 release={} 不一致",
@@ -891,6 +896,10 @@ impl LocalizedPatchService {
})?, })?,
); );
verify_localized_release_files(&previous_path, restored_manifest.as_ref().unwrap())?; verify_localized_release_files(&previous_path, restored_manifest.as_ref().unwrap())?;
verify_localized_distribution_manifest_at(
&previous_path,
restored_release_id.as_deref().unwrap_or_default(),
)?;
} }
let previous_state_bytes = read_file_no_symlink(&state_path, "汉化版本状态") let previous_state_bytes = read_file_no_symlink(&state_path, "汉化版本状态")
@@ -2200,6 +2209,54 @@ fn verify_localized_release_files(
Ok(()) Ok(())
} }
fn verify_localized_distribution_manifest_at(
version_path: &Path,
localized_release_id: &str,
) -> anyhow::Result<()> {
let path = version_path.join(LOCALIZED_DISTRIBUTION_MANIFEST_FILE);
let Some(bytes) = read_file_no_symlink(&path, "localized distribution manifest")
.map_err(anyhow::Error::msg)?
else {
// 旧 localized release 可能在 distribution manifest 引入前发布;
// 保留兼容 rollback,新的 publish 仍会在有 official manifest 时生成它。
return Ok(());
};
let manifest: LocalizedDistributionManifest = serde_json::from_slice(&bytes)?;
if manifest.version != 1 || manifest.localized_release_id != localized_release_id {
return Err(anyhow::anyhow!(
"localized distribution manifest identity 不一致:expected={} actual={}",
localized_release_id,
manifest.localized_release_id
));
}
let mut destinations = BTreeSet::new();
for entry in &manifest.entries {
if !destinations.insert(entry.destination.as_str()) {
return Err(anyhow::anyhow!(
"localized distribution manifest 存在重复 destination{}",
entry.destination
));
}
let file_path = version_path.join(&entry.destination);
ensure_path_within_root(version_path, &file_path).map_err(anyhow::Error::msg)?;
ensure_safe_file_target(version_path, &file_path, "localized distribution 文件")
.map_err(anyhow::Error::msg)?;
let file_bytes = fs::read(&file_path)?;
let actual = blake3::hash(&file_bytes).to_hex().to_string();
if file_bytes.len() as u64 != entry.bytes || actual != entry.blake3 {
return Err(anyhow::anyhow!(
"localized distribution 文件完整性失败 {}expected bytes={} blake3={} actual bytes={} blake3={}",
entry.destination,
entry.bytes,
entry.blake3,
file_bytes.len(),
actual
));
}
}
Ok(())
}
fn write_localized_transaction( fn write_localized_transaction(
localized_output_root: &Path, localized_output_root: &Path,
transaction: &LocalizedReleaseTransaction, transaction: &LocalizedReleaseTransaction,
@@ -2302,7 +2359,9 @@ fn recover_localized_transaction(localized_output_root: &Path) -> anyhow::Result
remove_owned_path(&transaction.version_path)?; remove_owned_path(&transaction.version_path)?;
} else { } else {
if let Some(target) = transaction.previous_current_target.as_deref() { if let Some(target) = transaction.previous_current_target.as_deref() {
ensure_path_within_root(localized_output_root, target).map_err(anyhow::Error::msg)?; let target_path = localized_output_root.join(target);
ensure_path_within_root(localized_output_root, &target_path)
.map_err(anyhow::Error::msg)?;
restore_current_symlink(localized_output_root, &current_path, Some(target))?; restore_current_symlink(localized_output_root, &current_path, Some(target))?;
} else if transaction.operation == "publish" { } else if transaction.operation == "publish" {
restore_current_symlink(localized_output_root, &current_path, None)?; restore_current_symlink(localized_output_root, &current_path, None)?;
@@ -2328,6 +2387,19 @@ fn recover_localized_transaction(localized_output_root: &Path) -> anyhow::Result
remove_localized_transaction(localized_output_root) remove_localized_transaction(localized_output_root)
} }
pub(crate) fn recover_localized_output_transaction(root: &Path) -> anyhow::Result<()> {
recover_localized_transaction(root)
}
pub(crate) fn localized_output_transaction_pending(root: &Path) -> anyhow::Result<bool> {
Ok(read_file_no_symlink(
&root.join(LOCALIZED_TRANSACTION_FILE),
"localized release transaction",
)
.map_err(anyhow::Error::msg)?
.is_some())
}
/// Inspects one localized release without changing state, staging, current or /// Inspects one localized release without changing state, staging, current or
/// any repair target. /// any repair target.
pub fn inspect_localized_release_artifact( pub fn inspect_localized_release_artifact(
@@ -3359,6 +3431,10 @@ fn default_localized_version_state_version() -> u32 {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[cfg(unix)]
use std::sync::mpsc;
#[cfg(unix)]
use std::time::Duration;
use tempfile::TempDir; use tempfile::TempDir;
fn push_u32_be(data: &mut Vec<u8>, value: u32) { fn push_u32_be(data: &mut Vec<u8>, value: u32) {
@@ -3381,6 +3457,82 @@ mod tests {
data.extend_from_slice(&value.to_le_bytes()); data.extend_from_slice(&value.to_le_bytes());
} }
#[cfg(unix)]
#[test]
fn localized_output_lock_serializes_independent_handles() {
let temp = TempDir::new().unwrap();
let first = LocalizedOutputLock::acquire(temp.path()).unwrap();
let (sender, receiver) = mpsc::channel();
let root = temp.path().to_path_buf();
let worker = std::thread::spawn(move || {
let second = LocalizedOutputLock::acquire(&root).unwrap();
sender.send(()).unwrap();
drop(second);
});
assert!(receiver.recv_timeout(Duration::from_millis(50)).is_err());
drop(first);
receiver.recv_timeout(Duration::from_secs(1)).unwrap();
worker.join().unwrap();
}
#[cfg(unix)]
#[test]
fn interrupted_publish_transaction_is_recovered_before_next_mutation() {
use std::os::unix::fs::symlink;
let temp = TempDir::new().unwrap();
let root = temp.path().join("localized");
let old = root.join(LOCALIZED_VERSIONS_DIR).join("old");
let new = root.join(LOCALIZED_VERSIONS_DIR).join("new");
let staging = root.join(LOCALIZED_STAGING_DIR).join("new");
fs::create_dir_all(&old).unwrap();
fs::create_dir_all(&new).unwrap();
fs::create_dir_all(&staging).unwrap();
symlink(
Path::new(LOCALIZED_VERSIONS_DIR).join("new"),
root.join(LOCALIZED_CURRENT_LINK),
)
.unwrap();
let old_state = LocalizedVersionState {
state_version: LOCALIZED_VERSION_STATE_VERSION,
official_release_id: "official-old".to_string(),
current_release_id: Some("old".to_string()),
status: "localized".to_string(),
translation_workflow_status: None,
updated_unix_seconds: 1,
};
let old_state_bytes = serde_json::to_vec_pretty(&old_state).unwrap();
write_file_atomic(
&root.join(LOCALIZED_VERSION_STATE_FILE),
&old_state_bytes,
STATE_FILE_MODE,
"test state",
)
.unwrap();
write_localized_transaction(
&root,
&LocalizedReleaseTransaction::publish(
"new",
new.clone(),
staging.clone(),
Some(PathBuf::from("versions/old")),
Some(old_state_bytes),
),
)
.unwrap();
write_localized_version_state(&root, &old_state).unwrap();
assert_eq!(
fs::read_link(root.join(LOCALIZED_CURRENT_LINK)).unwrap(),
PathBuf::from("versions/old")
);
assert!(!new.exists());
assert!(!staging.exists());
assert!(!localized_output_transaction_pending(&root).unwrap());
}
fn push_i16_le(data: &mut Vec<u8>, value: i16) { fn push_i16_le(data: &mut Vec<u8>, value: i16) {
data.extend_from_slice(&value.to_le_bytes()); data.extend_from_slice(&value.to_le_bytes());
} }
+46 -20
View File
@@ -3255,9 +3255,21 @@ fn is_release_local_mutable_state(path: &Path) -> bool {
matches!( matches!(
path.file_name().and_then(|name| name.to_str()), path.file_name().and_then(|name| name.to_str()),
Some( Some(
"translation-tasks.sqlite" OFFICIAL_DOWNLOAD_MANIFEST_FILE
| OFFICIAL_SYNC_SNAPSHOT_FILE
| "official-parse-cache.json"
| "official-textunit-index.json"
| "official-textunit-tasks.json"
| "crowdin-textunit-queue.json"
| "official-resource-changes.json"
| "crowdin-translation-handoff.json"
| "translation-tasks.sqlite"
| "translation-tasks.sqlite-wal" | "translation-tasks.sqlite-wal"
| "translation-tasks.sqlite-shm" | "translation-tasks.sqlite-shm"
| "translation-handoff.json"
| OFFICIAL_LAUNCHER_BOOTSTRAP_FILE
| OFFICIAL_LAUNCHER_BOOTSTRAP_PENDING_FILE
| "official-cas-reuse-references.json"
) )
) )
} }
@@ -4443,29 +4455,43 @@ mod tests {
let source = temp.path().join("old"); let source = temp.path().join("old");
let destination = temp.path().join("new"); let destination = temp.path().join("new");
fs::create_dir_all(&source).unwrap(); fs::create_dir_all(&source).unwrap();
fs::write(source.join("translation-tasks.sqlite"), b"old-db").unwrap(); let mutable_files = [
fs::write(source.join("translation-tasks.sqlite-wal"), b"old-wal").unwrap(); OFFICIAL_DOWNLOAD_MANIFEST_FILE,
OFFICIAL_SYNC_SNAPSHOT_FILE,
"official-parse-cache.json",
"official-textunit-index.json",
"official-textunit-tasks.json",
"crowdin-textunit-queue.json",
"official-resource-changes.json",
"crowdin-translation-handoff.json",
"translation-tasks.sqlite",
"translation-tasks.sqlite-wal",
"translation-tasks.sqlite-shm",
"translation-handoff.json",
OFFICIAL_LAUNCHER_BOOTSTRAP_FILE,
OFFICIAL_LAUNCHER_BOOTSTRAP_PENDING_FILE,
"official-cas-reuse-references.json",
];
for (index, name) in mutable_files.iter().enumerate() {
fs::write(source.join(name), format!("old-{index}")).unwrap();
}
fs::write(source.join("immutable.bundle"), b"payload").unwrap(); fs::write(source.join("immutable.bundle"), b"payload").unwrap();
copy_tree_no_symlink(&source, &destination, false).unwrap(); copy_tree_no_symlink(&source, &destination, false).unwrap();
let old_db_inode = fs::metadata(source.join("translation-tasks.sqlite")) for (index, name) in mutable_files.iter().enumerate() {
.unwrap() assert_ne!(
.ino(); fs::metadata(source.join(name)).unwrap().ino(),
let new_db_inode = fs::metadata(destination.join("translation-tasks.sqlite")) fs::metadata(destination.join(name)).unwrap().ino(),
.unwrap() "mutable state unexpectedly hard-linked: {name}"
.ino(); );
assert_ne!(old_db_inode, new_db_inode); fs::write(destination.join(name), format!("new-{index}")).unwrap();
fs::write(destination.join("translation-tasks.sqlite"), b"new-db").unwrap(); assert_eq!(
fs::write(destination.join("translation-tasks.sqlite-wal"), b"new-wal").unwrap(); fs::read(source.join(name)).unwrap(),
assert_eq!( format!("old-{index}").as_bytes(),
fs::read(source.join("translation-tasks.sqlite")).unwrap(), "historical mutable state changed: {name}"
b"old-db" );
); }
assert_eq!(
fs::read(source.join("translation-tasks.sqlite-wal")).unwrap(),
b"old-wal"
);
assert_eq!( assert_eq!(
fs::metadata(source.join("immutable.bundle")).unwrap().ino(), fs::metadata(source.join("immutable.bundle")).unwrap().ino(),
fs::metadata(destination.join("immutable.bundle")) fs::metadata(destination.join("immutable.bundle"))
+7
View File
@@ -439,6 +439,11 @@ fn select_release_distribution_metadata(
if !path.is_dir() { if !path.is_dir() {
return Ok(None); return Ok(None);
} }
if channel == Channel::Localized
&& crate::localized_patch::localized_output_transaction_pending(localized_root)?
{
return Ok(None);
}
let pointer_id = read_managed_current_id(&root.join(current_link), versions_dir); let pointer_id = read_managed_current_id(&root.join(current_link), versions_dir);
let current = current_id.as_deref() == Some(id) && pointer_id.as_deref() == Some(id); let current = current_id.as_deref() == Some(id) && pointer_id.as_deref() == Some(id);
if selected_id.is_none() && !current { if selected_id.is_none() && !current {
@@ -619,6 +624,8 @@ pub fn cleanup_releases(
params: &ReleaseCleanupParams, params: &ReleaseCleanupParams,
unzip_command: &Path, unzip_command: &Path,
) -> anyhow::Result<ReleaseCleanupReport> { ) -> anyhow::Result<ReleaseCleanupReport> {
let _localized_lock = crate::localized_patch::acquire_localized_output_lock(localized_root)?;
crate::localized_patch::recover_localized_output_transaction(localized_root)?;
let plan = build_cleanup_plan(official_root, localized_root, cas_root, unzip_command)?; let plan = build_cleanup_plan(official_root, localized_root, cas_root, unzip_command)?;
if !params.execute { if !params.execute {
return Ok(ReleaseCleanupReport { return Ok(ReleaseCleanupReport {