feat(translation): add Rust Translation Memory and config migration

This commit is contained in:
2026-09-06 22:48:51 +08:00
parent 93f4bc69b3
commit 7d6389806b
18 changed files with 4414 additions and 384 deletions
+6
View File
@@ -4,6 +4,7 @@ pub mod game_client;
pub mod game_version; pub mod game_version;
pub mod resource; pub mod resource;
pub mod translation; pub mod translation;
pub mod translation_memory;
pub use game_client::{ClientStatus, GameClient, GameRegion}; pub use game_client::{ClientStatus, GameClient, GameRegion};
pub use game_version::{GameVersion, UnityVersion}; pub use game_version::{GameVersion, UnityVersion};
@@ -14,3 +15,8 @@ pub use translation::{
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText, ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
TranslationStatus, TranslationStatus,
}; };
pub use translation_memory::{
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
TranslationMemorySourceTrace, TranslationMemorySummary, TranslationMemoryTrustStatus,
};
+228
View File
@@ -0,0 +1,228 @@
//! Translation Memory 领域对象。
use std::collections::BTreeMap;
/// TM 记录的来源类型。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationMemorySourceKind {
/// 来自 provider 输出。
Provider,
/// 来自人工确认。
Manual,
/// 来自外部导入。
Imported,
}
impl TranslationMemorySourceKind {
/// 返回稳定的持久化标签。
pub const fn as_str(&self) -> &'static str {
match self {
Self::Provider => "provider",
Self::Manual => "manual",
Self::Imported => "imported",
}
}
}
/// TM 记录的可信状态。
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationMemoryTrustStatus {
/// 候选记录,不能自动复用。
Candidate,
/// 已确认可信,可在强匹配时自动复用。
Trusted,
/// 已被后续记录取代。
Superseded,
/// 已明确拒绝。
Rejected,
}
impl TranslationMemoryTrustStatus {
/// 返回稳定的持久化标签。
pub const fn as_str(&self) -> &'static str {
match self {
Self::Candidate => "candidate",
Self::Trusted => "trusted",
Self::Superseded => "superseded",
Self::Rejected => "rejected",
}
}
}
/// TM 查询结果的匹配类型。
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationMemoryMatchKind {
/// 原始 source 和上下文都完全匹配,且记录可信,可自动复用。
StrongExact,
/// 原始 source 完全匹配,但上下文不同或不足,不能自动复用。
CandidateExact,
/// 原始 source 匹配,但上下文不兼容,不能自动复用。
SourceOnly,
}
impl TranslationMemoryMatchKind {
/// 返回稳定的查询结果标签。
pub const fn as_str(&self) -> &'static str {
match self {
Self::StrongExact => "strong_exact",
Self::CandidateExact => "candidate_exact",
Self::SourceOnly => "source_only",
}
}
}
/// 稳定的上下文键值。
pub type TranslationMemoryContext = BTreeMap<String, String>;
/// TM 记录的 TextUnit / provider 溯源信息。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TranslationMemorySourceTrace {
/// 源官方 release ID。
pub official_release_id: String,
/// 来源 TextUnit ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unit_id: Option<String>,
/// 来源任务 ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub task_id: Option<String>,
/// 源资源 destination。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub destination: Option<String>,
/// 源 ZIP/archive entry。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Unity serialized file。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree 字段路径。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// TextUnit format。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
/// TextAsset 名称。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset_name: Option<String>,
/// TextUnit 来源类型。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_source_kind: Option<String>,
/// 源 URL。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_url: Option<String>,
}
/// TM 记录的候选输入。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TranslationMemoryDraft {
/// 原始 source text。
pub source_text: String,
/// 稳定上下文。
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub source_context: TranslationMemoryContext,
/// 译文。
pub translated_text: String,
/// 译文来源类型。
pub translation_source_kind: TranslationMemorySourceKind,
/// 源官方 release。
pub official_release_id: String,
/// 原始 TextUnit / provider 溯源。
pub source_trace: TranslationMemorySourceTrace,
/// provider。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
/// provider run。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>,
/// 创建时间。
pub observed_unix_seconds: u64,
}
/// 持久化 TM 记录。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TranslationMemoryEntry {
/// 稳定记录 ID。
pub record_id: String,
/// 原始 source text。
pub source_text: String,
/// source text hash。
pub source_hash: String,
/// 保守归一化后的 source text,仅用于辅助查询。
pub normalized_source_text: String,
/// 稳定上下文。
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub source_context: TranslationMemoryContext,
/// 上下文 hash。
pub source_context_hash: String,
/// 译文。
pub translated_text: String,
/// 译文来源类型。
pub translation_source_kind: TranslationMemorySourceKind,
/// 当前可信状态。
pub trust_status: TranslationMemoryTrustStatus,
/// 源官方 release。
pub official_release_id: String,
/// 原始 TextUnit / provider 溯源。
pub source_trace: TranslationMemorySourceTrace,
/// provider。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
/// provider run。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>,
/// 创建时间。
pub created_unix_seconds: u64,
/// 更新时间。
pub updated_unix_seconds: u64,
/// 可信确认时间。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trusted_unix_seconds: Option<u64>,
/// 可信确认人。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trusted_by: Option<String>,
/// 可信确认说明。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trusted_reason: Option<String>,
/// 该记录替代了哪条记录。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supersedes_record_id: Option<String>,
/// 该记录被哪条记录替代。
#[serde(default, skip_serializing_if = "Option::is_none")]
pub superseded_by_record_id: Option<String>,
}
/// TM 查询结果。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TranslationMemoryMatch {
/// 记录本体。
pub entry: TranslationMemoryEntry,
/// 匹配类型。
pub match_kind: TranslationMemoryMatchKind,
/// 是否允许自动复用。
pub can_auto_reuse: bool,
}
/// TM 仓储摘要。
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TranslationMemorySummary {
/// schema 版本。
pub schema_version: u32,
/// 记录总数。
pub record_count: u64,
/// 可信记录数。
pub trusted_count: u64,
/// 候选记录数。
pub candidate_count: u64,
/// 已替代记录数。
pub superseded_count: u64,
/// 已拒绝记录数。
pub rejected_count: u64,
}
+2
View File
@@ -4,8 +4,10 @@
pub mod cas_repository; pub mod cas_repository;
pub mod resource_repository; pub mod resource_repository;
pub mod translation_memory_repository;
pub mod translation_repository; pub mod translation_repository;
pub use cas_repository::CasRepository; pub use cas_repository::CasRepository;
pub use resource_repository::ResourceRepository; pub use resource_repository::ResourceRepository;
pub use translation_memory_repository::TranslationMemoryRepository;
pub use translation_repository::TranslationRepository; pub use translation_repository::TranslationRepository;
@@ -0,0 +1,47 @@
//! Translation Memory 仓储契约。
use crate::domain::{
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
TranslationMemoryMatch, TranslationMemorySummary,
};
use async_trait::async_trait;
/// 跨 official release 持久化的 Translation Memory 仓储。
///
/// 该契约只描述 V1 的精确查询和明确人工确认。仓储实现不得把
/// `TranslationTaskStatus::Completed` 或 provider 成功隐式解释为 trusted。
#[async_trait]
pub trait TranslationMemoryRepository: Send + Sync {
/// 保存一条 provider/manual/imported 译文候选。
///
/// 相同 source、上下文、译文、来源 release 和来源类型的重复写入必须幂等;
/// 已 trusted 的记录不得被普通候选静默覆盖。
async fn upsert_candidate(
&self,
draft: TranslationMemoryDraft,
) -> crate::Result<TranslationMemoryEntry>;
/// 按原始 source text 和上下文查询精确匹配。
///
/// 实现可以返回 source 归一化后但原文不同的辅助候选,但这类结果不能自动复用。
async fn find_matches(
&self,
source_text: &str,
source_context: &TranslationMemoryContext,
limit: usize,
) -> crate::Result<Vec<TranslationMemoryMatch>>;
/// 显式确认一条记录为 trusted。
async fn confirm(
&self,
record_id: &str,
reviewer: &str,
reason: Option<String>,
) -> crate::Result<TranslationMemoryEntry>;
/// 按稳定记录 ID 读取一条 TM 记录。
async fn find(&self, record_id: &str) -> crate::Result<TranslationMemoryEntry>;
/// 读取数据库和记录统计。
async fn summary(&self) -> crate::Result<TranslationMemorySummary>;
}
File diff suppressed because it is too large Load Diff
+386 -42
View File
@@ -17,9 +17,11 @@ fn parse_with_env(values: &[&str], env: &[(&str, &str)]) -> anyhow::Result<CliOp
.iter() .iter()
.map(|(key, value)| (key.to_string(), value.to_string())) .map(|(key, value)| (key.to_string(), value.to_string()))
.collect(); .collect();
parse_args_with_env(values.iter().map(|value| value.to_string()), move |key| { parse_args_with_env(
map.get(key).cloned() values.iter().map(|value| value.to_string()),
}) move |key| map.get(key).cloned(),
None,
)
} }
#[test] #[test]
@@ -105,7 +107,7 @@ fn env_watch_daemon_only_affect_bare_run() {
#[test] #[test]
fn env_values_do_not_break_status_and_reload_guard() { fn env_values_do_not_break_status_and_reload_guard() {
// .env 提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。 // 配置文件/环境变量提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。
let options = parse_with_env( let options = parse_with_env(
&["bat", "status"], &["bat", "status"],
&[ &[
@@ -222,6 +224,148 @@ fn translation_worker_command_options_are_validated() {
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err()); assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
} }
#[test]
fn translation_memory_commands_parse_and_validate() {
let summary = parse(&["bat", "i18n", "memory", "summary"]).unwrap();
assert_eq!(summary.command, CliCommand::TranslationMemorySummary);
let query = parse(&[
"bat",
"i18n",
"tm",
"query",
"--tm-source-text",
"Hello",
"--tm-context-json",
r#"{"destination":"story.bundle"}"#,
"--limit",
"5",
])
.unwrap();
assert_eq!(query.command, CliCommand::TranslationMemoryQuery);
assert_eq!(
query.translation_memory_source_text.as_deref(),
Some("Hello")
);
let confirm = parse(&[
"bat",
"i18n",
"memory",
"confirm",
"--tm-record-id",
"tm-record",
"--tm-reviewer",
"reviewer",
"--tm-reason",
"accepted",
])
.unwrap();
assert_eq!(confirm.command, CliCommand::TranslationMemoryConfirm);
assert!(parse(&["bat", "i18n", "memory", "query"]).is_err());
assert!(parse(&[
"bat",
"i18n",
"memory",
"confirm",
"--tm-record-id",
"tm-record"
])
.is_err());
}
#[test]
fn restart_reload_accept_translation_worker_startup_options() {
let restart = parse(&[
"bat",
"restart",
"--translation-provider",
"mock",
"--translation-memory-path",
"/tmp/tm.sqlite",
"--worker-concurrency",
"2",
])
.unwrap();
assert_eq!(restart.command, CliCommand::Restart);
assert!(restart.translation_worker_option_explicit);
assert_eq!(
restart.translation_memory_path,
Some(PathBuf::from("/tmp/tm.sqlite"))
);
assert_eq!(restart.worker_concurrency, 2);
assert!(restart.config.auto_discover);
let reload = parse(&["bat", "reload", "--worker-id", "reload-worker"]).unwrap();
assert_eq!(reload.command, CliCommand::Reload);
assert!(reload.translation_worker_option_explicit);
assert_eq!(reload.worker_id.as_deref(), Some("reload-worker"));
assert!(reload.config.auto_discover);
}
#[test]
fn translation_memory_command_options_do_not_apply_to_other_commands() {
let error = parse(&["bat", "restart", "--tm-source-text", "Hello"]).unwrap_err();
assert!(error.to_string().contains("查询/confirm"));
let error = parse(&["bat", "status", "--tm-record-id", "tm-record"]).unwrap_err();
assert!(error.to_string().contains("查询/confirm"));
}
#[test]
fn translation_memory_subcommands_reject_irrelevant_options() {
let error = parse(&[
"bat",
"i18n",
"memory",
"summary",
"--tm-source-text",
"Hello",
])
.unwrap_err();
assert!(error.to_string().contains("summary"));
let error = parse(&["bat", "i18n", "memory", "summary", "--limit", "5"]).unwrap_err();
assert!(error.to_string().contains("summary"));
let error = parse(&[
"bat",
"i18n",
"memory",
"query",
"--tm-source-text",
"Hello",
"--tm-record-id",
"tm-record",
])
.unwrap_err();
assert!(error.to_string().contains("confirm 参数"));
let error = parse(&[
"bat",
"i18n",
"memory",
"query",
"--tm-source-text",
"Hello",
"--offset",
"1",
])
.unwrap_err();
assert!(error.to_string().contains("查询过滤"));
let error = parse(&[
"bat",
"i18n",
"memory",
"confirm",
"--tm-record-id",
"tm-record",
"--tm-reviewer",
"reviewer",
"--tm-context-json",
"{}",
])
.unwrap_err();
assert!(error.to_string().contains("query 参数"));
}
#[test] #[test]
fn translation_worker_env_defaults_apply() { fn translation_worker_env_defaults_apply() {
let options = parse_with_env( let options = parse_with_env(
@@ -229,6 +373,7 @@ fn translation_worker_env_defaults_apply() {
&[ &[
("BAT_TRANSLATION_PROVIDER", "mock"), ("BAT_TRANSLATION_PROVIDER", "mock"),
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"), ("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
("BAT_TRANSLATION_MEMORY_PATH", "/tmp/tm.sqlite"),
("BAT_TRANSLATION_CONCURRENCY", "16"), ("BAT_TRANSLATION_CONCURRENCY", "16"),
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"), ("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
("BAT_TRANSLATION_LEASE_SECONDS", "120"), ("BAT_TRANSLATION_LEASE_SECONDS", "120"),
@@ -244,6 +389,10 @@ fn translation_worker_env_defaults_apply() {
options.translation_fixture, options.translation_fixture,
Some(PathBuf::from("/tmp/fixture.json")) Some(PathBuf::from("/tmp/fixture.json"))
); );
assert_eq!(
options.translation_memory_path,
Some(PathBuf::from("/tmp/tm.sqlite"))
);
assert_eq!(options.worker_concurrency, 16); assert_eq!(options.worker_concurrency, 16);
assert_eq!(options.worker_max_attempts, 5); assert_eq!(options.worker_max_attempts, 5);
assert_eq!(options.worker_lease_seconds, 120); assert_eq!(options.worker_lease_seconds, 120);
@@ -610,6 +759,8 @@ fn translation_workbench_commands_read_update_and_clear_entries() {
translated_text: None, translated_text: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
translated_unix_seconds: None, translated_unix_seconds: None,
review_status: None, review_status: None,
format: Some("plain".to_string()), format: Some("plain".to_string()),
@@ -1030,46 +1181,72 @@ fn cli_download_concurrency_is_preserved_for_daemon_child() {
} }
#[test] #[test]
fn parse_env_line_handles_quotes_and_rejects_bad_keys() { fn config_file_is_applied_before_env_and_cli() {
assert_eq!( let temp = tempfile::TempDir::new().unwrap();
parse_env_line("KEY=value"), let config_path = temp.path().join(super::config_file::CONFIG_FILE_NAME);
Some(("KEY".to_string(), "value".to_string())) std::fs::write(
); &config_path,
assert_eq!( r#"
parse_env_line("KEY=\"quoted value\""), [runtime]
Some(("KEY".to_string(), "quoted value".to_string())) state_dir = '/srv/state'
); output_format = 'json'
assert_eq!(
parse_env_line("KEY='single'"),
Some(("KEY".to_string(), "single".to_string()))
);
assert_eq!(
parse_env_line("BAT_OUTPUT = ./x"),
Some(("BAT_OUTPUT".to_string(), "./x".to_string()))
);
assert_eq!(parse_env_line("no_equals_sign"), None);
assert_eq!(parse_env_line("1BAD=x"), None);
assert_eq!(parse_env_line("BAD KEY=x"), None);
}
#[test] [resource]
fn env_template_is_parseable_and_bootstrap_ready() { output_root = '/srv/from-config'
// 模板每个非注释行必须可解析;无参启动所需的最小配置默认启用。 auto_discover = true
let mut keys = Vec::new();
for line in ENV_TEMPLATE.lines() { [localized]
let line = line.trim(); output_root = '/srv/from-config-localized'
if line.is_empty() || line.starts_with('#') {
continue; [network]
} proxy = 'none'
let (key, _) = parse_env_line(line).unwrap_or_else(|| panic!("模板行必须可解析:{line}")); download_concurrency = 12
keys.push(key);
[translation.worker]
translation_memory_path = '/srv/config-tm.sqlite'
"#,
)
.unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut permissions = std::fs::metadata(&config_path).unwrap().permissions();
permissions.set_mode(0o600);
std::fs::set_permissions(&config_path, permissions).unwrap();
} }
assert!(keys.contains(&"BAT_OUTPUT".to_string())); let config = super::config_file::load_from_binary_dir(temp.path())
assert!(keys.contains(&"BAT_LOCALIZED_OUTPUT".to_string())); .unwrap()
assert!(keys.contains(&"BAT_IMPORT_REPOSITORY".to_string())); .unwrap();
assert!(keys.contains(&"BAT_IMPORT_CAS_ROOT".to_string())); let options = parse_args_with_env(
assert!(keys.contains(&"BAT_IMPORT_RESOURCE_DB".to_string())); vec![
assert!(keys.contains(&"BAT_AUTO_DISCOVER".to_string())); "bat".to_string(),
"--daemon".to_string(),
"--output".to_string(),
"/cli/output".to_string(),
"--translation-memory-path".to_string(),
"/cli/tm.sqlite".to_string(),
],
|key| match key {
"BAT_OUTPUT" => Some("/env/output".to_string()),
"BAT_TRANSLATION_MEMORY_PATH" => Some("/env/tm.sqlite".to_string()),
_ => None,
},
Some(&config),
)
.unwrap();
assert_eq!(options.state_dir, PathBuf::from("/srv/state"));
assert_eq!(options.output_format, OutputFormat::Json);
assert_eq!(options.config.output_root, PathBuf::from("/cli/output"));
assert_eq!(
options.config.localized_output_root,
PathBuf::from("/srv/from-config-localized")
);
assert_eq!(options.config.download_concurrency, 12);
assert_eq!(options.config.curl_proxy.mode(), &CurlProxyMode::Disabled);
assert_eq!(
options.translation_memory_path,
Some(PathBuf::from("/cli/tm.sqlite"))
);
} }
#[test] #[test]
@@ -1326,6 +1503,14 @@ fn rejects_proxy_with_unsupported_scheme() {
assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok()); assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok());
} }
#[test]
fn proxy_parse_errors_redact_credentials() {
let error = parse(&["bat", "--proxy", "http://user:secret@"]).unwrap_err();
let message = error.to_string();
assert!(!message.contains("secret"));
assert!(message.contains("<redacted>"));
}
#[test] #[test]
fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() { fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() {
let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap(); let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap();
@@ -1997,6 +2182,54 @@ fn daemon_child_args_preserve_sync_options() {
assert!(args.contains(&"--no-banner".to_string())); assert!(args.contains(&"--no-banner".to_string()));
} }
#[test]
fn daemon_child_args_preserve_translation_worker_configuration() {
let options = parse(&[
"bat",
"--daemon",
"--output",
"/tmp/daemon-output",
"--translation-provider",
"mock",
"--translation-fixture",
"/tmp/provider.json",
"--translation-memory-path",
"/tmp/tm.sqlite",
"--worker-concurrency",
"4",
"--worker-max-attempts",
"5",
"--worker-lease-seconds",
"60",
"--worker-retry-backoff-seconds",
"2",
"--worker-max-tasks",
"3",
"--worker-id",
"daemon-worker",
])
.unwrap();
let args = daemon_child_args(&options);
let mut child_args = vec!["bat".to_string()];
child_args.extend(args);
let child = parse_args_from(child_args).unwrap();
assert_eq!(child.translation_provider.as_deref(), Some("mock"));
assert_eq!(
child.translation_fixture,
Some(PathBuf::from("/tmp/provider.json"))
);
assert_eq!(
child.translation_memory_path,
Some(PathBuf::from("/tmp/tm.sqlite"))
);
assert_eq!(child.worker_concurrency, 4);
assert_eq!(child.worker_max_attempts, 5);
assert_eq!(child.worker_lease_seconds, 60);
assert_eq!(child.worker_retry_backoff, Duration::from_secs(2));
assert_eq!(child.worker_max_tasks, Some(3));
assert_eq!(child.worker_id.as_deref(), Some("daemon-worker"));
}
#[test] #[test]
fn curl_proxy_url_extracts_only_url_mode() { fn curl_proxy_url_extracts_only_url_mode() {
assert_eq!( assert_eq!(
@@ -2184,6 +2417,7 @@ fn test_task_context_with_config(base_config: OfficialUpdateConfig) -> DaemonTas
registry: TaskRegistry::new(), registry: TaskRegistry::new(),
queue, queue,
base_config, base_config,
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())), sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller, restart_controller: test_restart_controller,
} }
@@ -2354,6 +2588,7 @@ fn dispatch_daemon_doctor_returns_report() {
registry: TaskRegistry::new(), registry: TaskRegistry::new(),
queue, queue,
base_config, base_config,
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())), sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller, restart_controller: test_restart_controller,
}; };
@@ -2374,6 +2609,112 @@ fn dispatch_daemon_doctor_returns_report() {
assert!(checks.iter().any(|check| check["name"] == "daemon_rpc")); assert!(checks.iter().any(|check| check["name"] == "daemon_rpc"));
} }
#[test]
fn dispatch_translation_memory_summary_reports_missing_database_without_creating_it() {
let temp = tempfile::TempDir::new().unwrap();
let output_root = temp.path().join("output");
let state_dir = temp.path().join("state");
let (queue, _rx) = mpsc::channel::<TaskJob>();
let context = DaemonTaskContext {
registry: TaskRegistry::new(),
queue,
base_config: OfficialUpdateConfig {
output_root: output_root.clone(),
..Default::default()
},
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller,
};
let envelope = dispatch_rpc_method(
&rpc_request("translation.memory.summary", None),
&state_dir,
&new_daemon_control(),
&context,
"req-tm-summary-1".to_string(),
);
let value = serde_json::to_value(envelope).unwrap();
assert_eq!(value["ok"], true);
assert_eq!(value["data"]["available"], false);
assert_eq!(value["data"]["reason"], "database_missing");
assert!(!output_root.join("translation-memory.sqlite").exists());
}
#[test]
fn dispatch_translation_memory_summary_defaults_to_worker_config_path() {
let temp = tempfile::TempDir::new().unwrap();
let output_root = temp.path().join("output");
let state_dir = temp.path().join("state");
let configured_tm_path = temp.path().join("configured-tm.sqlite");
let (queue, _rx) = mpsc::channel::<TaskJob>();
let context = DaemonTaskContext {
registry: TaskRegistry::new(),
queue,
base_config: OfficialUpdateConfig {
output_root,
..Default::default()
},
translation_worker_config: TranslationWorkerConfig {
translation_memory_path: Some(configured_tm_path.clone()),
..TranslationWorkerConfig::default()
},
sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller,
};
let envelope = dispatch_rpc_method(
&rpc_request("translation.memory.summary", None),
&state_dir,
&new_daemon_control(),
&context,
"req-tm-summary-configured-1".to_string(),
);
let value = serde_json::to_value(envelope).unwrap();
assert_eq!(value["ok"], true);
assert_eq!(value["data"]["available"], false);
assert_eq!(
value["data"]["path"].as_str(),
Some(configured_tm_path.to_str().unwrap())
);
}
#[test]
fn dispatch_translation_memory_rejects_invalid_params_with_stable_error_code() {
let temp = tempfile::TempDir::new().unwrap();
let context = test_task_context_with_config(OfficialUpdateConfig {
output_root: temp.path().join("output"),
..Default::default()
});
for (method, params) in [
("translation.memory.query", None),
(
"translation.memory.query",
Some(serde_json::json!({ "source_text": "Hello", "limit": "1" })),
),
(
"translation.memory.summary",
Some(serde_json::json!({ "translation_memory_path": 42 })),
),
(
"translation.memory.confirm",
Some(serde_json::json!({ "record_id": "tm-record", "reviewer": 42 })),
),
] {
let envelope = dispatch_rpc_method(
&rpc_request(method, params),
temp.path(),
&new_daemon_control(),
&context,
format!("req-invalid-{method}"),
);
let value = serde_json::to_value(envelope).unwrap();
assert_eq!(value["ok"], false, "method={method}");
assert_eq!(value["error"]["code"], "BAT-ERR-700002", "method={method}");
}
}
#[test] #[test]
fn dispatch_resource_sync_enqueues_task() { fn dispatch_resource_sync_enqueues_task() {
let temp = tempfile::TempDir::new().unwrap(); let temp = tempfile::TempDir::new().unwrap();
@@ -2384,6 +2725,7 @@ fn dispatch_resource_sync_enqueues_task() {
registry: TaskRegistry::new(), registry: TaskRegistry::new(),
queue, queue,
base_config: OfficialUpdateConfig::default(), base_config: OfficialUpdateConfig::default(),
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())), sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller, restart_controller: test_restart_controller,
}; };
@@ -2445,6 +2787,7 @@ fn dispatch_resource_repair_enqueues_repair_task() {
registry: TaskRegistry::new(), registry: TaskRegistry::new(),
queue, queue,
base_config, base_config,
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())), sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller, restart_controller: test_restart_controller,
}; };
@@ -3737,6 +4080,7 @@ fn dispatch_catalog_refresh_enqueues_task() {
registry: TaskRegistry::new(), registry: TaskRegistry::new(),
queue, queue,
base_config: OfficialUpdateConfig::default(), base_config: OfficialUpdateConfig::default(),
translation_worker_config: TranslationWorkerConfig::default(),
sync_lock: Arc::new(Mutex::new(())), sync_lock: Arc::new(Mutex::new(())),
restart_controller: test_restart_controller, restart_controller: test_restart_controller,
}; };
File diff suppressed because it is too large Load Diff
@@ -111,6 +111,13 @@ impl HumanReport for bat_infrastructure::TranslationWorkerReport {
print_field("失败任务", self.failed_count); print_field("失败任务", self.failed_count);
print_field("已安排重试", self.retry_scheduled_count); print_field("已安排重试", self.retry_scheduled_count);
print_field("剩余任务", self.remaining_count); print_field("剩余任务", self.remaining_count);
print_path_field("Translation Memory", &self.translation_memory_path);
print_field("TM 可用", format_bool(self.translation_memory_available));
print_field("TM 命中 TextUnit", self.translation_memory_hit_count);
print_field("Provider TextUnit", self.provider_unit_count);
for failure in &self.translation_memory_failures {
println!(" - TM: {failure}");
}
for failure in &self.failures { for failure in &self.failures {
println!( println!(
" - {} [{}] retryable={} {}", " - {} [{}] retryable={} {}",
+12 -5
View File
@@ -519,6 +519,8 @@ pub(super) struct DaemonTaskContext {
pub(super) registry: TaskRegistry, pub(super) registry: TaskRegistry,
pub(super) queue: mpsc::Sender<TaskJob>, pub(super) queue: mpsc::Sender<TaskJob>,
pub(super) base_config: OfficialUpdateConfig, pub(super) base_config: OfficialUpdateConfig,
/// daemon 中未显式传入参数的 translation worker 默认配置。
pub(super) translation_worker_config: TranslationWorkerConfig,
/// 串行化会读取或修改已发布资源状态的 daemon 操作。 /// 串行化会读取或修改已发布资源状态的 daemon 操作。
pub(super) sync_lock: Arc<Mutex<()>>, pub(super) sync_lock: Arc<Mutex<()>>,
pub(super) restart_controller: DaemonRestartController, pub(super) restart_controller: DaemonRestartController,
@@ -562,11 +564,15 @@ pub(super) fn run_task_worker(
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build()?; .build()?;
let cancel_check = Arc::clone(&cancel);
runtime runtime
.block_on(bat_infrastructure::run_translation_worker_at( .block_on(
&resource_root, bat_infrastructure::run_translation_worker_at_with_cancellation(
worker_config, &resource_root,
)) worker_config,
Arc::new(move || cancel_check.load(Ordering::Relaxed)),
),
)
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from)) .and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
}) })
} else { } else {
@@ -608,7 +614,8 @@ pub(super) fn run_task_worker(
record.result = Some(report); record.result = Some(report);
}), }),
Err(error) => { Err(error) => {
let cancelled = cancel.load(Ordering::Relaxed); let cancelled =
cancel.load(Ordering::Relaxed) || daemon_control_stop_requested(Some(&control));
// 下载失败携带类型化 DownloadError(含准确网络域码);其余归 internal。 // 下载失败携带类型化 DownloadError(含准确网络域码);其余归 internal。
let code = error let code = error
.downcast_ref::<bat_infrastructure::DownloadError>() .downcast_ref::<bat_infrastructure::DownloadError>()
+9 -13
View File
@@ -71,28 +71,24 @@ pub(super) fn print_startup_banner() {
eprintln!("{STARTUP_BANNER}"); eprintln!("{STARTUP_BANNER}");
} }
pub(super) fn print_env_template_created(path: &Path) { pub(super) fn print_config_template_created(path: &Path) {
eprintln!( eprintln!(
"已生成配置模板 {}(编辑其中的 BAT_* 配置后,直接运行 `bat` 即可按 .env 启动", "已生成配置模板 {}(编辑 `config.toml``config.toml.example` 不会被程序自动读取",
path.display() path.display()
); );
} }
pub(super) fn print_env_template_warning(path: &Path, error: impl std::fmt::Display) { pub(super) fn print_config_template_warning(path: &Path, error: impl std::fmt::Display) {
eprintln!("警告:生成 .env 配置模板失败 {}{error}", path.display());
}
pub(super) fn print_env_read_warning(path: &Path, error: impl std::fmt::Display) {
eprintln!("警告:读取 .env 失败 {}{error}", path.display());
}
pub(super) fn print_env_parse_warning(line_number: usize, raw_line: &str) {
eprintln!( eprintln!(
"警告:.env 第 {} 行无法解析,已忽略:{raw_line}", "警告:生成 config.toml.example 模板失败 {}{error}",
line_number path.display()
); );
} }
pub(super) fn print_deprecated_env_file_warning() {
eprintln!("警告:BAT_SKIP_ENV_FILE 已废弃且不再影响启动,已忽略");
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(super) struct ProgressLogger { pub(super) struct ProgressLogger {
enabled: bool, enabled: bool,
@@ -1,4 +1,8 @@
use super::report_output::print_json_value;
use super::*; use super::*;
use bat_core::domain::TranslationMemoryContext;
use bat_core::repositories::TranslationMemoryRepository;
use std::collections::BTreeMap;
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
struct TranslationTaskResultUpdateParam { struct TranslationTaskResultUpdateParam {
@@ -332,6 +336,8 @@ fn build_manual_translation_results(
unit_id: unit_id.to_string(), unit_id: unit_id.to_string(),
source_text: param.source_text.clone(), source_text: param.source_text.clone(),
translated_text: param.translated_text.clone(), translated_text: param.translated_text.clone(),
source_kind: bat_infrastructure::TranslationTaskResultSourceKind::Manual,
translation_memory_record_id: None,
provider: provider.to_string(), provider: provider.to_string(),
provider_run_id: provider_run_id.to_string(), provider_run_id: provider_run_id.to_string(),
translated_unix_seconds, translated_unix_seconds,
@@ -355,3 +361,241 @@ pub(super) fn translation_task_query_json(query: &OfficialTextUnitTaskQuery) ->
"has_failure_reason": query.has_failure_reason, "has_failure_reason": query.has_failure_reason,
}) })
} }
pub(super) fn run_translation_memory_command(options: &CliOptions) -> anyhow::Result<()> {
let method = match options.command {
CliCommand::TranslationMemorySummary => RPC_METHOD_TRANSLATION_MEMORY_SUMMARY,
CliCommand::TranslationMemoryQuery => RPC_METHOD_TRANSLATION_MEMORY_QUERY,
CliCommand::TranslationMemoryConfirm => RPC_METHOD_TRANSLATION_MEMORY_CONFIRM,
_ => return Err(anyhow::anyhow!("不是 Translation Memory 命令")),
};
if daemon_rpc_available(&options.state_dir)
&& options.resource_root.is_none()
&& !options.output_explicit
{
let _control_lock = DaemonControlLock::acquire(&options.state_dir)?;
let report = daemon_rpc_call(
&options.state_dir,
method,
translation_memory_cli_params(options)?,
)?;
print_json_value(options.output_format, &report)?;
return Ok(());
}
let path = translation_memory_cli_path(options)?;
let report = match options.command {
CliCommand::TranslationMemorySummary => build_translation_memory_summary_report(&path)?,
CliCommand::TranslationMemoryQuery => {
let source_text = options
.translation_memory_source_text
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
let context = parse_translation_memory_context(
options.translation_memory_context_json.as_deref(),
)?;
build_translation_memory_query_report(
&path,
source_text,
&context,
options.query_limit,
)?
}
CliCommand::TranslationMemoryConfirm => {
let record_id = options
.translation_memory_record_id
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
let reviewer = options
.translation_memory_reviewer
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
build_translation_memory_confirm_report(
&path,
record_id,
reviewer,
options.translation_memory_reason.clone(),
)?
}
_ => unreachable!(),
};
print_json_value(options.output_format, &report)
}
fn translation_memory_cli_path(options: &CliOptions) -> anyhow::Result<std::path::PathBuf> {
if let Some(path) = options.translation_memory_path.as_ref() {
return lexical_absolute(path).map_err(anyhow::Error::msg);
}
let resource_root = options
.resource_root
.as_deref()
.map(lexical_absolute)
.transpose()
.map_err(anyhow::Error::msg)?
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
Ok(bat_infrastructure::translation_memory_repository_path(
&resource_root,
))
}
fn translation_memory_cli_params(
options: &CliOptions,
) -> anyhow::Result<Option<serde_json::Value>> {
let mut params = serde_json::Map::new();
if let Some(path) = options.translation_memory_path.as_ref() {
params.insert(
"translation_memory_path".to_string(),
serde_json::json!(path),
);
}
match options.command {
CliCommand::TranslationMemorySummary => {}
CliCommand::TranslationMemoryQuery => {
let source_text = options
.translation_memory_source_text
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
let context = parse_translation_memory_context(
options.translation_memory_context_json.as_deref(),
)?;
params.insert("source_text".to_string(), serde_json::json!(source_text));
params.insert("source_context".to_string(), serde_json::json!(context));
params.insert("limit".to_string(), serde_json::json!(options.query_limit));
}
CliCommand::TranslationMemoryConfirm => {
let record_id = options
.translation_memory_record_id
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
let reviewer = options
.translation_memory_reviewer
.as_deref()
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
params.insert("record_id".to_string(), serde_json::json!(record_id));
params.insert("reviewer".to_string(), serde_json::json!(reviewer));
if let Some(reason) = options.translation_memory_reason.as_deref() {
params.insert("reason".to_string(), serde_json::json!(reason));
}
}
_ => unreachable!(),
}
Ok(Some(serde_json::Value::Object(params)))
}
pub(super) fn build_translation_memory_summary_report(
path: &std::path::Path,
) -> anyhow::Result<serde_json::Value> {
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
return Ok(serde_json::json!({
"available": false,
"path": path,
"reason": "database_missing",
}));
}
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let summary = runtime.block_on(async {
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
.await
.map_err(|error| anyhow::anyhow!("{error}"))?;
repository
.summary()
.await
.map_err(|error| anyhow::anyhow!("{error}"))
})?;
Ok(serde_json::json!({
"available": true,
"path": path,
"schema_version": summary.schema_version,
"summary": summary,
}))
}
pub(super) fn build_translation_memory_query_report(
path: &std::path::Path,
source_text: &str,
source_context: &TranslationMemoryContext,
limit: usize,
) -> anyhow::Result<serde_json::Value> {
if source_text.trim().is_empty() {
return Err(anyhow::anyhow!("TM query 的 source_text 不能为空"));
}
if !(1..=1000).contains(&limit) {
return Err(anyhow::anyhow!("TM query 的 limit 必须在 1..=1000 范围内"));
}
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
return Ok(serde_json::json!({
"available": false,
"path": path,
"source_text": source_text,
"source_context": source_context,
"matches": [],
"reason": "database_missing",
}));
}
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let matches = runtime.block_on(async {
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
.await
.map_err(|error| anyhow::anyhow!("{error}"))?;
repository
.find_matches(source_text, source_context, limit)
.await
.map_err(|error| anyhow::anyhow!("{error}"))
})?;
Ok(serde_json::json!({
"available": true,
"path": path,
"source_text": source_text,
"source_context": source_context,
"matches": matches,
}))
}
pub(super) fn build_translation_memory_confirm_report(
path: &std::path::Path,
record_id: &str,
reviewer: &str,
reason: Option<String>,
) -> anyhow::Result<serde_json::Value> {
if record_id.trim().is_empty() || reviewer.trim().is_empty() {
return Err(anyhow::anyhow!(
"TM confirm 必须指定非空 record_id 和 reviewer"
));
}
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
return Err(anyhow::anyhow!(
"Translation Memory 数据库不存在:{}",
path.display()
));
}
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let entry = runtime.block_on(async {
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
.await
.map_err(|error| anyhow::anyhow!("{error}"))?;
repository
.confirm(record_id, reviewer, reason)
.await
.map_err(|error| anyhow::anyhow!("{error}"))
})?;
Ok(serde_json::json!({
"available": true,
"path": path,
"entry": entry,
}))
}
fn parse_translation_memory_context(
value: Option<&str>,
) -> anyhow::Result<TranslationMemoryContext> {
let Some(value) = value else {
return Ok(BTreeMap::new());
};
serde_json::from_str::<TranslationMemoryContext>(value)
.map_err(|error| anyhow::anyhow!("--tm-context-json 必须是 JSON object{error}"))
}
@@ -261,26 +261,10 @@ pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()>
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg)) .map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
.transpose()? .transpose()?
.unwrap_or(active_official_resource_root(&options.config.output_root)?); .unwrap_or(active_official_resource_root(&options.config.output_root)?);
let provider = options let config = super::translation_worker_config_from_options(
.translation_provider options,
.as_deref() &format!("bat-worker-{}", std::process::id()),
.unwrap_or(TranslationProviderKind::Mock.as_str()); )?;
let provider = TranslationProviderKind::parse(provider)
.ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效:{provider}"))?;
let config = TranslationWorkerConfig {
provider,
fixture_path: options.translation_fixture.clone(),
concurrency: options.worker_concurrency,
max_attempts: options.worker_max_attempts,
lease_seconds: options.worker_lease_seconds,
retry_backoff: options.worker_retry_backoff,
max_tasks: options.worker_max_tasks,
worker_id: options
.worker_id
.clone()
.unwrap_or_else(|| format!("bat-worker-{}", std::process::id())),
};
config.validate()?;
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build()?; .build()?;
+18 -12
View File
@@ -29,6 +29,7 @@ pub mod patch_ops;
pub mod path_security; pub mod path_security;
pub mod release_flow; pub mod release_flow;
pub mod resources; pub mod resources;
pub mod translation_memory;
pub mod translation_tasks; pub mod translation_tasks;
pub mod translation_worker; pub mod translation_worker;
pub mod translation_workflow; pub mod translation_workflow;
@@ -140,24 +141,29 @@ pub use path_security::{
}; };
pub use release_flow::ReleaseFlowStatusCode; pub use release_flow::ReleaseFlowStatusCode;
pub use resources::{InMemoryResourceRepository, SqliteResourceRepository}; pub use resources::{InMemoryResourceRepository, SqliteResourceRepository};
pub use translation_memory::{
translation_memory_context, translation_memory_repository_path,
SqliteTranslationMemoryRepository, TRANSLATION_MEMORY_REPOSITORY_FILE,
TRANSLATION_MEMORY_SCHEMA_COMPONENT, TRANSLATION_MEMORY_SCHEMA_VERSION,
};
pub use translation_tasks::{ pub use translation_tasks::{
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at, build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState, write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff, ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus, TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskResultSourceKind,
TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus, TranslationTaskStatus, TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit,
TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE, TranslationUnitStatus, TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION,
TRANSLATION_TASK_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE, TRANSLATION_TASK_SCHEMA_VERSION,
}; };
pub use translation_worker::{ pub use translation_worker::{
run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider, run_translation_worker_at, run_translation_worker_at_with_cancellation,
MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass, run_translation_worker_with_provider, CrowdinProvider, MockTranslationProvider,
TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse, TranslationProvider, TranslationProviderFailureClass, TranslationProviderKind,
TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig, TranslationProviderRequest, TranslationProviderResponse, TranslationProviderUnit,
TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, TranslationProviderUnitResult, TranslationWorkerConfig, TranslationWorkerFailure,
DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS,
DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF,
MOCK_TRANSLATION_FIXTURE_VERSION, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, MOCK_TRANSLATION_FIXTURE_VERSION,
}; };
pub use translation_workflow::{ pub use translation_workflow::{
completed_worker_translation_workbench, export_completed_worker_translation_workbench, completed_worker_translation_workbench, export_completed_worker_translation_workbench,
+24
View File
@@ -91,6 +91,12 @@ pub struct LocalizedPatchOperationMetadata {
/// Provider run ID that produced the text, if applicable. /// Provider run ID that produced the text, if applicable.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>, pub provider_run_id: Option<String>,
/// Source kind of the translation result.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_source_kind: Option<String>,
/// Trusted Translation Memory record used for the text, if applicable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_memory_record_id: Option<String>,
/// Review state used by the publication input. /// Review state used by the publication input.
pub review_status: String, pub review_status: String,
} }
@@ -331,6 +337,12 @@ pub struct LocalizedPatchOperation {
/// Provider run ID that produced the text, if applicable. /// Provider run ID that produced the text, if applicable.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>, pub provider_run_id: Option<String>,
/// Source kind of the translation result.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_source_kind: Option<String>,
/// Trusted Translation Memory record used for the text, if applicable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_memory_record_id: Option<String>,
/// Review state used by the publication input. /// Review state used by the publication input.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub review_status: Option<String>, pub review_status: Option<String>,
@@ -941,6 +953,8 @@ impl LocalizedPatchOperation {
source_text_blake3: None, source_text_blake3: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
review_status: None, review_status: None,
}, },
metadata, metadata,
@@ -966,6 +980,8 @@ impl LocalizedPatchOperation {
source_text_blake3: None, source_text_blake3: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
review_status: None, review_status: None,
}, },
metadata, metadata,
@@ -990,6 +1006,8 @@ impl LocalizedPatchOperation {
source_text_blake3: None, source_text_blake3: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
review_status: None, review_status: None,
}, },
metadata, metadata,
@@ -1005,6 +1023,8 @@ impl LocalizedPatchOperation {
operation.source_text_blake3 = Some(metadata.source_text_blake3.clone()); operation.source_text_blake3 = Some(metadata.source_text_blake3.clone());
operation.translation_provider = metadata.translation_provider.clone(); operation.translation_provider = metadata.translation_provider.clone();
operation.provider_run_id = metadata.provider_run_id.clone(); operation.provider_run_id = metadata.provider_run_id.clone();
operation.translation_source_kind = metadata.translation_source_kind.clone();
operation.translation_memory_record_id = metadata.translation_memory_record_id.clone();
operation.review_status = Some(metadata.review_status.clone()); operation.review_status = Some(metadata.review_status.clone());
} }
operation operation
@@ -1029,6 +1049,8 @@ impl Default for LocalizedPatchOperation {
source_text_blake3: None, source_text_blake3: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
review_status: None, review_status: None,
} }
} }
@@ -1401,6 +1423,8 @@ mod tests {
source_text_blake3: Some(blake3::hash(source).to_hex().to_string()), source_text_blake3: Some(blake3::hash(source).to_hex().to_string()),
translation_provider: Some("mock".to_string()), translation_provider: Some("mock".to_string()),
provider_run_id: Some("mock:unit-1:attempt-1".to_string()), provider_run_id: Some("mock:unit-1:attempt-1".to_string()),
translation_source_kind: Some("provider".to_string()),
translation_memory_record_id: None,
review_status: Some("provider_completed".to_string()), review_status: Some("provider_completed".to_string()),
}], }],
}], }],
+859
View File
@@ -0,0 +1,859 @@
//! 跨 official release 的 Translation Memory SQLite 仓储。
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
use async_trait::async_trait;
use bat_core::domain::{
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
TranslationMemorySummary, TranslationMemoryTrustStatus,
};
use bat_core::repositories::TranslationMemoryRepository;
use bat_core::{Error, Result};
use serde::de::DeserializeOwned;
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
use sqlx::{Row, SqlitePool};
use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// TM SQLite schema 版本。
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1;
/// TM schema migration component。
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
/// 默认 TM 数据库文件名。
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
/// SQLite-backed Translation Memory 仓储。
#[derive(Debug, Clone)]
pub struct SqliteTranslationMemoryRepository {
pub(crate) pool: SqlitePool,
}
impl SqliteTranslationMemoryRepository {
/// 创建或打开 TM 数据库并执行迁移。
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
Self::open_with(path.as_ref(), true).await
}
/// 只打开已有 TM 数据库,不创建新文件。
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
Self::open_with(path.as_ref(), false).await
}
/// 根据 active release 根目录计算默认的跨 release TM 路径。
///
/// 正式 release 根目录形如 `<output>/versions/<id>`,因此默认结果为
/// `<output>/translation-memory.sqlite`,不会写入已发布版本目录。
pub fn repository_path(resource_root: &Path) -> PathBuf {
if resource_root
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "versions")
{
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
}
}
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
}
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
let absolute = bat_infrastructure_absolute(path)?;
let parent = absolute.parent().ok_or_else(|| {
Error::InvalidArgument(format!("TM 数据库缺少父目录:{}", absolute.display()))
})?;
ensure_safe_tm_parent(parent)?;
if create_if_missing {
tokio::fs::create_dir_all(parent).await?;
ensure_safe_tm_parent(parent)?;
}
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
if metadata.file_type().is_symlink() {
return Err(Error::InvalidArgument(format!(
"TM 数据库不能是 symlink{}",
absolute.display()
)));
}
if !metadata.is_file() {
return Err(Error::InvalidArgument(format!(
"TM 数据库不是普通文件:{}",
absolute.display()
)));
}
} else if !create_if_missing {
return Err(Error::NotFound(absolute.display().to_string()));
}
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
.map_err(|error| Error::Other(error.into()))?
.create_if_missing(create_if_missing)
.journal_mode(SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30));
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await
.map_err(db_error)?;
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
.map_err(Error::InvalidArgument)?;
let repository = Self { pool };
repository.init_schema().await?;
Ok(repository)
}
async fn init_schema(&self) -> Result<()> {
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)
"#,
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS translation_memory (
record_id TEXT PRIMARY KEY NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
normalized_source_text TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
translated_text TEXT NOT NULL,
translation_source_kind TEXT NOT NULL,
trust_status TEXT NOT NULL,
official_release_id TEXT NOT NULL,
source_trace_json TEXT NOT NULL,
provider TEXT,
provider_run_id TEXT,
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
trusted_unix_seconds INTEGER,
trusted_by TEXT,
trusted_reason TEXT,
supersedes_record_id TEXT,
superseded_by_record_id TEXT,
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(official_release_id) > 0),
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
)
"#,
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
ON translation_memory(source_hash)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
ON translation_memory(normalized_source_text)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
ON translation_memory(source_hash, source_context_hash)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
let current: Option<i64> =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_optional(&self.pool)
.await
.map_err(db_error)?;
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
return Err(Error::InvalidArgument(format!(
"不支持的 Translation Memory schema 版本:{}",
current.unwrap_or_default()
)));
}
sqlx::query(
r#"
INSERT INTO schema_migrations(component, version)
VALUES (?1, ?2)
ON CONFLICT(component) DO UPDATE SET version = excluded.version
"#,
)
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
.execute(&self.pool)
.await
.map_err(db_error)?;
Ok(())
}
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
let row = sqlx::query(
r#"
SELECT record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
FROM translation_memory
WHERE record_id = ?1
"#,
)
.bind(record_id)
.fetch_optional(&self.pool)
.await
.map_err(db_error)?;
row.map(row_to_entry).transpose()
}
}
#[async_trait]
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
async fn upsert_candidate(
&self,
draft: TranslationMemoryDraft,
) -> Result<TranslationMemoryEntry> {
validate_draft(&draft)?;
let entry = entry_from_draft(draft)?;
if let Some(existing) = self.find_optional(&entry.record_id).await? {
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
return Ok(existing);
}
let source_trace_json = serde_json::to_string(&entry.source_trace)
.map_err(|error| Error::Serialization(error.to_string()))?;
sqlx::query(
r#"
UPDATE translation_memory
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
updated_unix_seconds = ?5
WHERE record_id = ?1 AND trust_status = 'candidate'
"#,
)
.bind(&entry.record_id)
.bind(source_trace_json)
.bind(&entry.provider)
.bind(&entry.provider_run_id)
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
.execute(&self.pool)
.await
.map_err(db_error)?;
return self.find(&entry.record_id).await;
}
let source_context_json = serde_json::to_string(&entry.source_context)
.map_err(|error| Error::Serialization(error.to_string()))?;
let source_trace_json = serde_json::to_string(&entry.source_trace)
.map_err(|error| Error::Serialization(error.to_string()))?;
let result = sqlx::query(
r#"
INSERT INTO translation_memory (
record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds,
trusted_unix_seconds, trusted_by, trusted_reason,
supersedes_record_id, superseded_by_record_id
)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
ON CONFLICT(record_id) DO UPDATE SET
source_trace_json = excluded.source_trace_json,
provider = excluded.provider,
provider_run_id = excluded.provider_run_id,
updated_unix_seconds = excluded.updated_unix_seconds
WHERE translation_memory.trust_status = 'candidate'
"#,
)
.bind(&entry.record_id)
.bind(&entry.source_text)
.bind(&entry.source_hash)
.bind(&entry.normalized_source_text)
.bind(source_context_json)
.bind(&entry.source_context_hash)
.bind(&entry.translated_text)
.bind(entry.translation_source_kind.as_str())
.bind(entry.trust_status.as_str())
.bind(&entry.official_release_id)
.bind(source_trace_json)
.bind(&entry.provider)
.bind(&entry.provider_run_id)
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
.bind(
entry
.trusted_unix_seconds
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
)
.bind(&entry.trusted_by)
.bind(&entry.trusted_reason)
.bind(&entry.supersedes_record_id)
.bind(&entry.superseded_by_record_id)
.execute(&self.pool)
.await
.map_err(db_error)?;
if result.rows_affected() == 0 {
return self.find(&entry.record_id).await;
}
Ok(entry)
}
async fn find_matches(
&self,
source_text: &str,
source_context: &TranslationMemoryContext,
limit: usize,
) -> Result<Vec<TranslationMemoryMatch>> {
if source_text.is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory 查询 source_text 不能为空".to_string(),
));
}
if limit == 0 {
return Err(Error::InvalidArgument(
"Translation Memory 查询 limit 必须大于 0".to_string(),
));
}
let source_hash = hash_text(source_text);
let normalized_source_text = normalize_source_text(source_text);
let rows = sqlx::query(
r#"
SELECT record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
FROM translation_memory
WHERE source_hash = ?1 OR normalized_source_text = ?2
ORDER BY updated_unix_seconds DESC, record_id ASC
"#,
)
.bind(source_hash)
.bind(&normalized_source_text)
.fetch_all(&self.pool)
.await
.map_err(db_error)?;
let mut matches = rows
.into_iter()
.map(row_to_entry)
.collect::<Result<Vec<_>>>()?
.into_iter()
.filter_map(|entry| {
let raw_exact = entry.source_text == source_text;
let normalized_exact = entry.normalized_source_text == normalized_source_text;
if !raw_exact && !normalized_exact {
return None;
}
let same_context = !source_context.is_empty()
&& !entry.source_context.is_empty()
&& entry.source_context == *source_context;
let strong = raw_exact
&& same_context
&& entry.trust_status == TranslationMemoryTrustStatus::Trusted;
let match_kind = if strong {
TranslationMemoryMatchKind::StrongExact
} else if raw_exact {
TranslationMemoryMatchKind::CandidateExact
} else {
TranslationMemoryMatchKind::SourceOnly
};
Some(TranslationMemoryMatch {
can_auto_reuse: strong,
entry,
match_kind,
})
})
.collect::<Vec<_>>();
matches.sort_by(|left, right| {
match_rank(left)
.cmp(&match_rank(right))
.then_with(|| {
right
.entry
.updated_unix_seconds
.cmp(&left.entry.updated_unix_seconds)
})
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
});
matches.truncate(limit);
Ok(matches)
}
async fn confirm(
&self,
record_id: &str,
reviewer: &str,
reason: Option<String>,
) -> Result<TranslationMemoryEntry> {
if reviewer.trim().is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory reviewer 不能为空".to_string(),
));
}
let current = self.find(record_id).await?;
if current.trust_status == TranslationMemoryTrustStatus::Trusted {
return Ok(current);
}
if current.trust_status != TranslationMemoryTrustStatus::Candidate {
return Err(Error::InvalidArgument(format!(
"Translation Memory 记录 {} 当前状态为 {},不能确认",
record_id,
current.trust_status.as_str()
)));
}
let now = unix_seconds_now();
sqlx::query(
r#"
UPDATE translation_memory
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
WHERE record_id = ?1 AND trust_status = 'candidate'
"#,
)
.bind(record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.bind(reviewer.trim())
.bind(reason.filter(|value| !value.trim().is_empty()))
.execute(&self.pool)
.await
.map_err(db_error)?;
self.find(record_id).await
}
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
self.find_optional(record_id)
.await?
.ok_or_else(|| Error::NotFound(record_id.to_string()))
}
async fn summary(&self) -> Result<TranslationMemorySummary> {
let row = sqlx::query(
r#"
SELECT COUNT(*) AS record_count,
SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count,
SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count,
SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count,
SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count
FROM translation_memory
"#,
)
.fetch_one(&self.pool)
.await
.map_err(db_error)?;
Ok(TranslationMemorySummary {
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
record_count: row.try_get::<i64, _>("record_count").map_err(db_error)? as u64,
trusted_count: row.try_get::<i64, _>("trusted_count").map_err(db_error)? as u64,
candidate_count: row.try_get::<i64, _>("candidate_count").map_err(db_error)? as u64,
superseded_count: row
.try_get::<i64, _>("superseded_count")
.map_err(db_error)? as u64,
rejected_count: row.try_get::<i64, _>("rejected_count").map_err(db_error)? as u64,
})
}
}
/// 由 active official release 根目录计算默认 TM 数据库路径。
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
SqliteTranslationMemoryRepository::repository_path(resource_root)
}
/// 从 TextUnit 定位字段构建 TM 上下文。
#[allow(clippy::too_many_arguments)]
pub fn translation_memory_context(
destination: &str,
archive_entry: Option<&str>,
serialized_file: Option<&str>,
path_id: Option<i64>,
class_id: Option<i32>,
field_path: Option<&str>,
format: Option<&str>,
asset_name: Option<&str>,
text_source_kind: Option<&str>,
parser_context: &TranslationMemoryContext,
) -> TranslationMemoryContext {
let mut context = TranslationMemoryContext::new();
context.insert("destination".to_string(), destination.to_string());
insert_optional(&mut context, "archive_entry", archive_entry);
insert_optional(&mut context, "serialized_file", serialized_file);
if let Some(value) = path_id {
context.insert("path_id".to_string(), value.to_string());
}
if let Some(value) = class_id {
context.insert("class_id".to_string(), value.to_string());
}
insert_optional(&mut context, "field_path", field_path);
insert_optional(&mut context, "format", format);
insert_optional(&mut context, "asset_name", asset_name);
insert_optional(&mut context, "text_source_kind", text_source_kind);
for (key, value) in parser_context {
context.insert(format!("context.{key}"), value.clone());
}
context
}
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
if let Some(value) = value.filter(|value| !value.is_empty()) {
context.insert(key.to_string(), value.to_string());
}
}
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
let source_hash = hash_text(&draft.source_text);
let normalized_source_text = normalize_source_text(&draft.source_text);
let source_context_hash = hash_context(&draft.source_context)?;
let source_kind = draft.translation_source_kind;
let record_id = record_id(
&source_hash,
&source_context_hash,
&draft.translated_text,
&draft.official_release_id,
&source_kind,
);
let observed = draft.observed_unix_seconds;
Ok(TranslationMemoryEntry {
record_id,
source_text: draft.source_text,
source_hash,
normalized_source_text,
source_context: draft.source_context,
source_context_hash,
translated_text: draft.translated_text,
translation_source_kind: source_kind,
trust_status: TranslationMemoryTrustStatus::Candidate,
official_release_id: draft.official_release_id,
source_trace: draft.source_trace,
provider: draft.provider,
provider_run_id: draft.provider_run_id,
created_unix_seconds: observed,
updated_unix_seconds: observed,
trusted_unix_seconds: None,
trusted_by: None,
trusted_reason: None,
supersedes_record_id: None,
superseded_by_record_id: None,
})
}
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
if draft.source_text.is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory source_text 不能为空".to_string(),
));
}
if draft.translated_text.trim().is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory translated_text 不能为空".to_string(),
));
}
if draft.official_release_id.trim().is_empty()
|| draft.source_trace.official_release_id.trim().is_empty()
{
return Err(Error::InvalidArgument(
"Translation Memory official_release_id 不能为空".to_string(),
));
}
if draft.official_release_id != draft.source_trace.official_release_id {
return Err(Error::InvalidArgument(
"Translation Memory draft 的 release provenance 不一致".to_string(),
));
}
Ok(())
}
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
Ok(TranslationMemoryEntry {
record_id: row.try_get("record_id").map_err(db_error)?,
source_text: row.try_get("source_text").map_err(db_error)?,
source_hash: row.try_get("source_hash").map_err(db_error)?,
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
source_context,
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
translated_text: row.try_get("translated_text").map_err(db_error)?,
translation_source_kind: parse_source_kind(
row.try_get::<String, _>("translation_source_kind")
.map_err(db_error)?
.as_str(),
)?,
trust_status: parse_trust_status(
row.try_get::<String, _>("trust_status")
.map_err(db_error)?
.as_str(),
)?,
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
source_trace,
provider: row.try_get("provider").map_err(db_error)?,
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
created_unix_seconds: i64_to_u64(
row.try_get("created_unix_seconds").map_err(db_error)?,
"created",
)?,
updated_unix_seconds: i64_to_u64(
row.try_get("updated_unix_seconds").map_err(db_error)?,
"updated",
)?,
trusted_unix_seconds: optional_i64_to_u64(
row.try_get("trusted_unix_seconds").map_err(db_error)?,
"trusted",
)?,
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
})
}
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
}
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
match value {
"provider" => Ok(TranslationMemorySourceKind::Provider),
"manual" => Ok(TranslationMemorySourceKind::Manual),
"imported" => Ok(TranslationMemorySourceKind::Imported),
_ => Err(Error::Serialization(format!(
"未知 Translation Memory source kind{value}"
))),
}
}
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
match value {
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
_ => Err(Error::Serialization(format!(
"未知 Translation Memory trust status{value}"
))),
}
}
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
match value.match_kind {
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
TranslationMemoryMatchKind::CandidateExact => 1,
TranslationMemoryMatchKind::SourceOnly => 2,
TranslationMemoryMatchKind::StrongExact => 1,
}
}
fn hash_text(value: &str) -> String {
blake3::hash(value.as_bytes()).to_hex().to_string()
}
fn normalize_source_text(value: &str) -> String {
value.replace("\r\n", "\n").replace('\r', "\n")
}
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
let bytes =
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
Ok(blake3::hash(&bytes).to_hex().to_string())
}
fn record_id(
source_hash: &str,
context_hash: &str,
translated_text: &str,
official_release_id: &str,
source_kind: &TranslationMemorySourceKind,
) -> String {
let mut key = Vec::new();
for value in [
source_hash,
context_hash,
translated_text,
official_release_id,
source_kind.as_str(),
] {
key.extend_from_slice(value.as_bytes());
key.push(0);
}
format!("tm-{}", blake3::hash(&key).to_hex())
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
u64::try_from(value)
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
}
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
value.map(|value| i64_to_u64(value, label)).transpose()
}
fn db_error(error: sqlx::Error) -> Error {
Error::Other(error.into())
}
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
}
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
.map_err(Error::InvalidArgument)
}
#[cfg(test)]
mod tests {
use super::*;
use bat_core::domain::TranslationMemorySourceTrace;
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
let source_trace = TranslationMemorySourceTrace {
official_release_id: release.to_string(),
unit_id: Some(format!("{release}-unit")),
task_id: Some(format!("{release}-task")),
destination: Some("Bundles/story.bundle".to_string()),
archive_entry: None,
serialized_file: Some("CAB-story".to_string()),
path_id: Some(1),
class_id: Some(49),
field_path: Some("m_Text".to_string()),
format: Some("plain".to_string()),
asset_name: Some("Story".to_string()),
text_source_kind: Some("text_asset".to_string()),
source_url: Some("https://example.invalid/story".to_string()),
};
TranslationMemoryDraft {
source_text: source.to_string(),
source_context: translation_memory_context(
"Bundles/story.bundle",
None,
Some("CAB-story"),
Some(1),
Some(49),
Some("m_Text"),
Some("plain"),
Some("Story"),
Some("text_asset"),
&TranslationMemoryContext::new(),
),
translated_text: translated.to_string(),
translation_source_kind: TranslationMemorySourceKind::Provider,
official_release_id: release.to_string(),
source_trace,
provider: Some("mock".to_string()),
provider_run_id: Some(format!("mock:{release}")),
observed_unix_seconds: 1,
}
}
#[tokio::test]
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
)
.await
.unwrap();
let entry = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
let trusted = repository
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
.await
.unwrap();
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
let query = draft("release-2", "Hello", "ignored");
let matches = repository
.find_matches("Hello", &query.source_context, 10)
.await
.unwrap();
assert_eq!(matches.len(), 1);
assert_eq!(
matches[0].match_kind,
TranslationMemoryMatchKind::StrongExact
);
assert!(matches[0].can_auto_reuse);
assert_eq!(matches[0].entry.translated_text, "你好");
assert_eq!(matches[0].entry.official_release_id, "release-1");
}
#[tokio::test]
async fn rejects_future_schema_version() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("tm.sqlite");
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let error = SqliteTranslationMemoryRepository::new(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("不支持的 Translation Memory schema"));
}
#[tokio::test]
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let first = draft("release-1", "Hello", "你好");
repository.upsert_candidate(first.clone()).await.unwrap();
repository.upsert_candidate(first).await.unwrap();
assert_eq!(repository.summary().await.unwrap().record_count, 1);
let mut different = draft("release-2", "Hello", "你好");
different
.source_context
.insert("field_path".to_string(), "m_Other".to_string());
let matches = repository
.find_matches("Hello", &different.source_context, 10)
.await
.unwrap();
assert_eq!(
matches[0].match_kind,
TranslationMemoryMatchKind::CandidateExact
);
assert!(!matches[0].can_auto_reuse);
}
#[tokio::test]
async fn rejects_empty_translation_candidates() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let error = repository
.upsert_candidate(draft("release-1", "Hello", " \n"))
.await
.unwrap_err();
assert!(error.to_string().contains("translated_text"));
assert_eq!(repository.summary().await.unwrap().record_count, 0);
}
}
+69 -9
View File
@@ -161,6 +161,12 @@ pub struct TranslationTaskUnitResult {
pub source_text: String, pub source_text: String,
/// Provider-produced or human-supplied translation. /// Provider-produced or human-supplied translation.
pub translated_text: String, pub translated_text: String,
/// Result source kind.
#[serde(default)]
pub source_kind: TranslationTaskResultSourceKind,
/// Trusted Translation Memory record used for this result, when applicable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_memory_record_id: Option<String>,
/// Provider identifier. /// Provider identifier.
pub provider: String, pub provider: String,
/// Provider run that produced this result. /// Provider run that produced this result.
@@ -169,6 +175,30 @@ pub struct TranslationTaskUnitResult {
pub translated_unix_seconds: u64, pub translated_unix_seconds: u64,
} }
/// Source of one persisted TextUnit translation result.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TranslationTaskResultSourceKind {
/// Result returned by the configured provider.
#[default]
Provider,
/// Result submitted through the manual task update interface.
Manual,
/// Result reused from a trusted Translation Memory entry.
TranslationMemory,
}
impl TranslationTaskResultSourceKind {
/// Returns the stable JSON label.
pub const fn as_str(self) -> &'static str {
match self {
Self::Provider => "provider",
Self::Manual => "manual",
Self::TranslationMemory => "translation_memory",
}
}
}
/// One provider execution associated with one or more translation units. /// One provider execution associated with one or more translation units.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProviderRun { pub struct ProviderRun {
@@ -247,12 +277,18 @@ pub fn build_translation_handoff(
Vec::new(), Vec::new(),
) )
}); });
let unit_status = match state.0 { let unit_status = if state.5.is_empty() {
TranslationTaskStatus::Queued => TranslationUnitStatus::Queued, match state.0 {
TranslationTaskStatus::Running => TranslationUnitStatus::Translating, TranslationTaskStatus::Queued => TranslationUnitStatus::Queued,
TranslationTaskStatus::Failed => TranslationUnitStatus::Failed, TranslationTaskStatus::Running => TranslationUnitStatus::Translating,
TranslationTaskStatus::Completed => TranslationUnitStatus::Translated, TranslationTaskStatus::Failed => TranslationUnitStatus::Failed,
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped, TranslationTaskStatus::Completed => TranslationUnitStatus::Translated,
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped,
}
} else {
// A task can retain successful TM hits while the remaining provider
// units are failed or waiting for retry.
TranslationUnitStatus::Translated
}; };
let unit = TranslationUnit { let unit = TranslationUnit {
unit_id: task.task_id.clone(), unit_id: task.task_id.clone(),
@@ -1168,11 +1204,29 @@ impl SqliteTranslationTaskRepository {
pub async fn fail_claim( pub async fn fail_claim(
&self, &self,
failure: TranslationTaskFailure, failure: TranslationTaskFailure,
) -> Result<PersistedTranslationTask> {
self.fail_claim_with_results(failure, &[]).await
}
/// Records a provider failure while retaining any already-resolved TextUnit
/// results, such as trusted Translation Memory hits.
pub async fn fail_claim_with_results(
&self,
failure: TranslationTaskFailure,
translation_results: &[TranslationTaskUnitResult],
) -> Result<PersistedTranslationTask> { ) -> Result<PersistedTranslationTask> {
let now = unix_seconds_now_i64(); let now = unix_seconds_now_i64();
let next_attempt = failure let next_attempt = failure
.next_attempt_unix_seconds .next_attempt_unix_seconds
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)); .map(|value| i64::try_from(value).unwrap_or(i64::MAX));
let translation_results_json = if translation_results.is_empty() {
None
} else {
Some(
serde_json::to_string(translation_results)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?,
)
};
let result = sqlx::query( let result = sqlx::query(
r#" r#"
UPDATE translation_tasks UPDATE translation_tasks
@@ -1183,11 +1237,12 @@ impl SqliteTranslationTaskRepository {
lease_expires_unix_seconds = NULL, lease_expires_unix_seconds = NULL,
failure_class = ?4, failure_class = ?4,
failure_retryable = ?5, failure_retryable = ?5,
next_attempt_unix_seconds = ?6 next_attempt_unix_seconds = ?6,
translation_results_json = COALESCE(?7, translation_results_json)
WHERE task_id = ?1 WHERE task_id = ?1
AND worker_status = 'running' AND worker_status = 'running'
AND lease_owner = ?7 AND lease_owner = ?8
AND provider_run_id = ?8 AND provider_run_id = ?9
"#, "#,
) )
.bind(&failure.task_id) .bind(&failure.task_id)
@@ -1196,6 +1251,7 @@ impl SqliteTranslationTaskRepository {
.bind(&failure.failure_class) .bind(&failure.failure_class)
.bind(if failure.retryable { 1_i64 } else { 0_i64 }) .bind(if failure.retryable { 1_i64 } else { 0_i64 })
.bind(next_attempt) .bind(next_attempt)
.bind(translation_results_json)
.bind(&failure.worker_id) .bind(&failure.worker_id)
.bind(&failure.provider_run_id) .bind(&failure.provider_run_id)
.execute(&self.pool) .execute(&self.pool)
@@ -1747,6 +1803,8 @@ mod tests {
unit_id: "unit-a".to_string(), unit_id: "unit-a".to_string(),
source_text: "source".to_string(), source_text: "source".to_string(),
translated_text: "manual translation".to_string(), translated_text: "manual translation".to_string(),
source_kind: TranslationTaskResultSourceKind::Manual,
translation_memory_record_id: None,
provider: "manual".to_string(), provider: "manual".to_string(),
provider_run_id: "manual-run-1".to_string(), provider_run_id: "manual-run-1".to_string(),
translated_unix_seconds: 321, translated_unix_seconds: 321,
@@ -1864,6 +1922,8 @@ mod tests {
unit_id: "unit-a".to_string(), unit_id: "unit-a".to_string(),
source_text: "source".to_string(), source_text: "source".to_string(),
translated_text: "translated".to_string(), translated_text: "translated".to_string(),
source_kind: TranslationTaskResultSourceKind::Provider,
translation_memory_record_id: None,
provider: "mock".to_string(), provider: "mock".to_string(),
provider_run_id: second_run.clone(), provider_run_id: second_run.clone(),
translated_unix_seconds: 1, translated_unix_seconds: 1,
+633 -68
View File
@@ -2,15 +2,21 @@
//! //!
//! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态,不 //! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态,不
//! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract;状态、 //! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract;状态、
//! 租约和译文结果始终写入 `translation-tasks.sqlite` //! 租约和任务结果写入 release 级 `translation-tasks.sqlite`,跨 release 的
//! Translation Memory 写入项目级独立 SQLite 数据库。
use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit}; use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit};
use crate::official_textunit_queue::read_textunit_task_queue_at; use crate::official_textunit_queue::read_textunit_task_queue_at;
use crate::translation_memory::{translation_memory_context, SqliteTranslationMemoryRepository};
use crate::translation_tasks::{ use crate::translation_tasks::{
PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure, PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure,
TranslationTaskUnitResult, TranslationTaskResultSourceKind, TranslationTaskUnitResult,
}; };
use async_trait::async_trait; use async_trait::async_trait;
use bat_core::domain::{
TranslationMemoryDraft, TranslationMemorySourceKind, TranslationMemorySourceTrace,
};
use bat_core::repositories::TranslationMemoryRepository;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use std::env; use std::env;
@@ -89,6 +95,9 @@ pub struct TranslationWorkerConfig {
pub max_tasks: Option<usize>, pub max_tasks: Option<usize>,
/// worker 实例前缀,用于 lease 诊断。 /// worker 实例前缀,用于 lease 诊断。
pub worker_id: String, pub worker_id: String,
/// Translation Memory SQLite path. `None` uses the output-root default.
#[serde(skip_serializing_if = "Option::is_none")]
pub translation_memory_path: Option<PathBuf>,
} }
impl Default for TranslationWorkerConfig { impl Default for TranslationWorkerConfig {
@@ -102,6 +111,7 @@ impl Default for TranslationWorkerConfig {
retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF, retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
max_tasks: None, max_tasks: None,
worker_id: format!("bat-worker-{}", std::process::id()), worker_id: format!("bat-worker-{}", std::process::id()),
translation_memory_path: None,
} }
} }
} }
@@ -576,6 +586,16 @@ pub struct TranslationWorkerReport {
pub remaining_count: usize, pub remaining_count: usize,
/// 失败诊断。 /// 失败诊断。
pub failures: Vec<TranslationWorkerFailure>, pub failures: Vec<TranslationWorkerFailure>,
/// Translation Memory database path used by this run.
pub translation_memory_path: PathBuf,
/// Whether the Translation Memory repository was available.
pub translation_memory_available: bool,
/// Number of TextUnits reused from trusted Translation Memory.
pub translation_memory_hit_count: usize,
/// Number of TextUnits sent to the provider after Translation Memory lookup.
pub provider_unit_count: usize,
/// Translation Memory diagnostics that did not invalidate provider work.
pub translation_memory_failures: Vec<String>,
} }
/// worker 失败诊断。 /// worker 失败诊断。
@@ -599,7 +619,10 @@ struct WorkerStats {
completed_count: AtomicUsize, completed_count: AtomicUsize,
failed_count: AtomicUsize, failed_count: AtomicUsize,
retry_scheduled_count: AtomicUsize, retry_scheduled_count: AtomicUsize,
translation_memory_hit_count: AtomicUsize,
provider_unit_count: AtomicUsize,
failures: Mutex<Vec<TranslationWorkerFailure>>, failures: Mutex<Vec<TranslationWorkerFailure>>,
translation_memory_failures: Mutex<Vec<String>>,
} }
struct WorkerTaskContext<'a> { struct WorkerTaskContext<'a> {
@@ -611,12 +634,25 @@ struct WorkerTaskContext<'a> {
max_attempts: u32, max_attempts: u32,
retry_backoff: Duration, retry_backoff: Duration,
stats: &'a WorkerStats, stats: &'a WorkerStats,
translation_memory: Option<&'a dyn TranslationMemoryRepository>,
} }
/// 运行一个 provider worker 轮次。 /// 运行一个 provider worker 轮次。
pub async fn run_translation_worker_at( pub async fn run_translation_worker_at(
resource_root: &Path, resource_root: &Path,
config: &TranslationWorkerConfig, config: &TranslationWorkerConfig,
) -> anyhow::Result<TranslationWorkerReport> {
run_translation_worker_at_with_cancellation(resource_root, config, Arc::new(|| false)).await
}
/// 运行一个可协作取消的 provider worker 轮次。
///
/// 取消只在 claim 循环边界检查;已经开始的 provider 请求会先完成,
/// 避免丢失 lease 结果或遗留未清理的外部子进程。
pub async fn run_translation_worker_at_with_cancellation(
resource_root: &Path,
config: &TranslationWorkerConfig,
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
) -> anyhow::Result<TranslationWorkerReport> { ) -> anyhow::Result<TranslationWorkerReport> {
config.validate()?; config.validate()?;
let provider: Arc<dyn TranslationProvider> = match config.provider { let provider: Arc<dyn TranslationProvider> = match config.provider {
@@ -625,7 +661,13 @@ pub async fn run_translation_worker_at(
)?), )?),
TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?), TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?),
}; };
run_translation_worker_with_provider(resource_root, config, provider).await run_translation_worker_with_provider_and_cancellation(
resource_root,
config,
provider,
should_cancel,
)
.await
} }
/// 使用指定 provider 运行 worker,供测试和插件宿主使用。 /// 使用指定 provider 运行 worker,供测试和插件宿主使用。
@@ -633,6 +675,21 @@ pub async fn run_translation_worker_with_provider(
resource_root: &Path, resource_root: &Path,
config: &TranslationWorkerConfig, config: &TranslationWorkerConfig,
provider: Arc<dyn TranslationProvider>, provider: Arc<dyn TranslationProvider>,
) -> anyhow::Result<TranslationWorkerReport> {
run_translation_worker_with_provider_and_cancellation(
resource_root,
config,
provider,
Arc::new(|| false),
)
.await
}
async fn run_translation_worker_with_provider_and_cancellation(
resource_root: &Path,
config: &TranslationWorkerConfig,
provider: Arc<dyn TranslationProvider>,
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
) -> anyhow::Result<TranslationWorkerReport> { ) -> anyhow::Result<TranslationWorkerReport> {
config.validate()?; config.validate()?;
let queue = read_textunit_task_queue_at(resource_root) let queue = read_textunit_task_queue_at(resource_root)
@@ -643,6 +700,21 @@ pub async fn run_translation_worker_with_provider(
.map_err(anyhow::Error::msg)? .map_err(anyhow::Error::msg)?
.ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?, .ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?,
); );
let translation_memory_path = config
.translation_memory_path
.clone()
.unwrap_or_else(|| SqliteTranslationMemoryRepository::repository_path(resource_root));
let (translation_memory, translation_memory_startup_failure) =
match SqliteTranslationMemoryRepository::new(&translation_memory_path).await {
Ok(repository) => (Some(Arc::new(repository)), None),
Err(error) => (
None,
Some(format!(
"打开 Translation Memory 数据库失败 {}{error}",
translation_memory_path.display()
)),
),
};
let repository = Arc::new( let repository = Arc::new(
SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path( SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path(
resource_root, resource_root,
@@ -659,6 +731,13 @@ pub async fn run_translation_worker_with_provider(
.await .await
.map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败:{error}"))?; .map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败:{error}"))?;
let stats = Arc::new(WorkerStats::default()); let stats = Arc::new(WorkerStats::default());
if let Some(failure) = translation_memory_startup_failure {
stats
.translation_memory_failures
.lock()
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
.push(failure);
}
let claimed_limit = Arc::new(AtomicUsize::new(0)); let claimed_limit = Arc::new(AtomicUsize::new(0));
let mut handles = Vec::with_capacity(config.concurrency); let mut handles = Vec::with_capacity(config.concurrency);
@@ -674,8 +753,13 @@ pub async fn run_translation_worker_with_provider(
let max_attempts = config.max_attempts; let max_attempts = config.max_attempts;
let lease_seconds = config.lease_seconds; let lease_seconds = config.lease_seconds;
let retry_backoff = config.retry_backoff; let retry_backoff = config.retry_backoff;
let translation_memory = translation_memory.clone();
let should_cancel = Arc::clone(&should_cancel);
handles.push(tokio::spawn(async move { handles.push(tokio::spawn(async move {
loop { loop {
if should_cancel() {
return Err(anyhow::anyhow!("翻译 worker 已取消"));
}
if let Some(max_tasks) = max_tasks { if let Some(max_tasks) = max_tasks {
let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel); let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel);
if reservation >= max_tasks { if reservation >= max_tasks {
@@ -704,6 +788,9 @@ pub async fn run_translation_worker_with_provider(
max_attempts, max_attempts,
retry_backoff, retry_backoff,
stats: &stats, stats: &stats,
translation_memory: translation_memory
.as_deref()
.map(|repository| repository as &dyn TranslationMemoryRepository),
}, },
&task, &task,
) )
@@ -712,10 +799,24 @@ pub async fn run_translation_worker_with_provider(
Ok::<(), anyhow::Error>(()) Ok::<(), anyhow::Error>(())
})); }));
} }
let mut first_worker_error = None;
for handle in handles { for handle in handles {
handle match handle.await {
.await Ok(Ok(())) => {}
.map_err(|error| anyhow::anyhow!("等待翻译 worker 失败:{error}"))??; Ok(Err(error)) => {
if first_worker_error.is_none() {
first_worker_error = Some(error);
}
}
Err(error) => {
if first_worker_error.is_none() {
first_worker_error = Some(anyhow::anyhow!("等待翻译 worker 失败:{error}"));
}
}
}
}
if let Some(error) = first_worker_error {
return Err(error);
} }
let remaining_count = repository let remaining_count = repository
@@ -736,6 +837,11 @@ pub async fn run_translation_worker_with_provider(
.map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))? .map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))?
.clone(); .clone();
let failed_count = stats.failed_count.load(Ordering::Relaxed); let failed_count = stats.failed_count.load(Ordering::Relaxed);
let translation_memory_failures = stats
.translation_memory_failures
.lock()
.map_err(|_| anyhow::anyhow!("读取 Translation Memory 诊断时 mutex poisoned"))?
.clone();
Ok(TranslationWorkerReport { Ok(TranslationWorkerReport {
command: "translation-worker", command: "translation-worker",
status: if failed_count == 0 { status: if failed_count == 0 {
@@ -752,6 +858,11 @@ pub async fn run_translation_worker_with_provider(
retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed), retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed),
remaining_count, remaining_count,
failures, failures,
translation_memory_path,
translation_memory_available: translation_memory.is_some(),
translation_memory_hit_count: stats.translation_memory_hit_count.load(Ordering::Relaxed),
provider_unit_count: stats.provider_unit_count.load(Ordering::Relaxed),
translation_memory_failures,
}) })
} }
@@ -759,66 +870,176 @@ async fn process_claimed_task(
context: &WorkerTaskContext<'_>, context: &WorkerTaskContext<'_>,
task: &PersistedTranslationTask, task: &PersistedTranslationTask,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let request = match provider_request(task, context.index) { let task_units = task_index_units(task, context.index)?;
Ok(request) => request, let mut results = BTreeMap::new();
Err(error) => { let mut provider_units = Vec::new();
record_provider_failure( for unit in &task_units {
context, if let Some(translation_memory) = context.translation_memory {
task, let source_context = translation_memory_context(
TranslationProviderError::new( &unit.destination,
TranslationProviderFailureClass::InvalidRequest, unit.archive_entry.as_deref(),
error.to_string(), unit.serialized_file.as_deref(),
), unit.path_id,
) unit.class_id,
.await?; unit.field_path.as_deref(),
return Ok(()); unit.format.as_deref(),
} unit.asset_name.as_deref(),
}; unit.text_source_kind.as_deref(),
match context.provider.translate(request.clone()).await { &unit.context,
Ok(response) => { );
let results = match translation_memory
match validate_provider_response(&request, response, context.provider_name) { .find_matches(&unit.source_text, &source_context, 1)
Ok(results) => results, .await
Err(error) => { {
record_provider_failure( Ok(matches) => {
context, if let Some(found) = matches.into_iter().find(|item| item.can_auto_reuse) {
task, context
TranslationProviderError::new( .stats
TranslationProviderFailureClass::InvalidRequest, .translation_memory_hit_count
error.to_string(), .fetch_add(1, Ordering::Relaxed);
), results.insert(
) unit.id.clone(),
.await?; translation_memory_result(task, unit, &found.entry),
return Ok(()); );
continue;
} }
}; }
context Err(error) => {
.repository record_translation_memory_failure(
.complete_claim( context,
&task.task.task_id, format!(
context.worker_id, "任务 {} TextUnit {} 查询失败:{}",
&task.provider_run_id.clone().unwrap_or_default(), task.task.task_id, unit.id, error
context.provider_name, ),
)?;
}
}
}
provider_units.push(*unit);
}
if !provider_units.is_empty() {
context
.stats
.provider_unit_count
.fetch_add(provider_units.len(), Ordering::Relaxed);
let request = match provider_request(task, &provider_units) {
Ok(request) => request,
Err(error) => {
record_provider_failure(
context,
task,
TranslationProviderError::new(
TranslationProviderFailureClass::InvalidRequest,
error.to_string(),
),
&results, &results,
) )
.await .await?;
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?; return Ok(());
context }
.stats };
.completed_count match context.provider.translate(request.clone()).await {
.fetch_add(1, Ordering::Relaxed); Ok(response) => {
} let provider_results =
Err(error) => { match validate_provider_response(&request, response, context.provider_name) {
record_provider_failure(context, task, error).await?; Ok(results) => results,
Err(error) => {
record_provider_failure(
context,
task,
TranslationProviderError::new(
TranslationProviderFailureClass::InvalidRequest,
error.to_string(),
),
&results,
)
.await?;
return Ok(());
}
};
for result in &provider_results {
results.insert(result.unit_id.clone(), result.clone());
if let Some(unit) = provider_units.iter().find(|unit| unit.id == result.unit_id)
{
let source_context = translation_memory_context(
&unit.destination,
unit.archive_entry.as_deref(),
unit.serialized_file.as_deref(),
unit.path_id,
unit.class_id,
unit.field_path.as_deref(),
unit.format.as_deref(),
unit.asset_name.as_deref(),
unit.text_source_kind.as_deref(),
&unit.context,
);
let draft = TranslationMemoryDraft {
source_text: result.source_text.clone(),
source_context,
translated_text: result.translated_text.clone(),
translation_source_kind: TranslationMemorySourceKind::Provider,
official_release_id: task.task.official_release_id.clone(),
source_trace: translation_memory_trace(task, unit),
provider: Some(context.provider_name.to_string()),
provider_run_id: Some(request.provider_run_id.clone()),
observed_unix_seconds: unix_seconds_now(),
};
if let Some(translation_memory) = context.translation_memory {
if let Err(error) = translation_memory.upsert_candidate(draft).await {
record_translation_memory_failure(
context,
format!(
"任务 {} TextUnit {} 写入候选失败:{}",
task.task.task_id, unit.id, error
),
)?;
}
}
}
}
}
Err(error) => {
record_provider_failure(context, task, error, &results).await?;
return Ok(());
}
} }
} }
let ordered_results = task_units
.iter()
.filter_map(|unit| results.get(&unit.id).cloned())
.collect::<Vec<_>>();
if ordered_results.len() != task_units.len() {
return Err(anyhow::anyhow!(
"任务 {} 的译文结果不完整:expected={} actual={}",
task.task.task_id,
task_units.len(),
ordered_results.len()
));
}
context
.repository
.complete_claim(
&task.task.task_id,
context.worker_id,
&task.provider_run_id.clone().unwrap_or_default(),
context.provider_name,
&ordered_results,
)
.await
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?;
context
.stats
.completed_count
.fetch_add(1, Ordering::Relaxed);
Ok(()) Ok(())
} }
fn provider_request( fn task_index_units<'a>(
task: &PersistedTranslationTask, task: &PersistedTranslationTask,
index: &crate::official_parse::OfficialTextUnitIndex, index: &'a crate::official_parse::OfficialTextUnitIndex,
) -> anyhow::Result<TranslationProviderRequest> { ) -> anyhow::Result<Vec<&'a OfficialTextUnitIndexUnit>> {
let parse_entry_key = task let parse_entry_key = task
.task .task
.parse_entry_key .parse_entry_key
@@ -832,7 +1053,6 @@ fn provider_request(
&& unit.destination == task.task.destination && unit.destination == task.task.destination
&& unit.archive_entry == task.task.archive_entry && unit.archive_entry == task.task.archive_entry
}) })
.map(|unit| provider_unit(task, unit))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if units.is_empty() { if units.is_empty() {
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
@@ -840,6 +1060,13 @@ fn provider_request(
task.task.task_id task.task.task_id
)); ));
} }
Ok(units)
}
fn provider_request(
task: &PersistedTranslationTask,
index_units: &[&OfficialTextUnitIndexUnit],
) -> anyhow::Result<TranslationProviderRequest> {
let provider_run_id = task let provider_run_id = task
.provider_run_id .provider_run_id
.clone() .clone()
@@ -850,7 +1077,10 @@ fn provider_request(
task_id: task.task.task_id.clone(), task_id: task.task.task_id.clone(),
destination: task.task.destination.clone(), destination: task.task.destination.clone(),
archive_entry: task.task.archive_entry.clone(), archive_entry: task.task.archive_entry.clone(),
units, units: index_units
.iter()
.map(|unit| provider_unit(task, unit))
.collect(),
}) })
} }
@@ -914,10 +1144,18 @@ fn validate_provider_response(
result.unit_id result.unit_id
)); ));
} }
if result.translated_text.trim().is_empty() {
return Err(anyhow::anyhow!(
"provider 返回空 translated_text{}",
result.unit_id
));
}
results.push(TranslationTaskUnitResult { results.push(TranslationTaskUnitResult {
unit_id: result.unit_id, unit_id: result.unit_id,
source_text: result.source_text, source_text: result.source_text,
translated_text: result.translated_text, translated_text: result.translated_text,
source_kind: TranslationTaskResultSourceKind::Provider,
translation_memory_record_id: None,
provider: provider_name.to_string(), provider: provider_name.to_string(),
provider_run_id: request.provider_run_id.clone(), provider_run_id: request.provider_run_id.clone(),
translated_unix_seconds: unix_seconds_now(), translated_unix_seconds: unix_seconds_now(),
@@ -933,25 +1171,81 @@ fn validate_provider_response(
Ok(results) Ok(results)
} }
fn translation_memory_result(
task: &PersistedTranslationTask,
unit: &OfficialTextUnitIndexUnit,
entry: &bat_core::domain::TranslationMemoryEntry,
) -> TranslationTaskUnitResult {
TranslationTaskUnitResult {
unit_id: unit.id.clone(),
source_text: unit.source_text.clone(),
translated_text: entry.translated_text.clone(),
source_kind: TranslationTaskResultSourceKind::TranslationMemory,
translation_memory_record_id: Some(entry.record_id.clone()),
provider: "translation_memory".to_string(),
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
translated_unix_seconds: unix_seconds_now(),
}
}
fn translation_memory_trace(
task: &PersistedTranslationTask,
unit: &OfficialTextUnitIndexUnit,
) -> TranslationMemorySourceTrace {
TranslationMemorySourceTrace {
official_release_id: task.task.official_release_id.clone(),
unit_id: Some(unit.id.clone()),
task_id: Some(task.task.task_id.clone()),
destination: Some(unit.destination.clone()),
archive_entry: unit.archive_entry.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
format: unit.format.clone(),
asset_name: unit.asset_name.clone(),
text_source_kind: unit.text_source_kind.clone(),
source_url: Some(unit.source_url.clone()),
}
}
fn record_translation_memory_failure(
context: &WorkerTaskContext<'_>,
message: String,
) -> anyhow::Result<()> {
context
.stats
.translation_memory_failures
.lock()
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
.push(message);
Ok(())
}
async fn record_provider_failure( async fn record_provider_failure(
context: &WorkerTaskContext<'_>, context: &WorkerTaskContext<'_>,
task: &PersistedTranslationTask, task: &PersistedTranslationTask,
error: TranslationProviderError, error: TranslationProviderError,
partial_results: &BTreeMap<String, TranslationTaskUnitResult>,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let retryable = error.retryable && task.attempt_count < context.max_attempts; let retryable = error.retryable && task.attempt_count < context.max_attempts;
let next_attempt = let next_attempt =
retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs())); retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs()));
let partial_results = partial_results.values().cloned().collect::<Vec<_>>();
context context
.repository .repository
.fail_claim(TranslationTaskFailure { .fail_claim_with_results(
task_id: task.task.task_id.clone(), TranslationTaskFailure {
worker_id: context.worker_id.to_string(), task_id: task.task.task_id.clone(),
provider_run_id: task.provider_run_id.clone().unwrap_or_default(), worker_id: context.worker_id.to_string(),
failure_class: error.class.as_str().to_string(), provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
failure_reason: error.message.clone(), failure_class: error.class.as_str().to_string(),
retryable, failure_reason: error.message.clone(),
next_attempt_unix_seconds: next_attempt, retryable,
}) next_attempt_unix_seconds: next_attempt,
},
&partial_results,
)
.await .await
.map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败:{failure}"))?; .map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败:{failure}"))?;
context.stats.failed_count.fetch_add(1, Ordering::Relaxed); context.stats.failed_count.fetch_add(1, Ordering::Relaxed);
@@ -1161,6 +1455,277 @@ mod tests {
assert_eq!(task.translation_results[0].translated_text, "translated-0"); assert_eq!(task.translation_results[0].translated_text, "translated-0");
} }
#[tokio::test]
async fn worker_honors_cancellation_before_claiming_tasks() {
let (temp, queue) = fixture_root();
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
crate::official_parse::write_textunit_index_at(temp.path(), &index(temp.path())).unwrap();
let config = TranslationWorkerConfig {
concurrency: 2,
..TranslationWorkerConfig::default()
};
let result =
run_translation_worker_at_with_cancellation(temp.path(), &config, Arc::new(|| true))
.await;
assert!(result.unwrap_err().to_string().contains("已取消"));
let repository = SqliteTranslationTaskRepository::open(
SqliteTranslationTaskRepository::repository_path(temp.path()),
)
.await
.unwrap();
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
assert_eq!(
task.task_status,
crate::translation_tasks::TranslationTaskStatus::Queued
);
assert_eq!(task.attempt_count, 0);
}
#[tokio::test]
async fn worker_reuses_trusted_tm_for_part_of_a_task_and_calls_provider_for_the_rest() {
let (temp, queue) = fixture_root();
let textunit_index = index(temp.path());
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
let translation_memory_path = temp.path().join("translation-memory.sqlite");
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
.await
.unwrap();
let unit = &textunit_index.units[0];
let source_context = translation_memory_context(
&unit.destination,
unit.archive_entry.as_deref(),
unit.serialized_file.as_deref(),
unit.path_id,
unit.class_id,
unit.field_path.as_deref(),
unit.format.as_deref(),
unit.asset_name.as_deref(),
unit.text_source_kind.as_deref(),
&unit.context,
);
let candidate = translation_memory
.upsert_candidate(TranslationMemoryDraft {
source_text: unit.source_text.clone(),
source_context,
translated_text: "trusted-from-tm".to_string(),
translation_source_kind: TranslationMemorySourceKind::Manual,
official_release_id: queue.official_release_id.clone(),
source_trace: TranslationMemorySourceTrace {
official_release_id: queue.official_release_id.clone(),
unit_id: Some(unit.id.clone()),
task_id: Some(queue.tasks[0].task_id.clone()),
destination: Some(unit.destination.clone()),
archive_entry: unit.archive_entry.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
format: unit.format.clone(),
asset_name: unit.asset_name.clone(),
text_source_kind: unit.text_source_kind.clone(),
source_url: Some(unit.source_url.clone()),
},
provider: None,
provider_run_id: None,
observed_unix_seconds: 1,
})
.await
.unwrap();
let trusted = translation_memory
.confirm(
&candidate.record_id,
"test-reviewer",
Some("accepted".to_string()),
)
.await
.unwrap();
assert_eq!(
trusted.trust_status,
bat_core::domain::TranslationMemoryTrustStatus::Trusted
);
let fixture = temp.path().join("partial-mock.json");
std::fs::write(
&fixture,
serde_json::to_vec(&serde_json::json!({
"schema_version": 1,
"translations": {
"direct:bundle#unit:1": "translated-by-provider"
}
}))
.unwrap(),
)
.unwrap();
let config = TranslationWorkerConfig {
fixture_path: Some(fixture),
translation_memory_path: Some(translation_memory_path),
concurrency: 1,
retry_backoff: Duration::ZERO,
..TranslationWorkerConfig::default()
};
let report = run_translation_worker_at(temp.path(), &config)
.await
.unwrap();
assert_eq!(report.translation_memory_hit_count, 1);
assert_eq!(report.provider_unit_count, 1);
assert_eq!(report.completed_count, 1);
let repository = SqliteTranslationTaskRepository::open(
SqliteTranslationTaskRepository::repository_path(temp.path()),
)
.await
.unwrap();
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
assert_eq!(task.translation_results.len(), 2);
assert_eq!(
task.translation_results[0].translated_text,
"trusted-from-tm"
);
assert_eq!(
task.translation_results[0].source_kind,
TranslationTaskResultSourceKind::TranslationMemory
);
assert_eq!(
task.translation_results[0]
.translation_memory_record_id
.as_deref(),
Some(trusted.record_id.as_str())
);
assert_eq!(
task.translation_results[1].translated_text,
"translated-by-provider"
);
assert_eq!(
task.translation_results[1].source_kind,
TranslationTaskResultSourceKind::Provider
);
}
#[tokio::test]
async fn worker_retains_tm_hits_when_provider_fails_for_remaining_units() {
let (temp, queue) = fixture_root();
let textunit_index = index(temp.path());
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
let translation_memory_path = temp.path().join("translation-memory.sqlite");
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
.await
.unwrap();
let unit = &textunit_index.units[0];
let source_context = translation_memory_context(
&unit.destination,
unit.archive_entry.as_deref(),
unit.serialized_file.as_deref(),
unit.path_id,
unit.class_id,
unit.field_path.as_deref(),
unit.format.as_deref(),
unit.asset_name.as_deref(),
unit.text_source_kind.as_deref(),
&unit.context,
);
let candidate = translation_memory
.upsert_candidate(TranslationMemoryDraft {
source_text: unit.source_text.clone(),
source_context,
translated_text: "trusted-from-tm".to_string(),
translation_source_kind: TranslationMemorySourceKind::Manual,
official_release_id: queue.official_release_id.clone(),
source_trace: TranslationMemorySourceTrace {
official_release_id: queue.official_release_id.clone(),
unit_id: Some(unit.id.clone()),
task_id: Some(queue.tasks[0].task_id.clone()),
destination: Some(unit.destination.clone()),
archive_entry: unit.archive_entry.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
format: unit.format.clone(),
asset_name: unit.asset_name.clone(),
text_source_kind: unit.text_source_kind.clone(),
source_url: Some(unit.source_url.clone()),
},
provider: None,
provider_run_id: None,
observed_unix_seconds: 1,
})
.await
.unwrap();
let trusted = translation_memory
.confirm(
&candidate.record_id,
"test-reviewer",
Some("accepted".to_string()),
)
.await
.unwrap();
let fixture = temp.path().join("provider-failure.json");
std::fs::write(
&fixture,
serde_json::to_vec(&serde_json::json!({
"schema_version": 1,
"failures": {
"textunit/release-1/bundle": {
"class": "rate_limited",
"message": "fixture throttled",
"retryable": false
}
}
}))
.unwrap(),
)
.unwrap();
let config = TranslationWorkerConfig {
fixture_path: Some(fixture),
translation_memory_path: Some(translation_memory_path),
concurrency: 1,
max_attempts: 1,
retry_backoff: Duration::ZERO,
..TranslationWorkerConfig::default()
};
let report = run_translation_worker_at(temp.path(), &config)
.await
.unwrap();
assert_eq!(report.translation_memory_hit_count, 1);
assert_eq!(report.provider_unit_count, 1);
assert_eq!(report.failed_count, 1);
let repository = SqliteTranslationTaskRepository::open(
SqliteTranslationTaskRepository::repository_path(temp.path()),
)
.await
.unwrap();
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
assert_eq!(
task.task_status,
crate::translation_tasks::TranslationTaskStatus::Failed
);
assert_eq!(task.translation_results.len(), 1);
assert_eq!(
task.translation_results[0]
.translation_memory_record_id
.as_deref(),
Some(trusted.record_id.as_str())
);
let handoff = crate::translation_tasks::build_translation_handoff(&queue, &[task]);
assert_eq!(
handoff.units[0].status,
crate::translation_tasks::TranslationUnitStatus::Translated
);
assert_eq!(
handoff.job.status,
crate::translation_tasks::TranslationJobStatus::Failed
);
}
#[tokio::test] #[tokio::test]
async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() { async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() {
let (temp, queue) = fixture_root(); let (temp, queue) = fixture_root();
+31 -7
View File
@@ -9,7 +9,7 @@ use crate::path_security::{
use crate::{ use crate::{
LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch, LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch,
LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository, LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository,
TranslationTaskStatus, TranslationTaskUnitResult, TranslationTaskResultSourceKind, TranslationTaskStatus, TranslationTaskUnitResult,
}; };
use bat_assetbundle::{ use bat_assetbundle::{
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch, patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
@@ -72,6 +72,12 @@ pub struct TranslationWorkbenchEntry {
/// Provider run that produced this translation, when imported from worker output. /// Provider run that produced this translation, when imported from worker output.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>, pub provider_run_id: Option<String>,
/// Source of the worker result (`provider`, `manual`, or `translation_memory`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_source_kind: Option<String>,
/// Trusted Translation Memory record used for this translation, when applicable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub translation_memory_record_id: Option<String>,
/// Worker completion time for provider-produced text. /// Worker completion time for provider-produced text.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub translated_unix_seconds: Option<u64>, pub translated_unix_seconds: Option<u64>,
@@ -616,6 +622,8 @@ fn localized_patch_metadata(entry: &TranslationWorkbenchEntry) -> LocalizedPatch
.to_string(), .to_string(),
translation_provider: entry.translation_provider.clone(), translation_provider: entry.translation_provider.clone(),
provider_run_id: entry.provider_run_id.clone(), provider_run_id: entry.provider_run_id.clone(),
translation_source_kind: entry.translation_source_kind.clone(),
translation_memory_record_id: entry.translation_memory_record_id.clone(),
review_status: entry review_status: entry
.review_status .review_status
.clone() .clone()
@@ -887,18 +895,30 @@ fn workbench_entry_from_worker_result(
) -> TranslationWorkbenchEntry { ) -> TranslationWorkbenchEntry {
let mut entry = TranslationWorkbenchEntry::from_index(unit); let mut entry = TranslationWorkbenchEntry::from_index(unit);
entry.translated_text = Some(result.translated_text.clone()); entry.translated_text = Some(result.translated_text.clone());
entry.translation_provider = task entry.translation_provider = match result.source_kind {
.provider TranslationTaskResultSourceKind::TranslationMemory => None,
.clone() TranslationTaskResultSourceKind::Provider | TranslationTaskResultSourceKind::Manual => task
.or_else(|| Some(result.provider.clone())) .provider
.filter(|provider| !provider.trim().is_empty()); .clone()
.or_else(|| Some(result.provider.clone()))
.filter(|provider| !provider.trim().is_empty()),
};
entry.provider_run_id = task entry.provider_run_id = task
.provider_run_id .provider_run_id
.clone() .clone()
.or_else(|| Some(result.provider_run_id.clone())) .or_else(|| Some(result.provider_run_id.clone()))
.filter(|provider_run_id| !provider_run_id.trim().is_empty()); .filter(|provider_run_id| !provider_run_id.trim().is_empty());
entry.translation_source_kind = Some(result.source_kind.as_str().to_string());
entry.translation_memory_record_id = result.translation_memory_record_id.clone();
entry.translated_unix_seconds = Some(result.translated_unix_seconds); entry.translated_unix_seconds = Some(result.translated_unix_seconds);
entry.review_status = Some("provider_completed".to_string()); entry.review_status = Some(
match result.source_kind {
TranslationTaskResultSourceKind::Provider => "provider_completed",
TranslationTaskResultSourceKind::Manual => "manual_submitted",
TranslationTaskResultSourceKind::TranslationMemory => "translation_memory_reused",
}
.to_string(),
);
entry entry
} }
@@ -938,6 +958,8 @@ impl TranslationWorkbenchEntry {
translated_text: None, translated_text: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
translated_unix_seconds: None, translated_unix_seconds: None,
review_status: None, review_status: None,
format: unit.format.clone(), format: unit.format.clone(),
@@ -975,6 +997,8 @@ mod tests {
translated_text: None, translated_text: None,
translation_provider: None, translation_provider: None,
provider_run_id: None, provider_run_id: None,
translation_source_kind: None,
translation_memory_record_id: None,
translated_unix_seconds: None, translated_unix_seconds: None,
review_status: None, review_status: None,
format: Some("plain".to_string()), format: Some("plain".to_string()),