mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 05:34:55 +08:00
feat(translation): add Rust Translation Memory and config migration
This commit is contained in:
@@ -4,6 +4,7 @@ pub mod game_client;
|
||||
pub mod game_version;
|
||||
pub mod resource;
|
||||
pub mod translation;
|
||||
pub mod translation_memory;
|
||||
|
||||
pub use game_client::{ClientStatus, GameClient, GameRegion};
|
||||
pub use game_version::{GameVersion, UnityVersion};
|
||||
@@ -14,3 +15,8 @@ pub use translation::{
|
||||
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
|
||||
TranslationStatus,
|
||||
};
|
||||
pub use translation_memory::{
|
||||
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||||
TranslationMemorySourceTrace, TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Translation Memory 领域对象。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// TM 记录的来源类型。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TranslationMemorySourceKind {
|
||||
/// 来自 provider 输出。
|
||||
Provider,
|
||||
/// 来自人工确认。
|
||||
Manual,
|
||||
/// 来自外部导入。
|
||||
Imported,
|
||||
}
|
||||
|
||||
impl TranslationMemorySourceKind {
|
||||
/// 返回稳定的持久化标签。
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Provider => "provider",
|
||||
Self::Manual => "manual",
|
||||
Self::Imported => "imported",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// TM 记录的可信状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TranslationMemoryTrustStatus {
|
||||
/// 候选记录,不能自动复用。
|
||||
Candidate,
|
||||
/// 已确认可信,可在强匹配时自动复用。
|
||||
Trusted,
|
||||
/// 已被后续记录取代。
|
||||
Superseded,
|
||||
/// 已明确拒绝。
|
||||
Rejected,
|
||||
}
|
||||
|
||||
impl TranslationMemoryTrustStatus {
|
||||
/// 返回稳定的持久化标签。
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Candidate => "candidate",
|
||||
Self::Trusted => "trusted",
|
||||
Self::Superseded => "superseded",
|
||||
Self::Rejected => "rejected",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// TM 查询结果的匹配类型。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TranslationMemoryMatchKind {
|
||||
/// 原始 source 和上下文都完全匹配,且记录可信,可自动复用。
|
||||
StrongExact,
|
||||
/// 原始 source 完全匹配,但上下文不同或不足,不能自动复用。
|
||||
CandidateExact,
|
||||
/// 原始 source 匹配,但上下文不兼容,不能自动复用。
|
||||
SourceOnly,
|
||||
}
|
||||
|
||||
impl TranslationMemoryMatchKind {
|
||||
/// 返回稳定的查询结果标签。
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::StrongExact => "strong_exact",
|
||||
Self::CandidateExact => "candidate_exact",
|
||||
Self::SourceOnly => "source_only",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 稳定的上下文键值。
|
||||
pub type TranslationMemoryContext = BTreeMap<String, String>;
|
||||
|
||||
/// TM 记录的 TextUnit / provider 溯源信息。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TranslationMemorySourceTrace {
|
||||
/// 源官方 release ID。
|
||||
pub official_release_id: String,
|
||||
/// 来源 TextUnit ID。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unit_id: Option<String>,
|
||||
/// 来源任务 ID。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub task_id: Option<String>,
|
||||
/// 源资源 destination。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub destination: Option<String>,
|
||||
/// 源 ZIP/archive entry。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub archive_entry: Option<String>,
|
||||
/// Unity serialized file。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialized_file: Option<String>,
|
||||
/// Unity object path ID。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path_id: Option<i64>,
|
||||
/// Unity class ID。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub class_id: Option<i32>,
|
||||
/// TypeTree 字段路径。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub field_path: Option<String>,
|
||||
/// TextUnit format。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub format: Option<String>,
|
||||
/// TextAsset 名称。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub asset_name: Option<String>,
|
||||
/// TextUnit 来源类型。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_source_kind: Option<String>,
|
||||
/// 源 URL。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source_url: Option<String>,
|
||||
}
|
||||
|
||||
/// TM 记录的候选输入。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TranslationMemoryDraft {
|
||||
/// 原始 source text。
|
||||
pub source_text: String,
|
||||
/// 稳定上下文。
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub source_context: TranslationMemoryContext,
|
||||
/// 译文。
|
||||
pub translated_text: String,
|
||||
/// 译文来源类型。
|
||||
pub translation_source_kind: TranslationMemorySourceKind,
|
||||
/// 源官方 release。
|
||||
pub official_release_id: String,
|
||||
/// 原始 TextUnit / provider 溯源。
|
||||
pub source_trace: TranslationMemorySourceTrace,
|
||||
/// provider。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider: Option<String>,
|
||||
/// provider run。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// 创建时间。
|
||||
pub observed_unix_seconds: u64,
|
||||
}
|
||||
|
||||
/// 持久化 TM 记录。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TranslationMemoryEntry {
|
||||
/// 稳定记录 ID。
|
||||
pub record_id: String,
|
||||
/// 原始 source text。
|
||||
pub source_text: String,
|
||||
/// source text hash。
|
||||
pub source_hash: String,
|
||||
/// 保守归一化后的 source text,仅用于辅助查询。
|
||||
pub normalized_source_text: String,
|
||||
/// 稳定上下文。
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub source_context: TranslationMemoryContext,
|
||||
/// 上下文 hash。
|
||||
pub source_context_hash: String,
|
||||
/// 译文。
|
||||
pub translated_text: String,
|
||||
/// 译文来源类型。
|
||||
pub translation_source_kind: TranslationMemorySourceKind,
|
||||
/// 当前可信状态。
|
||||
pub trust_status: TranslationMemoryTrustStatus,
|
||||
/// 源官方 release。
|
||||
pub official_release_id: String,
|
||||
/// 原始 TextUnit / provider 溯源。
|
||||
pub source_trace: TranslationMemorySourceTrace,
|
||||
/// provider。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider: Option<String>,
|
||||
/// provider run。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// 创建时间。
|
||||
pub created_unix_seconds: u64,
|
||||
/// 更新时间。
|
||||
pub updated_unix_seconds: u64,
|
||||
/// 可信确认时间。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub trusted_unix_seconds: Option<u64>,
|
||||
/// 可信确认人。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub trusted_by: Option<String>,
|
||||
/// 可信确认说明。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub trusted_reason: Option<String>,
|
||||
/// 该记录替代了哪条记录。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supersedes_record_id: Option<String>,
|
||||
/// 该记录被哪条记录替代。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub superseded_by_record_id: Option<String>,
|
||||
}
|
||||
|
||||
/// TM 查询结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TranslationMemoryMatch {
|
||||
/// 记录本体。
|
||||
pub entry: TranslationMemoryEntry,
|
||||
/// 匹配类型。
|
||||
pub match_kind: TranslationMemoryMatchKind,
|
||||
/// 是否允许自动复用。
|
||||
pub can_auto_reuse: bool,
|
||||
}
|
||||
|
||||
/// TM 仓储摘要。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TranslationMemorySummary {
|
||||
/// schema 版本。
|
||||
pub schema_version: u32,
|
||||
/// 记录总数。
|
||||
pub record_count: u64,
|
||||
/// 可信记录数。
|
||||
pub trusted_count: u64,
|
||||
/// 候选记录数。
|
||||
pub candidate_count: u64,
|
||||
/// 已替代记录数。
|
||||
pub superseded_count: u64,
|
||||
/// 已拒绝记录数。
|
||||
pub rejected_count: u64,
|
||||
}
|
||||
@@ -4,8 +4,10 @@
|
||||
|
||||
pub mod cas_repository;
|
||||
pub mod resource_repository;
|
||||
pub mod translation_memory_repository;
|
||||
pub mod translation_repository;
|
||||
|
||||
pub use cas_repository::CasRepository;
|
||||
pub use resource_repository::ResourceRepository;
|
||||
pub use translation_memory_repository::TranslationMemoryRepository;
|
||||
pub use translation_repository::TranslationRepository;
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
//! Translation Memory 仓储契约。
|
||||
|
||||
use crate::domain::{
|
||||
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||
TranslationMemoryMatch, TranslationMemorySummary,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
|
||||
/// 跨 official release 持久化的 Translation Memory 仓储。
|
||||
///
|
||||
/// 该契约只描述 V1 的精确查询和明确人工确认。仓储实现不得把
|
||||
/// `TranslationTaskStatus::Completed` 或 provider 成功隐式解释为 trusted。
|
||||
#[async_trait]
|
||||
pub trait TranslationMemoryRepository: Send + Sync {
|
||||
/// 保存一条 provider/manual/imported 译文候选。
|
||||
///
|
||||
/// 相同 source、上下文、译文、来源 release 和来源类型的重复写入必须幂等;
|
||||
/// 已 trusted 的记录不得被普通候选静默覆盖。
|
||||
async fn upsert_candidate(
|
||||
&self,
|
||||
draft: TranslationMemoryDraft,
|
||||
) -> crate::Result<TranslationMemoryEntry>;
|
||||
|
||||
/// 按原始 source text 和上下文查询精确匹配。
|
||||
///
|
||||
/// 实现可以返回 source 归一化后但原文不同的辅助候选,但这类结果不能自动复用。
|
||||
async fn find_matches(
|
||||
&self,
|
||||
source_text: &str,
|
||||
source_context: &TranslationMemoryContext,
|
||||
limit: usize,
|
||||
) -> crate::Result<Vec<TranslationMemoryMatch>>;
|
||||
|
||||
/// 显式确认一条记录为 trusted。
|
||||
async fn confirm(
|
||||
&self,
|
||||
record_id: &str,
|
||||
reviewer: &str,
|
||||
reason: Option<String>,
|
||||
) -> crate::Result<TranslationMemoryEntry>;
|
||||
|
||||
/// 按稳定记录 ID 读取一条 TM 记录。
|
||||
async fn find(&self, record_id: &str) -> crate::Result<TranslationMemoryEntry>;
|
||||
|
||||
/// 读取数据库和记录统计。
|
||||
async fn summary(&self) -> crate::Result<TranslationMemorySummary>;
|
||||
}
|
||||
+653
-208
File diff suppressed because it is too large
Load Diff
@@ -17,9 +17,11 @@ fn parse_with_env(values: &[&str], env: &[(&str, &str)]) -> anyhow::Result<CliOp
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect();
|
||||
parse_args_with_env(values.iter().map(|value| value.to_string()), move |key| {
|
||||
map.get(key).cloned()
|
||||
})
|
||||
parse_args_with_env(
|
||||
values.iter().map(|value| value.to_string()),
|
||||
move |key| map.get(key).cloned(),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -105,7 +107,7 @@ fn env_watch_daemon_only_affect_bare_run() {
|
||||
|
||||
#[test]
|
||||
fn env_values_do_not_break_status_and_reload_guard() {
|
||||
// .env 提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。
|
||||
// 配置文件/环境变量提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。
|
||||
let options = parse_with_env(
|
||||
&["bat", "status"],
|
||||
&[
|
||||
@@ -222,6 +224,148 @@ fn translation_worker_command_options_are_validated() {
|
||||
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_memory_commands_parse_and_validate() {
|
||||
let summary = parse(&["bat", "i18n", "memory", "summary"]).unwrap();
|
||||
assert_eq!(summary.command, CliCommand::TranslationMemorySummary);
|
||||
let query = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"tm",
|
||||
"query",
|
||||
"--tm-source-text",
|
||||
"Hello",
|
||||
"--tm-context-json",
|
||||
r#"{"destination":"story.bundle"}"#,
|
||||
"--limit",
|
||||
"5",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(query.command, CliCommand::TranslationMemoryQuery);
|
||||
assert_eq!(
|
||||
query.translation_memory_source_text.as_deref(),
|
||||
Some("Hello")
|
||||
);
|
||||
let confirm = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"confirm",
|
||||
"--tm-record-id",
|
||||
"tm-record",
|
||||
"--tm-reviewer",
|
||||
"reviewer",
|
||||
"--tm-reason",
|
||||
"accepted",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(confirm.command, CliCommand::TranslationMemoryConfirm);
|
||||
assert!(parse(&["bat", "i18n", "memory", "query"]).is_err());
|
||||
assert!(parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"confirm",
|
||||
"--tm-record-id",
|
||||
"tm-record"
|
||||
])
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_reload_accept_translation_worker_startup_options() {
|
||||
let restart = parse(&[
|
||||
"bat",
|
||||
"restart",
|
||||
"--translation-provider",
|
||||
"mock",
|
||||
"--translation-memory-path",
|
||||
"/tmp/tm.sqlite",
|
||||
"--worker-concurrency",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(restart.command, CliCommand::Restart);
|
||||
assert!(restart.translation_worker_option_explicit);
|
||||
assert_eq!(
|
||||
restart.translation_memory_path,
|
||||
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||
);
|
||||
assert_eq!(restart.worker_concurrency, 2);
|
||||
assert!(restart.config.auto_discover);
|
||||
|
||||
let reload = parse(&["bat", "reload", "--worker-id", "reload-worker"]).unwrap();
|
||||
assert_eq!(reload.command, CliCommand::Reload);
|
||||
assert!(reload.translation_worker_option_explicit);
|
||||
assert_eq!(reload.worker_id.as_deref(), Some("reload-worker"));
|
||||
assert!(reload.config.auto_discover);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_memory_command_options_do_not_apply_to_other_commands() {
|
||||
let error = parse(&["bat", "restart", "--tm-source-text", "Hello"]).unwrap_err();
|
||||
assert!(error.to_string().contains("查询/confirm"));
|
||||
|
||||
let error = parse(&["bat", "status", "--tm-record-id", "tm-record"]).unwrap_err();
|
||||
assert!(error.to_string().contains("查询/confirm"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_memory_subcommands_reject_irrelevant_options() {
|
||||
let error = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"summary",
|
||||
"--tm-source-text",
|
||||
"Hello",
|
||||
])
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("summary"));
|
||||
let error = parse(&["bat", "i18n", "memory", "summary", "--limit", "5"]).unwrap_err();
|
||||
assert!(error.to_string().contains("summary"));
|
||||
|
||||
let error = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"query",
|
||||
"--tm-source-text",
|
||||
"Hello",
|
||||
"--tm-record-id",
|
||||
"tm-record",
|
||||
])
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("confirm 参数"));
|
||||
let error = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"query",
|
||||
"--tm-source-text",
|
||||
"Hello",
|
||||
"--offset",
|
||||
"1",
|
||||
])
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("查询过滤"));
|
||||
|
||||
let error = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"memory",
|
||||
"confirm",
|
||||
"--tm-record-id",
|
||||
"tm-record",
|
||||
"--tm-reviewer",
|
||||
"reviewer",
|
||||
"--tm-context-json",
|
||||
"{}",
|
||||
])
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("query 参数"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_worker_env_defaults_apply() {
|
||||
let options = parse_with_env(
|
||||
@@ -229,6 +373,7 @@ fn translation_worker_env_defaults_apply() {
|
||||
&[
|
||||
("BAT_TRANSLATION_PROVIDER", "mock"),
|
||||
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
|
||||
("BAT_TRANSLATION_MEMORY_PATH", "/tmp/tm.sqlite"),
|
||||
("BAT_TRANSLATION_CONCURRENCY", "16"),
|
||||
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
|
||||
("BAT_TRANSLATION_LEASE_SECONDS", "120"),
|
||||
@@ -244,6 +389,10 @@ fn translation_worker_env_defaults_apply() {
|
||||
options.translation_fixture,
|
||||
Some(PathBuf::from("/tmp/fixture.json"))
|
||||
);
|
||||
assert_eq!(
|
||||
options.translation_memory_path,
|
||||
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||
);
|
||||
assert_eq!(options.worker_concurrency, 16);
|
||||
assert_eq!(options.worker_max_attempts, 5);
|
||||
assert_eq!(options.worker_lease_seconds, 120);
|
||||
@@ -610,6 +759,8 @@ fn translation_workbench_commands_read_update_and_clear_entries() {
|
||||
translated_text: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
translated_unix_seconds: None,
|
||||
review_status: None,
|
||||
format: Some("plain".to_string()),
|
||||
@@ -1030,46 +1181,72 @@ fn cli_download_concurrency_is_preserved_for_daemon_child() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_env_line_handles_quotes_and_rejects_bad_keys() {
|
||||
assert_eq!(
|
||||
parse_env_line("KEY=value"),
|
||||
Some(("KEY".to_string(), "value".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_env_line("KEY=\"quoted value\""),
|
||||
Some(("KEY".to_string(), "quoted value".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_env_line("KEY='single'"),
|
||||
Some(("KEY".to_string(), "single".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_env_line("BAT_OUTPUT = ./x"),
|
||||
Some(("BAT_OUTPUT".to_string(), "./x".to_string()))
|
||||
);
|
||||
assert_eq!(parse_env_line("no_equals_sign"), None);
|
||||
assert_eq!(parse_env_line("1BAD=x"), None);
|
||||
assert_eq!(parse_env_line("BAD KEY=x"), None);
|
||||
}
|
||||
fn config_file_is_applied_before_env_and_cli() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let config_path = temp.path().join(super::config_file::CONFIG_FILE_NAME);
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
r#"
|
||||
[runtime]
|
||||
state_dir = '/srv/state'
|
||||
output_format = 'json'
|
||||
|
||||
#[test]
|
||||
fn env_template_is_parseable_and_bootstrap_ready() {
|
||||
// 模板每个非注释行必须可解析;无参启动所需的最小配置默认启用。
|
||||
let mut keys = Vec::new();
|
||||
for line in ENV_TEMPLATE.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let (key, _) = parse_env_line(line).unwrap_or_else(|| panic!("模板行必须可解析:{line}"));
|
||||
keys.push(key);
|
||||
[resource]
|
||||
output_root = '/srv/from-config'
|
||||
auto_discover = true
|
||||
|
||||
[localized]
|
||||
output_root = '/srv/from-config-localized'
|
||||
|
||||
[network]
|
||||
proxy = 'none'
|
||||
download_concurrency = 12
|
||||
|
||||
[translation.worker]
|
||||
translation_memory_path = '/srv/config-tm.sqlite'
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut permissions = std::fs::metadata(&config_path).unwrap().permissions();
|
||||
permissions.set_mode(0o600);
|
||||
std::fs::set_permissions(&config_path, permissions).unwrap();
|
||||
}
|
||||
assert!(keys.contains(&"BAT_OUTPUT".to_string()));
|
||||
assert!(keys.contains(&"BAT_LOCALIZED_OUTPUT".to_string()));
|
||||
assert!(keys.contains(&"BAT_IMPORT_REPOSITORY".to_string()));
|
||||
assert!(keys.contains(&"BAT_IMPORT_CAS_ROOT".to_string()));
|
||||
assert!(keys.contains(&"BAT_IMPORT_RESOURCE_DB".to_string()));
|
||||
assert!(keys.contains(&"BAT_AUTO_DISCOVER".to_string()));
|
||||
let config = super::config_file::load_from_binary_dir(temp.path())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let options = parse_args_with_env(
|
||||
vec![
|
||||
"bat".to_string(),
|
||||
"--daemon".to_string(),
|
||||
"--output".to_string(),
|
||||
"/cli/output".to_string(),
|
||||
"--translation-memory-path".to_string(),
|
||||
"/cli/tm.sqlite".to_string(),
|
||||
],
|
||||
|key| match key {
|
||||
"BAT_OUTPUT" => Some("/env/output".to_string()),
|
||||
"BAT_TRANSLATION_MEMORY_PATH" => Some("/env/tm.sqlite".to_string()),
|
||||
_ => None,
|
||||
},
|
||||
Some(&config),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(options.state_dir, PathBuf::from("/srv/state"));
|
||||
assert_eq!(options.output_format, OutputFormat::Json);
|
||||
assert_eq!(options.config.output_root, PathBuf::from("/cli/output"));
|
||||
assert_eq!(
|
||||
options.config.localized_output_root,
|
||||
PathBuf::from("/srv/from-config-localized")
|
||||
);
|
||||
assert_eq!(options.config.download_concurrency, 12);
|
||||
assert_eq!(options.config.curl_proxy.mode(), &CurlProxyMode::Disabled);
|
||||
assert_eq!(
|
||||
options.translation_memory_path,
|
||||
Some(PathBuf::from("/cli/tm.sqlite"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1326,6 +1503,14 @@ fn rejects_proxy_with_unsupported_scheme() {
|
||||
assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_parse_errors_redact_credentials() {
|
||||
let error = parse(&["bat", "--proxy", "http://user:secret@"]).unwrap_err();
|
||||
let message = error.to_string();
|
||||
assert!(!message.contains("secret"));
|
||||
assert!(message.contains("<redacted>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() {
|
||||
let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap();
|
||||
@@ -1997,6 +2182,54 @@ fn daemon_child_args_preserve_sync_options() {
|
||||
assert!(args.contains(&"--no-banner".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn daemon_child_args_preserve_translation_worker_configuration() {
|
||||
let options = parse(&[
|
||||
"bat",
|
||||
"--daemon",
|
||||
"--output",
|
||||
"/tmp/daemon-output",
|
||||
"--translation-provider",
|
||||
"mock",
|
||||
"--translation-fixture",
|
||||
"/tmp/provider.json",
|
||||
"--translation-memory-path",
|
||||
"/tmp/tm.sqlite",
|
||||
"--worker-concurrency",
|
||||
"4",
|
||||
"--worker-max-attempts",
|
||||
"5",
|
||||
"--worker-lease-seconds",
|
||||
"60",
|
||||
"--worker-retry-backoff-seconds",
|
||||
"2",
|
||||
"--worker-max-tasks",
|
||||
"3",
|
||||
"--worker-id",
|
||||
"daemon-worker",
|
||||
])
|
||||
.unwrap();
|
||||
let args = daemon_child_args(&options);
|
||||
let mut child_args = vec!["bat".to_string()];
|
||||
child_args.extend(args);
|
||||
let child = parse_args_from(child_args).unwrap();
|
||||
assert_eq!(child.translation_provider.as_deref(), Some("mock"));
|
||||
assert_eq!(
|
||||
child.translation_fixture,
|
||||
Some(PathBuf::from("/tmp/provider.json"))
|
||||
);
|
||||
assert_eq!(
|
||||
child.translation_memory_path,
|
||||
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||
);
|
||||
assert_eq!(child.worker_concurrency, 4);
|
||||
assert_eq!(child.worker_max_attempts, 5);
|
||||
assert_eq!(child.worker_lease_seconds, 60);
|
||||
assert_eq!(child.worker_retry_backoff, Duration::from_secs(2));
|
||||
assert_eq!(child.worker_max_tasks, Some(3));
|
||||
assert_eq!(child.worker_id.as_deref(), Some("daemon-worker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn curl_proxy_url_extracts_only_url_mode() {
|
||||
assert_eq!(
|
||||
@@ -2184,6 +2417,7 @@ fn test_task_context_with_config(base_config: OfficialUpdateConfig) -> DaemonTas
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config,
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
}
|
||||
@@ -2354,6 +2588,7 @@ fn dispatch_daemon_doctor_returns_report() {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config,
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
@@ -2374,6 +2609,112 @@ fn dispatch_daemon_doctor_returns_report() {
|
||||
assert!(checks.iter().any(|check| check["name"] == "daemon_rpc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dispatch_translation_memory_summary_reports_missing_database_without_creating_it() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let output_root = temp.path().join("output");
|
||||
let state_dir = temp.path().join("state");
|
||||
let (queue, _rx) = mpsc::channel::<TaskJob>();
|
||||
let context = DaemonTaskContext {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config: OfficialUpdateConfig {
|
||||
output_root: output_root.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
|
||||
let envelope = dispatch_rpc_method(
|
||||
&rpc_request("translation.memory.summary", None),
|
||||
&state_dir,
|
||||
&new_daemon_control(),
|
||||
&context,
|
||||
"req-tm-summary-1".to_string(),
|
||||
);
|
||||
let value = serde_json::to_value(envelope).unwrap();
|
||||
assert_eq!(value["ok"], true);
|
||||
assert_eq!(value["data"]["available"], false);
|
||||
assert_eq!(value["data"]["reason"], "database_missing");
|
||||
assert!(!output_root.join("translation-memory.sqlite").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dispatch_translation_memory_summary_defaults_to_worker_config_path() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let output_root = temp.path().join("output");
|
||||
let state_dir = temp.path().join("state");
|
||||
let configured_tm_path = temp.path().join("configured-tm.sqlite");
|
||||
let (queue, _rx) = mpsc::channel::<TaskJob>();
|
||||
let context = DaemonTaskContext {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config: OfficialUpdateConfig {
|
||||
output_root,
|
||||
..Default::default()
|
||||
},
|
||||
translation_worker_config: TranslationWorkerConfig {
|
||||
translation_memory_path: Some(configured_tm_path.clone()),
|
||||
..TranslationWorkerConfig::default()
|
||||
},
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
|
||||
let envelope = dispatch_rpc_method(
|
||||
&rpc_request("translation.memory.summary", None),
|
||||
&state_dir,
|
||||
&new_daemon_control(),
|
||||
&context,
|
||||
"req-tm-summary-configured-1".to_string(),
|
||||
);
|
||||
let value = serde_json::to_value(envelope).unwrap();
|
||||
assert_eq!(value["ok"], true);
|
||||
assert_eq!(value["data"]["available"], false);
|
||||
assert_eq!(
|
||||
value["data"]["path"].as_str(),
|
||||
Some(configured_tm_path.to_str().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dispatch_translation_memory_rejects_invalid_params_with_stable_error_code() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let context = test_task_context_with_config(OfficialUpdateConfig {
|
||||
output_root: temp.path().join("output"),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
for (method, params) in [
|
||||
("translation.memory.query", None),
|
||||
(
|
||||
"translation.memory.query",
|
||||
Some(serde_json::json!({ "source_text": "Hello", "limit": "1" })),
|
||||
),
|
||||
(
|
||||
"translation.memory.summary",
|
||||
Some(serde_json::json!({ "translation_memory_path": 42 })),
|
||||
),
|
||||
(
|
||||
"translation.memory.confirm",
|
||||
Some(serde_json::json!({ "record_id": "tm-record", "reviewer": 42 })),
|
||||
),
|
||||
] {
|
||||
let envelope = dispatch_rpc_method(
|
||||
&rpc_request(method, params),
|
||||
temp.path(),
|
||||
&new_daemon_control(),
|
||||
&context,
|
||||
format!("req-invalid-{method}"),
|
||||
);
|
||||
let value = serde_json::to_value(envelope).unwrap();
|
||||
assert_eq!(value["ok"], false, "method={method}");
|
||||
assert_eq!(value["error"]["code"], "BAT-ERR-700002", "method={method}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dispatch_resource_sync_enqueues_task() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
@@ -2384,6 +2725,7 @@ fn dispatch_resource_sync_enqueues_task() {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config: OfficialUpdateConfig::default(),
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
@@ -2445,6 +2787,7 @@ fn dispatch_resource_repair_enqueues_repair_task() {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config,
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
@@ -3737,6 +4080,7 @@ fn dispatch_catalog_refresh_enqueues_task() {
|
||||
registry: TaskRegistry::new(),
|
||||
queue,
|
||||
base_config: OfficialUpdateConfig::default(),
|
||||
translation_worker_config: TranslationWorkerConfig::default(),
|
||||
sync_lock: Arc::new(Mutex::new(())),
|
||||
restart_controller: test_restart_controller,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -111,6 +111,13 @@ impl HumanReport for bat_infrastructure::TranslationWorkerReport {
|
||||
print_field("失败任务", self.failed_count);
|
||||
print_field("已安排重试", self.retry_scheduled_count);
|
||||
print_field("剩余任务", self.remaining_count);
|
||||
print_path_field("Translation Memory", &self.translation_memory_path);
|
||||
print_field("TM 可用", format_bool(self.translation_memory_available));
|
||||
print_field("TM 命中 TextUnit", self.translation_memory_hit_count);
|
||||
print_field("Provider TextUnit", self.provider_unit_count);
|
||||
for failure in &self.translation_memory_failures {
|
||||
println!(" - TM: {failure}");
|
||||
}
|
||||
for failure in &self.failures {
|
||||
println!(
|
||||
" - {} [{}] retryable={} {}",
|
||||
|
||||
@@ -519,6 +519,8 @@ pub(super) struct DaemonTaskContext {
|
||||
pub(super) registry: TaskRegistry,
|
||||
pub(super) queue: mpsc::Sender<TaskJob>,
|
||||
pub(super) base_config: OfficialUpdateConfig,
|
||||
/// daemon 中未显式传入参数的 translation worker 默认配置。
|
||||
pub(super) translation_worker_config: TranslationWorkerConfig,
|
||||
/// 串行化会读取或修改已发布资源状态的 daemon 操作。
|
||||
pub(super) sync_lock: Arc<Mutex<()>>,
|
||||
pub(super) restart_controller: DaemonRestartController,
|
||||
@@ -562,11 +564,15 @@ pub(super) fn run_task_worker(
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let cancel_check = Arc::clone(&cancel);
|
||||
runtime
|
||||
.block_on(bat_infrastructure::run_translation_worker_at(
|
||||
&resource_root,
|
||||
worker_config,
|
||||
))
|
||||
.block_on(
|
||||
bat_infrastructure::run_translation_worker_at_with_cancellation(
|
||||
&resource_root,
|
||||
worker_config,
|
||||
Arc::new(move || cancel_check.load(Ordering::Relaxed)),
|
||||
),
|
||||
)
|
||||
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
|
||||
})
|
||||
} else {
|
||||
@@ -608,7 +614,8 @@ pub(super) fn run_task_worker(
|
||||
record.result = Some(report);
|
||||
}),
|
||||
Err(error) => {
|
||||
let cancelled = cancel.load(Ordering::Relaxed);
|
||||
let cancelled =
|
||||
cancel.load(Ordering::Relaxed) || daemon_control_stop_requested(Some(&control));
|
||||
// 下载失败携带类型化 DownloadError(含准确网络域码);其余归 internal。
|
||||
let code = error
|
||||
.downcast_ref::<bat_infrastructure::DownloadError>()
|
||||
|
||||
@@ -71,28 +71,24 @@ pub(super) fn print_startup_banner() {
|
||||
eprintln!("{STARTUP_BANNER}");
|
||||
}
|
||||
|
||||
pub(super) fn print_env_template_created(path: &Path) {
|
||||
pub(super) fn print_config_template_created(path: &Path) {
|
||||
eprintln!(
|
||||
"已生成配置模板 {}(编辑其中的 BAT_* 配置后,直接运行 `bat` 即可按 .env 启动)",
|
||||
"已生成配置模板 {}(编辑 `config.toml`,`config.toml.example` 不会被程序自动读取)",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn print_env_template_warning(path: &Path, error: impl std::fmt::Display) {
|
||||
eprintln!("警告:生成 .env 配置模板失败 {}:{error}", path.display());
|
||||
}
|
||||
|
||||
pub(super) fn print_env_read_warning(path: &Path, error: impl std::fmt::Display) {
|
||||
eprintln!("警告:读取 .env 失败 {}:{error}", path.display());
|
||||
}
|
||||
|
||||
pub(super) fn print_env_parse_warning(line_number: usize, raw_line: &str) {
|
||||
pub(super) fn print_config_template_warning(path: &Path, error: impl std::fmt::Display) {
|
||||
eprintln!(
|
||||
"警告:.env 第 {} 行无法解析,已忽略:{raw_line}",
|
||||
line_number
|
||||
"警告:生成 config.toml.example 模板失败 {}:{error}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn print_deprecated_env_file_warning() {
|
||||
eprintln!("警告:BAT_SKIP_ENV_FILE 已废弃且不再影响启动,已忽略");
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct ProgressLogger {
|
||||
enabled: bool,
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
use super::report_output::print_json_value;
|
||||
use super::*;
|
||||
use bat_core::domain::TranslationMemoryContext;
|
||||
use bat_core::repositories::TranslationMemoryRepository;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TranslationTaskResultUpdateParam {
|
||||
@@ -332,6 +336,8 @@ fn build_manual_translation_results(
|
||||
unit_id: unit_id.to_string(),
|
||||
source_text: param.source_text.clone(),
|
||||
translated_text: param.translated_text.clone(),
|
||||
source_kind: bat_infrastructure::TranslationTaskResultSourceKind::Manual,
|
||||
translation_memory_record_id: None,
|
||||
provider: provider.to_string(),
|
||||
provider_run_id: provider_run_id.to_string(),
|
||||
translated_unix_seconds,
|
||||
@@ -355,3 +361,241 @@ pub(super) fn translation_task_query_json(query: &OfficialTextUnitTaskQuery) ->
|
||||
"has_failure_reason": query.has_failure_reason,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn run_translation_memory_command(options: &CliOptions) -> anyhow::Result<()> {
|
||||
let method = match options.command {
|
||||
CliCommand::TranslationMemorySummary => RPC_METHOD_TRANSLATION_MEMORY_SUMMARY,
|
||||
CliCommand::TranslationMemoryQuery => RPC_METHOD_TRANSLATION_MEMORY_QUERY,
|
||||
CliCommand::TranslationMemoryConfirm => RPC_METHOD_TRANSLATION_MEMORY_CONFIRM,
|
||||
_ => return Err(anyhow::anyhow!("不是 Translation Memory 命令")),
|
||||
};
|
||||
if daemon_rpc_available(&options.state_dir)
|
||||
&& options.resource_root.is_none()
|
||||
&& !options.output_explicit
|
||||
{
|
||||
let _control_lock = DaemonControlLock::acquire(&options.state_dir)?;
|
||||
let report = daemon_rpc_call(
|
||||
&options.state_dir,
|
||||
method,
|
||||
translation_memory_cli_params(options)?,
|
||||
)?;
|
||||
print_json_value(options.output_format, &report)?;
|
||||
return Ok(());
|
||||
}
|
||||
let path = translation_memory_cli_path(options)?;
|
||||
let report = match options.command {
|
||||
CliCommand::TranslationMemorySummary => build_translation_memory_summary_report(&path)?,
|
||||
CliCommand::TranslationMemoryQuery => {
|
||||
let source_text = options
|
||||
.translation_memory_source_text
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
|
||||
let context = parse_translation_memory_context(
|
||||
options.translation_memory_context_json.as_deref(),
|
||||
)?;
|
||||
build_translation_memory_query_report(
|
||||
&path,
|
||||
source_text,
|
||||
&context,
|
||||
options.query_limit,
|
||||
)?
|
||||
}
|
||||
CliCommand::TranslationMemoryConfirm => {
|
||||
let record_id = options
|
||||
.translation_memory_record_id
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
|
||||
let reviewer = options
|
||||
.translation_memory_reviewer
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
|
||||
build_translation_memory_confirm_report(
|
||||
&path,
|
||||
record_id,
|
||||
reviewer,
|
||||
options.translation_memory_reason.clone(),
|
||||
)?
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
print_json_value(options.output_format, &report)
|
||||
}
|
||||
|
||||
fn translation_memory_cli_path(options: &CliOptions) -> anyhow::Result<std::path::PathBuf> {
|
||||
if let Some(path) = options.translation_memory_path.as_ref() {
|
||||
return lexical_absolute(path).map_err(anyhow::Error::msg);
|
||||
}
|
||||
let resource_root = options
|
||||
.resource_root
|
||||
.as_deref()
|
||||
.map(lexical_absolute)
|
||||
.transpose()
|
||||
.map_err(anyhow::Error::msg)?
|
||||
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
||||
Ok(bat_infrastructure::translation_memory_repository_path(
|
||||
&resource_root,
|
||||
))
|
||||
}
|
||||
|
||||
fn translation_memory_cli_params(
|
||||
options: &CliOptions,
|
||||
) -> anyhow::Result<Option<serde_json::Value>> {
|
||||
let mut params = serde_json::Map::new();
|
||||
if let Some(path) = options.translation_memory_path.as_ref() {
|
||||
params.insert(
|
||||
"translation_memory_path".to_string(),
|
||||
serde_json::json!(path),
|
||||
);
|
||||
}
|
||||
match options.command {
|
||||
CliCommand::TranslationMemorySummary => {}
|
||||
CliCommand::TranslationMemoryQuery => {
|
||||
let source_text = options
|
||||
.translation_memory_source_text
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
|
||||
let context = parse_translation_memory_context(
|
||||
options.translation_memory_context_json.as_deref(),
|
||||
)?;
|
||||
params.insert("source_text".to_string(), serde_json::json!(source_text));
|
||||
params.insert("source_context".to_string(), serde_json::json!(context));
|
||||
params.insert("limit".to_string(), serde_json::json!(options.query_limit));
|
||||
}
|
||||
CliCommand::TranslationMemoryConfirm => {
|
||||
let record_id = options
|
||||
.translation_memory_record_id
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
|
||||
let reviewer = options
|
||||
.translation_memory_reviewer
|
||||
.as_deref()
|
||||
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
|
||||
params.insert("record_id".to_string(), serde_json::json!(record_id));
|
||||
params.insert("reviewer".to_string(), serde_json::json!(reviewer));
|
||||
if let Some(reason) = options.translation_memory_reason.as_deref() {
|
||||
params.insert("reason".to_string(), serde_json::json!(reason));
|
||||
}
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
Ok(Some(serde_json::Value::Object(params)))
|
||||
}
|
||||
|
||||
pub(super) fn build_translation_memory_summary_report(
|
||||
path: &std::path::Path,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||
return Ok(serde_json::json!({
|
||||
"available": false,
|
||||
"path": path,
|
||||
"reason": "database_missing",
|
||||
}));
|
||||
}
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let summary = runtime.block_on(async {
|
||||
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
repository
|
||||
.summary()
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||
})?;
|
||||
Ok(serde_json::json!({
|
||||
"available": true,
|
||||
"path": path,
|
||||
"schema_version": summary.schema_version,
|
||||
"summary": summary,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn build_translation_memory_query_report(
|
||||
path: &std::path::Path,
|
||||
source_text: &str,
|
||||
source_context: &TranslationMemoryContext,
|
||||
limit: usize,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
if source_text.trim().is_empty() {
|
||||
return Err(anyhow::anyhow!("TM query 的 source_text 不能为空"));
|
||||
}
|
||||
if !(1..=1000).contains(&limit) {
|
||||
return Err(anyhow::anyhow!("TM query 的 limit 必须在 1..=1000 范围内"));
|
||||
}
|
||||
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||
return Ok(serde_json::json!({
|
||||
"available": false,
|
||||
"path": path,
|
||||
"source_text": source_text,
|
||||
"source_context": source_context,
|
||||
"matches": [],
|
||||
"reason": "database_missing",
|
||||
}));
|
||||
}
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let matches = runtime.block_on(async {
|
||||
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
repository
|
||||
.find_matches(source_text, source_context, limit)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||
})?;
|
||||
Ok(serde_json::json!({
|
||||
"available": true,
|
||||
"path": path,
|
||||
"source_text": source_text,
|
||||
"source_context": source_context,
|
||||
"matches": matches,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn build_translation_memory_confirm_report(
|
||||
path: &std::path::Path,
|
||||
record_id: &str,
|
||||
reviewer: &str,
|
||||
reason: Option<String>,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
if record_id.trim().is_empty() || reviewer.trim().is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"TM confirm 必须指定非空 record_id 和 reviewer"
|
||||
));
|
||||
}
|
||||
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Translation Memory 数据库不存在:{}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let entry = runtime.block_on(async {
|
||||
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
repository
|
||||
.confirm(record_id, reviewer, reason)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||
})?;
|
||||
Ok(serde_json::json!({
|
||||
"available": true,
|
||||
"path": path,
|
||||
"entry": entry,
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_translation_memory_context(
|
||||
value: Option<&str>,
|
||||
) -> anyhow::Result<TranslationMemoryContext> {
|
||||
let Some(value) = value else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
serde_json::from_str::<TranslationMemoryContext>(value)
|
||||
.map_err(|error| anyhow::anyhow!("--tm-context-json 必须是 JSON object:{error}"))
|
||||
}
|
||||
|
||||
@@ -261,26 +261,10 @@ pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()>
|
||||
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
|
||||
.transpose()?
|
||||
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
||||
let provider = options
|
||||
.translation_provider
|
||||
.as_deref()
|
||||
.unwrap_or(TranslationProviderKind::Mock.as_str());
|
||||
let provider = TranslationProviderKind::parse(provider)
|
||||
.ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效:{provider}"))?;
|
||||
let config = TranslationWorkerConfig {
|
||||
provider,
|
||||
fixture_path: options.translation_fixture.clone(),
|
||||
concurrency: options.worker_concurrency,
|
||||
max_attempts: options.worker_max_attempts,
|
||||
lease_seconds: options.worker_lease_seconds,
|
||||
retry_backoff: options.worker_retry_backoff,
|
||||
max_tasks: options.worker_max_tasks,
|
||||
worker_id: options
|
||||
.worker_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("bat-worker-{}", std::process::id())),
|
||||
};
|
||||
config.validate()?;
|
||||
let config = super::translation_worker_config_from_options(
|
||||
options,
|
||||
&format!("bat-worker-{}", std::process::id()),
|
||||
)?;
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
+18
-12
@@ -29,6 +29,7 @@ pub mod patch_ops;
|
||||
pub mod path_security;
|
||||
pub mod release_flow;
|
||||
pub mod resources;
|
||||
pub mod translation_memory;
|
||||
pub mod translation_tasks;
|
||||
pub mod translation_worker;
|
||||
pub mod translation_workflow;
|
||||
@@ -140,24 +141,29 @@ pub use path_security::{
|
||||
};
|
||||
pub use release_flow::ReleaseFlowStatusCode;
|
||||
pub use resources::{InMemoryResourceRepository, SqliteResourceRepository};
|
||||
pub use translation_memory::{
|
||||
translation_memory_context, translation_memory_repository_path,
|
||||
SqliteTranslationMemoryRepository, TRANSLATION_MEMORY_REPOSITORY_FILE,
|
||||
TRANSLATION_MEMORY_SCHEMA_COMPONENT, TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||||
};
|
||||
pub use translation_tasks::{
|
||||
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
|
||||
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
|
||||
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
|
||||
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus,
|
||||
TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus,
|
||||
TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
|
||||
TRANSLATION_TASK_SCHEMA_VERSION,
|
||||
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskResultSourceKind,
|
||||
TranslationTaskStatus, TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit,
|
||||
TranslationUnitStatus, TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION,
|
||||
TRANSLATION_TASK_REPOSITORY_FILE, TRANSLATION_TASK_SCHEMA_VERSION,
|
||||
};
|
||||
pub use translation_worker::{
|
||||
run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider,
|
||||
MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass,
|
||||
TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse,
|
||||
TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig,
|
||||
TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY,
|
||||
DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS,
|
||||
DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY,
|
||||
MOCK_TRANSLATION_FIXTURE_VERSION,
|
||||
run_translation_worker_at, run_translation_worker_at_with_cancellation,
|
||||
run_translation_worker_with_provider, CrowdinProvider, MockTranslationProvider,
|
||||
TranslationProvider, TranslationProviderFailureClass, TranslationProviderKind,
|
||||
TranslationProviderRequest, TranslationProviderResponse, TranslationProviderUnit,
|
||||
TranslationProviderUnitResult, TranslationWorkerConfig, TranslationWorkerFailure,
|
||||
TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS,
|
||||
DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
||||
MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, MOCK_TRANSLATION_FIXTURE_VERSION,
|
||||
};
|
||||
pub use translation_workflow::{
|
||||
completed_worker_translation_workbench, export_completed_worker_translation_workbench,
|
||||
|
||||
@@ -91,6 +91,12 @@ pub struct LocalizedPatchOperationMetadata {
|
||||
/// Provider run ID that produced the text, if applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// Source kind of the translation result.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_source_kind: Option<String>,
|
||||
/// Trusted Translation Memory record used for the text, if applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_memory_record_id: Option<String>,
|
||||
/// Review state used by the publication input.
|
||||
pub review_status: String,
|
||||
}
|
||||
@@ -331,6 +337,12 @@ pub struct LocalizedPatchOperation {
|
||||
/// Provider run ID that produced the text, if applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// Source kind of the translation result.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_source_kind: Option<String>,
|
||||
/// Trusted Translation Memory record used for the text, if applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_memory_record_id: Option<String>,
|
||||
/// Review state used by the publication input.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub review_status: Option<String>,
|
||||
@@ -941,6 +953,8 @@ impl LocalizedPatchOperation {
|
||||
source_text_blake3: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
review_status: None,
|
||||
},
|
||||
metadata,
|
||||
@@ -966,6 +980,8 @@ impl LocalizedPatchOperation {
|
||||
source_text_blake3: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
review_status: None,
|
||||
},
|
||||
metadata,
|
||||
@@ -990,6 +1006,8 @@ impl LocalizedPatchOperation {
|
||||
source_text_blake3: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
review_status: None,
|
||||
},
|
||||
metadata,
|
||||
@@ -1005,6 +1023,8 @@ impl LocalizedPatchOperation {
|
||||
operation.source_text_blake3 = Some(metadata.source_text_blake3.clone());
|
||||
operation.translation_provider = metadata.translation_provider.clone();
|
||||
operation.provider_run_id = metadata.provider_run_id.clone();
|
||||
operation.translation_source_kind = metadata.translation_source_kind.clone();
|
||||
operation.translation_memory_record_id = metadata.translation_memory_record_id.clone();
|
||||
operation.review_status = Some(metadata.review_status.clone());
|
||||
}
|
||||
operation
|
||||
@@ -1029,6 +1049,8 @@ impl Default for LocalizedPatchOperation {
|
||||
source_text_blake3: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
review_status: None,
|
||||
}
|
||||
}
|
||||
@@ -1401,6 +1423,8 @@ mod tests {
|
||||
source_text_blake3: Some(blake3::hash(source).to_hex().to_string()),
|
||||
translation_provider: Some("mock".to_string()),
|
||||
provider_run_id: Some("mock:unit-1:attempt-1".to_string()),
|
||||
translation_source_kind: Some("provider".to_string()),
|
||||
translation_memory_record_id: None,
|
||||
review_status: Some("provider_completed".to_string()),
|
||||
}],
|
||||
}],
|
||||
|
||||
@@ -0,0 +1,859 @@
|
||||
//! 跨 official release 的 Translation Memory SQLite 仓储。
|
||||
|
||||
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
|
||||
use async_trait::async_trait;
|
||||
use bat_core::domain::{
|
||||
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||||
TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||||
};
|
||||
use bat_core::repositories::TranslationMemoryRepository;
|
||||
use bat_core::{Error, Result};
|
||||
use serde::de::DeserializeOwned;
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
|
||||
use sqlx::{Row, SqlitePool};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// TM SQLite schema 版本。
|
||||
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1;
|
||||
/// TM schema migration component。
|
||||
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
|
||||
/// 默认 TM 数据库文件名。
|
||||
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
|
||||
|
||||
/// SQLite-backed Translation Memory 仓储。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SqliteTranslationMemoryRepository {
|
||||
pub(crate) pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteTranslationMemoryRepository {
|
||||
/// 创建或打开 TM 数据库并执行迁移。
|
||||
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
|
||||
Self::open_with(path.as_ref(), true).await
|
||||
}
|
||||
|
||||
/// 只打开已有 TM 数据库,不创建新文件。
|
||||
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||
Self::open_with(path.as_ref(), false).await
|
||||
}
|
||||
|
||||
/// 根据 active release 根目录计算默认的跨 release TM 路径。
|
||||
///
|
||||
/// 正式 release 根目录形如 `<output>/versions/<id>`,因此默认结果为
|
||||
/// `<output>/translation-memory.sqlite`,不会写入已发布版本目录。
|
||||
pub fn repository_path(resource_root: &Path) -> PathBuf {
|
||||
if resource_root
|
||||
.parent()
|
||||
.and_then(Path::file_name)
|
||||
.is_some_and(|name| name == "versions")
|
||||
{
|
||||
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
|
||||
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
|
||||
}
|
||||
}
|
||||
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
|
||||
}
|
||||
|
||||
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
|
||||
let absolute = bat_infrastructure_absolute(path)?;
|
||||
let parent = absolute.parent().ok_or_else(|| {
|
||||
Error::InvalidArgument(format!("TM 数据库缺少父目录:{}", absolute.display()))
|
||||
})?;
|
||||
ensure_safe_tm_parent(parent)?;
|
||||
if create_if_missing {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
ensure_safe_tm_parent(parent)?;
|
||||
}
|
||||
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"TM 数据库不能是 symlink:{}",
|
||||
absolute.display()
|
||||
)));
|
||||
}
|
||||
if !metadata.is_file() {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"TM 数据库不是普通文件:{}",
|
||||
absolute.display()
|
||||
)));
|
||||
}
|
||||
} else if !create_if_missing {
|
||||
return Err(Error::NotFound(absolute.display().to_string()));
|
||||
}
|
||||
|
||||
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
|
||||
.map_err(|error| Error::Other(error.into()))?
|
||||
.create_if_missing(create_if_missing)
|
||||
.journal_mode(SqliteJournalMode::Wal)
|
||||
.busy_timeout(Duration::from_secs(30));
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
|
||||
.map_err(Error::InvalidArgument)?;
|
||||
let repository = Self { pool };
|
||||
repository.init_schema().await?;
|
||||
Ok(repository)
|
||||
}
|
||||
|
||||
async fn init_schema(&self) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
component TEXT PRIMARY KEY NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version >= 1)
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS translation_memory (
|
||||
record_id TEXT PRIMARY KEY NOT NULL,
|
||||
source_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
normalized_source_text TEXT NOT NULL,
|
||||
source_context_json TEXT NOT NULL,
|
||||
source_context_hash TEXT NOT NULL,
|
||||
translated_text TEXT NOT NULL,
|
||||
translation_source_kind TEXT NOT NULL,
|
||||
trust_status TEXT NOT NULL,
|
||||
official_release_id TEXT NOT NULL,
|
||||
source_trace_json TEXT NOT NULL,
|
||||
provider TEXT,
|
||||
provider_run_id TEXT,
|
||||
created_unix_seconds INTEGER NOT NULL,
|
||||
updated_unix_seconds INTEGER NOT NULL,
|
||||
trusted_unix_seconds INTEGER,
|
||||
trusted_by TEXT,
|
||||
trusted_reason TEXT,
|
||||
supersedes_record_id TEXT,
|
||||
superseded_by_record_id TEXT,
|
||||
CHECK (length(source_text) > 0),
|
||||
CHECK (length(source_hash) > 0),
|
||||
CHECK (length(source_context_hash) > 0),
|
||||
CHECK (length(official_release_id) > 0),
|
||||
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
|
||||
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
|
||||
ON translation_memory(source_hash)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
|
||||
ON translation_memory(normalized_source_text)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
|
||||
ON translation_memory(source_hash, source_context_hash)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
|
||||
let current: Option<i64> =
|
||||
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"不支持的 Translation Memory schema 版本:{}",
|
||||
current.unwrap_or_default()
|
||||
)));
|
||||
}
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO schema_migrations(component, version)
|
||||
VALUES (?1, ?2)
|
||||
ON CONFLICT(component) DO UPDATE SET version = excluded.version
|
||||
"#,
|
||||
)
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
|
||||
let row = sqlx::query(
|
||||
r#"
|
||||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||
FROM translation_memory
|
||||
WHERE record_id = ?1
|
||||
"#,
|
||||
)
|
||||
.bind(record_id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
row.map(row_to_entry).transpose()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
|
||||
async fn upsert_candidate(
|
||||
&self,
|
||||
draft: TranslationMemoryDraft,
|
||||
) -> Result<TranslationMemoryEntry> {
|
||||
validate_draft(&draft)?;
|
||||
let entry = entry_from_draft(draft)?;
|
||||
if let Some(existing) = self.find_optional(&entry.record_id).await? {
|
||||
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||
return Ok(existing);
|
||||
}
|
||||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_memory
|
||||
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
|
||||
updated_unix_seconds = ?5
|
||||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(&entry.record_id)
|
||||
.bind(source_trace_json)
|
||||
.bind(&entry.provider)
|
||||
.bind(&entry.provider_run_id)
|
||||
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
return self.find(&entry.record_id).await;
|
||||
}
|
||||
|
||||
let source_context_json = serde_json::to_string(&entry.source_context)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO translation_memory (
|
||||
record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds,
|
||||
trusted_unix_seconds, trusted_by, trusted_reason,
|
||||
supersedes_record_id, superseded_by_record_id
|
||||
)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
|
||||
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
|
||||
ON CONFLICT(record_id) DO UPDATE SET
|
||||
source_trace_json = excluded.source_trace_json,
|
||||
provider = excluded.provider,
|
||||
provider_run_id = excluded.provider_run_id,
|
||||
updated_unix_seconds = excluded.updated_unix_seconds
|
||||
WHERE translation_memory.trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(&entry.record_id)
|
||||
.bind(&entry.source_text)
|
||||
.bind(&entry.source_hash)
|
||||
.bind(&entry.normalized_source_text)
|
||||
.bind(source_context_json)
|
||||
.bind(&entry.source_context_hash)
|
||||
.bind(&entry.translated_text)
|
||||
.bind(entry.translation_source_kind.as_str())
|
||||
.bind(entry.trust_status.as_str())
|
||||
.bind(&entry.official_release_id)
|
||||
.bind(source_trace_json)
|
||||
.bind(&entry.provider)
|
||||
.bind(&entry.provider_run_id)
|
||||
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
|
||||
.bind(
|
||||
entry
|
||||
.trusted_unix_seconds
|
||||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
|
||||
)
|
||||
.bind(&entry.trusted_by)
|
||||
.bind(&entry.trusted_reason)
|
||||
.bind(&entry.supersedes_record_id)
|
||||
.bind(&entry.superseded_by_record_id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if result.rows_affected() == 0 {
|
||||
return self.find(&entry.record_id).await;
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
async fn find_matches(
|
||||
&self,
|
||||
source_text: &str,
|
||||
source_context: &TranslationMemoryContext,
|
||||
limit: usize,
|
||||
) -> Result<Vec<TranslationMemoryMatch>> {
|
||||
if source_text.is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory 查询 source_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if limit == 0 {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory 查询 limit 必须大于 0".to_string(),
|
||||
));
|
||||
}
|
||||
let source_hash = hash_text(source_text);
|
||||
let normalized_source_text = normalize_source_text(source_text);
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||
FROM translation_memory
|
||||
WHERE source_hash = ?1 OR normalized_source_text = ?2
|
||||
ORDER BY updated_unix_seconds DESC, record_id ASC
|
||||
"#,
|
||||
)
|
||||
.bind(source_hash)
|
||||
.bind(&normalized_source_text)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
let mut matches = rows
|
||||
.into_iter()
|
||||
.map(row_to_entry)
|
||||
.collect::<Result<Vec<_>>>()?
|
||||
.into_iter()
|
||||
.filter_map(|entry| {
|
||||
let raw_exact = entry.source_text == source_text;
|
||||
let normalized_exact = entry.normalized_source_text == normalized_source_text;
|
||||
if !raw_exact && !normalized_exact {
|
||||
return None;
|
||||
}
|
||||
let same_context = !source_context.is_empty()
|
||||
&& !entry.source_context.is_empty()
|
||||
&& entry.source_context == *source_context;
|
||||
let strong = raw_exact
|
||||
&& same_context
|
||||
&& entry.trust_status == TranslationMemoryTrustStatus::Trusted;
|
||||
let match_kind = if strong {
|
||||
TranslationMemoryMatchKind::StrongExact
|
||||
} else if raw_exact {
|
||||
TranslationMemoryMatchKind::CandidateExact
|
||||
} else {
|
||||
TranslationMemoryMatchKind::SourceOnly
|
||||
};
|
||||
Some(TranslationMemoryMatch {
|
||||
can_auto_reuse: strong,
|
||||
entry,
|
||||
match_kind,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
matches.sort_by(|left, right| {
|
||||
match_rank(left)
|
||||
.cmp(&match_rank(right))
|
||||
.then_with(|| {
|
||||
right
|
||||
.entry
|
||||
.updated_unix_seconds
|
||||
.cmp(&left.entry.updated_unix_seconds)
|
||||
})
|
||||
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
|
||||
});
|
||||
matches.truncate(limit);
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
async fn confirm(
|
||||
&self,
|
||||
record_id: &str,
|
||||
reviewer: &str,
|
||||
reason: Option<String>,
|
||||
) -> Result<TranslationMemoryEntry> {
|
||||
if reviewer.trim().is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory reviewer 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let current = self.find(record_id).await?;
|
||||
if current.trust_status == TranslationMemoryTrustStatus::Trusted {
|
||||
return Ok(current);
|
||||
}
|
||||
if current.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"Translation Memory 记录 {} 当前状态为 {},不能确认",
|
||||
record_id,
|
||||
current.trust_status.as_str()
|
||||
)));
|
||||
}
|
||||
let now = unix_seconds_now();
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_memory
|
||||
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
|
||||
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
|
||||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(record_id)
|
||||
.bind(i64::try_from(now).unwrap_or(i64::MAX))
|
||||
.bind(reviewer.trim())
|
||||
.bind(reason.filter(|value| !value.trim().is_empty()))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
self.find(record_id).await
|
||||
}
|
||||
|
||||
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
|
||||
self.find_optional(record_id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::NotFound(record_id.to_string()))
|
||||
}
|
||||
|
||||
async fn summary(&self) -> Result<TranslationMemorySummary> {
|
||||
let row = sqlx::query(
|
||||
r#"
|
||||
SELECT COUNT(*) AS record_count,
|
||||
SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count,
|
||||
SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count,
|
||||
SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count,
|
||||
SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count
|
||||
FROM translation_memory
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
Ok(TranslationMemorySummary {
|
||||
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||||
record_count: row.try_get::<i64, _>("record_count").map_err(db_error)? as u64,
|
||||
trusted_count: row.try_get::<i64, _>("trusted_count").map_err(db_error)? as u64,
|
||||
candidate_count: row.try_get::<i64, _>("candidate_count").map_err(db_error)? as u64,
|
||||
superseded_count: row
|
||||
.try_get::<i64, _>("superseded_count")
|
||||
.map_err(db_error)? as u64,
|
||||
rejected_count: row.try_get::<i64, _>("rejected_count").map_err(db_error)? as u64,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 由 active official release 根目录计算默认 TM 数据库路径。
|
||||
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
|
||||
SqliteTranslationMemoryRepository::repository_path(resource_root)
|
||||
}
|
||||
|
||||
/// 从 TextUnit 定位字段构建 TM 上下文。
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn translation_memory_context(
|
||||
destination: &str,
|
||||
archive_entry: Option<&str>,
|
||||
serialized_file: Option<&str>,
|
||||
path_id: Option<i64>,
|
||||
class_id: Option<i32>,
|
||||
field_path: Option<&str>,
|
||||
format: Option<&str>,
|
||||
asset_name: Option<&str>,
|
||||
text_source_kind: Option<&str>,
|
||||
parser_context: &TranslationMemoryContext,
|
||||
) -> TranslationMemoryContext {
|
||||
let mut context = TranslationMemoryContext::new();
|
||||
context.insert("destination".to_string(), destination.to_string());
|
||||
insert_optional(&mut context, "archive_entry", archive_entry);
|
||||
insert_optional(&mut context, "serialized_file", serialized_file);
|
||||
if let Some(value) = path_id {
|
||||
context.insert("path_id".to_string(), value.to_string());
|
||||
}
|
||||
if let Some(value) = class_id {
|
||||
context.insert("class_id".to_string(), value.to_string());
|
||||
}
|
||||
insert_optional(&mut context, "field_path", field_path);
|
||||
insert_optional(&mut context, "format", format);
|
||||
insert_optional(&mut context, "asset_name", asset_name);
|
||||
insert_optional(&mut context, "text_source_kind", text_source_kind);
|
||||
for (key, value) in parser_context {
|
||||
context.insert(format!("context.{key}"), value.clone());
|
||||
}
|
||||
context
|
||||
}
|
||||
|
||||
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
|
||||
if let Some(value) = value.filter(|value| !value.is_empty()) {
|
||||
context.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
|
||||
let source_hash = hash_text(&draft.source_text);
|
||||
let normalized_source_text = normalize_source_text(&draft.source_text);
|
||||
let source_context_hash = hash_context(&draft.source_context)?;
|
||||
let source_kind = draft.translation_source_kind;
|
||||
let record_id = record_id(
|
||||
&source_hash,
|
||||
&source_context_hash,
|
||||
&draft.translated_text,
|
||||
&draft.official_release_id,
|
||||
&source_kind,
|
||||
);
|
||||
let observed = draft.observed_unix_seconds;
|
||||
Ok(TranslationMemoryEntry {
|
||||
record_id,
|
||||
source_text: draft.source_text,
|
||||
source_hash,
|
||||
normalized_source_text,
|
||||
source_context: draft.source_context,
|
||||
source_context_hash,
|
||||
translated_text: draft.translated_text,
|
||||
translation_source_kind: source_kind,
|
||||
trust_status: TranslationMemoryTrustStatus::Candidate,
|
||||
official_release_id: draft.official_release_id,
|
||||
source_trace: draft.source_trace,
|
||||
provider: draft.provider,
|
||||
provider_run_id: draft.provider_run_id,
|
||||
created_unix_seconds: observed,
|
||||
updated_unix_seconds: observed,
|
||||
trusted_unix_seconds: None,
|
||||
trusted_by: None,
|
||||
trusted_reason: None,
|
||||
supersedes_record_id: None,
|
||||
superseded_by_record_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
|
||||
if draft.source_text.is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory source_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.translated_text.trim().is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory translated_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.official_release_id.trim().is_empty()
|
||||
|| draft.source_trace.official_release_id.trim().is_empty()
|
||||
{
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory official_release_id 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.official_release_id != draft.source_trace.official_release_id {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory draft 的 release provenance 不一致".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
|
||||
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
|
||||
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
|
||||
Ok(TranslationMemoryEntry {
|
||||
record_id: row.try_get("record_id").map_err(db_error)?,
|
||||
source_text: row.try_get("source_text").map_err(db_error)?,
|
||||
source_hash: row.try_get("source_hash").map_err(db_error)?,
|
||||
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
|
||||
source_context,
|
||||
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
|
||||
translated_text: row.try_get("translated_text").map_err(db_error)?,
|
||||
translation_source_kind: parse_source_kind(
|
||||
row.try_get::<String, _>("translation_source_kind")
|
||||
.map_err(db_error)?
|
||||
.as_str(),
|
||||
)?,
|
||||
trust_status: parse_trust_status(
|
||||
row.try_get::<String, _>("trust_status")
|
||||
.map_err(db_error)?
|
||||
.as_str(),
|
||||
)?,
|
||||
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
|
||||
source_trace,
|
||||
provider: row.try_get("provider").map_err(db_error)?,
|
||||
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
|
||||
created_unix_seconds: i64_to_u64(
|
||||
row.try_get("created_unix_seconds").map_err(db_error)?,
|
||||
"created",
|
||||
)?,
|
||||
updated_unix_seconds: i64_to_u64(
|
||||
row.try_get("updated_unix_seconds").map_err(db_error)?,
|
||||
"updated",
|
||||
)?,
|
||||
trusted_unix_seconds: optional_i64_to_u64(
|
||||
row.try_get("trusted_unix_seconds").map_err(db_error)?,
|
||||
"trusted",
|
||||
)?,
|
||||
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
|
||||
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
|
||||
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
|
||||
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
|
||||
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
|
||||
}
|
||||
|
||||
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
|
||||
match value {
|
||||
"provider" => Ok(TranslationMemorySourceKind::Provider),
|
||||
"manual" => Ok(TranslationMemorySourceKind::Manual),
|
||||
"imported" => Ok(TranslationMemorySourceKind::Imported),
|
||||
_ => Err(Error::Serialization(format!(
|
||||
"未知 Translation Memory source kind:{value}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
|
||||
match value {
|
||||
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
|
||||
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
|
||||
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
|
||||
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
|
||||
_ => Err(Error::Serialization(format!(
|
||||
"未知 Translation Memory trust status:{value}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
|
||||
match value.match_kind {
|
||||
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
|
||||
TranslationMemoryMatchKind::CandidateExact => 1,
|
||||
TranslationMemoryMatchKind::SourceOnly => 2,
|
||||
TranslationMemoryMatchKind::StrongExact => 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_text(value: &str) -> String {
|
||||
blake3::hash(value.as_bytes()).to_hex().to_string()
|
||||
}
|
||||
|
||||
fn normalize_source_text(value: &str) -> String {
|
||||
value.replace("\r\n", "\n").replace('\r', "\n")
|
||||
}
|
||||
|
||||
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
|
||||
let bytes =
|
||||
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
Ok(blake3::hash(&bytes).to_hex().to_string())
|
||||
}
|
||||
|
||||
fn record_id(
|
||||
source_hash: &str,
|
||||
context_hash: &str,
|
||||
translated_text: &str,
|
||||
official_release_id: &str,
|
||||
source_kind: &TranslationMemorySourceKind,
|
||||
) -> String {
|
||||
let mut key = Vec::new();
|
||||
for value in [
|
||||
source_hash,
|
||||
context_hash,
|
||||
translated_text,
|
||||
official_release_id,
|
||||
source_kind.as_str(),
|
||||
] {
|
||||
key.extend_from_slice(value.as_bytes());
|
||||
key.push(0);
|
||||
}
|
||||
format!("tm-{}", blake3::hash(&key).to_hex())
|
||||
}
|
||||
|
||||
fn unix_seconds_now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
|
||||
u64::try_from(value)
|
||||
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
|
||||
}
|
||||
|
||||
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
|
||||
value.map(|value| i64_to_u64(value, label)).transpose()
|
||||
}
|
||||
|
||||
fn db_error(error: sqlx::Error) -> Error {
|
||||
Error::Other(error.into())
|
||||
}
|
||||
|
||||
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
|
||||
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
|
||||
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
|
||||
.map_err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bat_core::domain::TranslationMemorySourceTrace;
|
||||
|
||||
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
|
||||
let source_trace = TranslationMemorySourceTrace {
|
||||
official_release_id: release.to_string(),
|
||||
unit_id: Some(format!("{release}-unit")),
|
||||
task_id: Some(format!("{release}-task")),
|
||||
destination: Some("Bundles/story.bundle".to_string()),
|
||||
archive_entry: None,
|
||||
serialized_file: Some("CAB-story".to_string()),
|
||||
path_id: Some(1),
|
||||
class_id: Some(49),
|
||||
field_path: Some("m_Text".to_string()),
|
||||
format: Some("plain".to_string()),
|
||||
asset_name: Some("Story".to_string()),
|
||||
text_source_kind: Some("text_asset".to_string()),
|
||||
source_url: Some("https://example.invalid/story".to_string()),
|
||||
};
|
||||
TranslationMemoryDraft {
|
||||
source_text: source.to_string(),
|
||||
source_context: translation_memory_context(
|
||||
"Bundles/story.bundle",
|
||||
None,
|
||||
Some("CAB-story"),
|
||||
Some(1),
|
||||
Some(49),
|
||||
Some("m_Text"),
|
||||
Some("plain"),
|
||||
Some("Story"),
|
||||
Some("text_asset"),
|
||||
&TranslationMemoryContext::new(),
|
||||
),
|
||||
translated_text: translated.to_string(),
|
||||
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||||
official_release_id: release.to_string(),
|
||||
source_trace,
|
||||
provider: Some("mock".to_string()),
|
||||
provider_run_id: Some(format!("mock:{release}")),
|
||||
observed_unix_seconds: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(
|
||||
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let entry = repository
|
||||
.upsert_candidate(draft("release-1", "Hello", "你好"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
|
||||
let trusted = repository
|
||||
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
|
||||
|
||||
let query = draft("release-2", "Hello", "ignored");
|
||||
let matches = repository
|
||||
.find_matches("Hello", &query.source_context, 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(matches.len(), 1);
|
||||
assert_eq!(
|
||||
matches[0].match_kind,
|
||||
TranslationMemoryMatchKind::StrongExact
|
||||
);
|
||||
assert!(matches[0].can_auto_reuse);
|
||||
assert_eq!(matches[0].entry.translated_text, "你好");
|
||||
assert_eq!(matches[0].entry.official_release_id, "release-1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_future_schema_version() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let path = temp.path().join("tm.sqlite");
|
||||
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
|
||||
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
|
||||
.execute(&repository.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
repository.pool.close().await;
|
||||
|
||||
let error = SqliteTranslationMemoryRepository::new(&path)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error
|
||||
.to_string()
|
||||
.contains("不支持的 Translation Memory schema"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = draft("release-1", "Hello", "你好");
|
||||
repository.upsert_candidate(first.clone()).await.unwrap();
|
||||
repository.upsert_candidate(first).await.unwrap();
|
||||
assert_eq!(repository.summary().await.unwrap().record_count, 1);
|
||||
|
||||
let mut different = draft("release-2", "Hello", "你好");
|
||||
different
|
||||
.source_context
|
||||
.insert("field_path".to_string(), "m_Other".to_string());
|
||||
let matches = repository
|
||||
.find_matches("Hello", &different.source_context, 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
matches[0].match_kind,
|
||||
TranslationMemoryMatchKind::CandidateExact
|
||||
);
|
||||
assert!(!matches[0].can_auto_reuse);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_empty_translation_candidates() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let error = repository
|
||||
.upsert_candidate(draft("release-1", "Hello", " \n"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("translated_text"));
|
||||
assert_eq!(repository.summary().await.unwrap().record_count, 0);
|
||||
}
|
||||
}
|
||||
@@ -161,6 +161,12 @@ pub struct TranslationTaskUnitResult {
|
||||
pub source_text: String,
|
||||
/// Provider-produced or human-supplied translation.
|
||||
pub translated_text: String,
|
||||
/// Result source kind.
|
||||
#[serde(default)]
|
||||
pub source_kind: TranslationTaskResultSourceKind,
|
||||
/// Trusted Translation Memory record used for this result, when applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_memory_record_id: Option<String>,
|
||||
/// Provider identifier.
|
||||
pub provider: String,
|
||||
/// Provider run that produced this result.
|
||||
@@ -169,6 +175,30 @@ pub struct TranslationTaskUnitResult {
|
||||
pub translated_unix_seconds: u64,
|
||||
}
|
||||
|
||||
/// Source of one persisted TextUnit translation result.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TranslationTaskResultSourceKind {
|
||||
/// Result returned by the configured provider.
|
||||
#[default]
|
||||
Provider,
|
||||
/// Result submitted through the manual task update interface.
|
||||
Manual,
|
||||
/// Result reused from a trusted Translation Memory entry.
|
||||
TranslationMemory,
|
||||
}
|
||||
|
||||
impl TranslationTaskResultSourceKind {
|
||||
/// Returns the stable JSON label.
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Provider => "provider",
|
||||
Self::Manual => "manual",
|
||||
Self::TranslationMemory => "translation_memory",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One provider execution associated with one or more translation units.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ProviderRun {
|
||||
@@ -247,12 +277,18 @@ pub fn build_translation_handoff(
|
||||
Vec::new(),
|
||||
)
|
||||
});
|
||||
let unit_status = match state.0 {
|
||||
TranslationTaskStatus::Queued => TranslationUnitStatus::Queued,
|
||||
TranslationTaskStatus::Running => TranslationUnitStatus::Translating,
|
||||
TranslationTaskStatus::Failed => TranslationUnitStatus::Failed,
|
||||
TranslationTaskStatus::Completed => TranslationUnitStatus::Translated,
|
||||
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped,
|
||||
let unit_status = if state.5.is_empty() {
|
||||
match state.0 {
|
||||
TranslationTaskStatus::Queued => TranslationUnitStatus::Queued,
|
||||
TranslationTaskStatus::Running => TranslationUnitStatus::Translating,
|
||||
TranslationTaskStatus::Failed => TranslationUnitStatus::Failed,
|
||||
TranslationTaskStatus::Completed => TranslationUnitStatus::Translated,
|
||||
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped,
|
||||
}
|
||||
} else {
|
||||
// A task can retain successful TM hits while the remaining provider
|
||||
// units are failed or waiting for retry.
|
||||
TranslationUnitStatus::Translated
|
||||
};
|
||||
let unit = TranslationUnit {
|
||||
unit_id: task.task_id.clone(),
|
||||
@@ -1168,11 +1204,29 @@ impl SqliteTranslationTaskRepository {
|
||||
pub async fn fail_claim(
|
||||
&self,
|
||||
failure: TranslationTaskFailure,
|
||||
) -> Result<PersistedTranslationTask> {
|
||||
self.fail_claim_with_results(failure, &[]).await
|
||||
}
|
||||
|
||||
/// Records a provider failure while retaining any already-resolved TextUnit
|
||||
/// results, such as trusted Translation Memory hits.
|
||||
pub async fn fail_claim_with_results(
|
||||
&self,
|
||||
failure: TranslationTaskFailure,
|
||||
translation_results: &[TranslationTaskUnitResult],
|
||||
) -> Result<PersistedTranslationTask> {
|
||||
let now = unix_seconds_now_i64();
|
||||
let next_attempt = failure
|
||||
.next_attempt_unix_seconds
|
||||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX));
|
||||
let translation_results_json = if translation_results.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
serde_json::to_string(translation_results)
|
||||
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?,
|
||||
)
|
||||
};
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
@@ -1183,11 +1237,12 @@ impl SqliteTranslationTaskRepository {
|
||||
lease_expires_unix_seconds = NULL,
|
||||
failure_class = ?4,
|
||||
failure_retryable = ?5,
|
||||
next_attempt_unix_seconds = ?6
|
||||
next_attempt_unix_seconds = ?6,
|
||||
translation_results_json = COALESCE(?7, translation_results_json)
|
||||
WHERE task_id = ?1
|
||||
AND worker_status = 'running'
|
||||
AND lease_owner = ?7
|
||||
AND provider_run_id = ?8
|
||||
AND lease_owner = ?8
|
||||
AND provider_run_id = ?9
|
||||
"#,
|
||||
)
|
||||
.bind(&failure.task_id)
|
||||
@@ -1196,6 +1251,7 @@ impl SqliteTranslationTaskRepository {
|
||||
.bind(&failure.failure_class)
|
||||
.bind(if failure.retryable { 1_i64 } else { 0_i64 })
|
||||
.bind(next_attempt)
|
||||
.bind(translation_results_json)
|
||||
.bind(&failure.worker_id)
|
||||
.bind(&failure.provider_run_id)
|
||||
.execute(&self.pool)
|
||||
@@ -1747,6 +1803,8 @@ mod tests {
|
||||
unit_id: "unit-a".to_string(),
|
||||
source_text: "source".to_string(),
|
||||
translated_text: "manual translation".to_string(),
|
||||
source_kind: TranslationTaskResultSourceKind::Manual,
|
||||
translation_memory_record_id: None,
|
||||
provider: "manual".to_string(),
|
||||
provider_run_id: "manual-run-1".to_string(),
|
||||
translated_unix_seconds: 321,
|
||||
@@ -1864,6 +1922,8 @@ mod tests {
|
||||
unit_id: "unit-a".to_string(),
|
||||
source_text: "source".to_string(),
|
||||
translated_text: "translated".to_string(),
|
||||
source_kind: TranslationTaskResultSourceKind::Provider,
|
||||
translation_memory_record_id: None,
|
||||
provider: "mock".to_string(),
|
||||
provider_run_id: second_run.clone(),
|
||||
translated_unix_seconds: 1,
|
||||
|
||||
@@ -2,15 +2,21 @@
|
||||
//!
|
||||
//! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态,不
|
||||
//! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract;状态、
|
||||
//! 租约和译文结果始终写入 `translation-tasks.sqlite`。
|
||||
//! 租约和任务结果写入 release 级 `translation-tasks.sqlite`,跨 release 的
|
||||
//! Translation Memory 写入项目级独立 SQLite 数据库。
|
||||
|
||||
use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit};
|
||||
use crate::official_textunit_queue::read_textunit_task_queue_at;
|
||||
use crate::translation_memory::{translation_memory_context, SqliteTranslationMemoryRepository};
|
||||
use crate::translation_tasks::{
|
||||
PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure,
|
||||
TranslationTaskUnitResult,
|
||||
TranslationTaskResultSourceKind, TranslationTaskUnitResult,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use bat_core::domain::{
|
||||
TranslationMemoryDraft, TranslationMemorySourceKind, TranslationMemorySourceTrace,
|
||||
};
|
||||
use bat_core::repositories::TranslationMemoryRepository;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::env;
|
||||
@@ -89,6 +95,9 @@ pub struct TranslationWorkerConfig {
|
||||
pub max_tasks: Option<usize>,
|
||||
/// worker 实例前缀,用于 lease 诊断。
|
||||
pub worker_id: String,
|
||||
/// Translation Memory SQLite path. `None` uses the output-root default.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub translation_memory_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Default for TranslationWorkerConfig {
|
||||
@@ -102,6 +111,7 @@ impl Default for TranslationWorkerConfig {
|
||||
retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
||||
max_tasks: None,
|
||||
worker_id: format!("bat-worker-{}", std::process::id()),
|
||||
translation_memory_path: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -576,6 +586,16 @@ pub struct TranslationWorkerReport {
|
||||
pub remaining_count: usize,
|
||||
/// 失败诊断。
|
||||
pub failures: Vec<TranslationWorkerFailure>,
|
||||
/// Translation Memory database path used by this run.
|
||||
pub translation_memory_path: PathBuf,
|
||||
/// Whether the Translation Memory repository was available.
|
||||
pub translation_memory_available: bool,
|
||||
/// Number of TextUnits reused from trusted Translation Memory.
|
||||
pub translation_memory_hit_count: usize,
|
||||
/// Number of TextUnits sent to the provider after Translation Memory lookup.
|
||||
pub provider_unit_count: usize,
|
||||
/// Translation Memory diagnostics that did not invalidate provider work.
|
||||
pub translation_memory_failures: Vec<String>,
|
||||
}
|
||||
|
||||
/// worker 失败诊断。
|
||||
@@ -599,7 +619,10 @@ struct WorkerStats {
|
||||
completed_count: AtomicUsize,
|
||||
failed_count: AtomicUsize,
|
||||
retry_scheduled_count: AtomicUsize,
|
||||
translation_memory_hit_count: AtomicUsize,
|
||||
provider_unit_count: AtomicUsize,
|
||||
failures: Mutex<Vec<TranslationWorkerFailure>>,
|
||||
translation_memory_failures: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
struct WorkerTaskContext<'a> {
|
||||
@@ -611,12 +634,25 @@ struct WorkerTaskContext<'a> {
|
||||
max_attempts: u32,
|
||||
retry_backoff: Duration,
|
||||
stats: &'a WorkerStats,
|
||||
translation_memory: Option<&'a dyn TranslationMemoryRepository>,
|
||||
}
|
||||
|
||||
/// 运行一个 provider worker 轮次。
|
||||
pub async fn run_translation_worker_at(
|
||||
resource_root: &Path,
|
||||
config: &TranslationWorkerConfig,
|
||||
) -> anyhow::Result<TranslationWorkerReport> {
|
||||
run_translation_worker_at_with_cancellation(resource_root, config, Arc::new(|| false)).await
|
||||
}
|
||||
|
||||
/// 运行一个可协作取消的 provider worker 轮次。
|
||||
///
|
||||
/// 取消只在 claim 循环边界检查;已经开始的 provider 请求会先完成,
|
||||
/// 避免丢失 lease 结果或遗留未清理的外部子进程。
|
||||
pub async fn run_translation_worker_at_with_cancellation(
|
||||
resource_root: &Path,
|
||||
config: &TranslationWorkerConfig,
|
||||
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||
) -> anyhow::Result<TranslationWorkerReport> {
|
||||
config.validate()?;
|
||||
let provider: Arc<dyn TranslationProvider> = match config.provider {
|
||||
@@ -625,7 +661,13 @@ pub async fn run_translation_worker_at(
|
||||
)?),
|
||||
TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?),
|
||||
};
|
||||
run_translation_worker_with_provider(resource_root, config, provider).await
|
||||
run_translation_worker_with_provider_and_cancellation(
|
||||
resource_root,
|
||||
config,
|
||||
provider,
|
||||
should_cancel,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 使用指定 provider 运行 worker,供测试和插件宿主使用。
|
||||
@@ -633,6 +675,21 @@ pub async fn run_translation_worker_with_provider(
|
||||
resource_root: &Path,
|
||||
config: &TranslationWorkerConfig,
|
||||
provider: Arc<dyn TranslationProvider>,
|
||||
) -> anyhow::Result<TranslationWorkerReport> {
|
||||
run_translation_worker_with_provider_and_cancellation(
|
||||
resource_root,
|
||||
config,
|
||||
provider,
|
||||
Arc::new(|| false),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn run_translation_worker_with_provider_and_cancellation(
|
||||
resource_root: &Path,
|
||||
config: &TranslationWorkerConfig,
|
||||
provider: Arc<dyn TranslationProvider>,
|
||||
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||
) -> anyhow::Result<TranslationWorkerReport> {
|
||||
config.validate()?;
|
||||
let queue = read_textunit_task_queue_at(resource_root)
|
||||
@@ -643,6 +700,21 @@ pub async fn run_translation_worker_with_provider(
|
||||
.map_err(anyhow::Error::msg)?
|
||||
.ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?,
|
||||
);
|
||||
let translation_memory_path = config
|
||||
.translation_memory_path
|
||||
.clone()
|
||||
.unwrap_or_else(|| SqliteTranslationMemoryRepository::repository_path(resource_root));
|
||||
let (translation_memory, translation_memory_startup_failure) =
|
||||
match SqliteTranslationMemoryRepository::new(&translation_memory_path).await {
|
||||
Ok(repository) => (Some(Arc::new(repository)), None),
|
||||
Err(error) => (
|
||||
None,
|
||||
Some(format!(
|
||||
"打开 Translation Memory 数据库失败 {}:{error}",
|
||||
translation_memory_path.display()
|
||||
)),
|
||||
),
|
||||
};
|
||||
let repository = Arc::new(
|
||||
SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path(
|
||||
resource_root,
|
||||
@@ -659,6 +731,13 @@ pub async fn run_translation_worker_with_provider(
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败:{error}"))?;
|
||||
let stats = Arc::new(WorkerStats::default());
|
||||
if let Some(failure) = translation_memory_startup_failure {
|
||||
stats
|
||||
.translation_memory_failures
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
|
||||
.push(failure);
|
||||
}
|
||||
let claimed_limit = Arc::new(AtomicUsize::new(0));
|
||||
let mut handles = Vec::with_capacity(config.concurrency);
|
||||
|
||||
@@ -674,8 +753,13 @@ pub async fn run_translation_worker_with_provider(
|
||||
let max_attempts = config.max_attempts;
|
||||
let lease_seconds = config.lease_seconds;
|
||||
let retry_backoff = config.retry_backoff;
|
||||
let translation_memory = translation_memory.clone();
|
||||
let should_cancel = Arc::clone(&should_cancel);
|
||||
handles.push(tokio::spawn(async move {
|
||||
loop {
|
||||
if should_cancel() {
|
||||
return Err(anyhow::anyhow!("翻译 worker 已取消"));
|
||||
}
|
||||
if let Some(max_tasks) = max_tasks {
|
||||
let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel);
|
||||
if reservation >= max_tasks {
|
||||
@@ -704,6 +788,9 @@ pub async fn run_translation_worker_with_provider(
|
||||
max_attempts,
|
||||
retry_backoff,
|
||||
stats: &stats,
|
||||
translation_memory: translation_memory
|
||||
.as_deref()
|
||||
.map(|repository| repository as &dyn TranslationMemoryRepository),
|
||||
},
|
||||
&task,
|
||||
)
|
||||
@@ -712,10 +799,24 @@ pub async fn run_translation_worker_with_provider(
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}));
|
||||
}
|
||||
let mut first_worker_error = None;
|
||||
for handle in handles {
|
||||
handle
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("等待翻译 worker 失败:{error}"))??;
|
||||
match handle.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => {
|
||||
if first_worker_error.is_none() {
|
||||
first_worker_error = Some(error);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
if first_worker_error.is_none() {
|
||||
first_worker_error = Some(anyhow::anyhow!("等待翻译 worker 失败:{error}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = first_worker_error {
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let remaining_count = repository
|
||||
@@ -736,6 +837,11 @@ pub async fn run_translation_worker_with_provider(
|
||||
.map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))?
|
||||
.clone();
|
||||
let failed_count = stats.failed_count.load(Ordering::Relaxed);
|
||||
let translation_memory_failures = stats
|
||||
.translation_memory_failures
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("读取 Translation Memory 诊断时 mutex poisoned"))?
|
||||
.clone();
|
||||
Ok(TranslationWorkerReport {
|
||||
command: "translation-worker",
|
||||
status: if failed_count == 0 {
|
||||
@@ -752,6 +858,11 @@ pub async fn run_translation_worker_with_provider(
|
||||
retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed),
|
||||
remaining_count,
|
||||
failures,
|
||||
translation_memory_path,
|
||||
translation_memory_available: translation_memory.is_some(),
|
||||
translation_memory_hit_count: stats.translation_memory_hit_count.load(Ordering::Relaxed),
|
||||
provider_unit_count: stats.provider_unit_count.load(Ordering::Relaxed),
|
||||
translation_memory_failures,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -759,66 +870,176 @@ async fn process_claimed_task(
|
||||
context: &WorkerTaskContext<'_>,
|
||||
task: &PersistedTranslationTask,
|
||||
) -> anyhow::Result<()> {
|
||||
let request = match provider_request(task, context.index) {
|
||||
Ok(request) => request,
|
||||
Err(error) => {
|
||||
record_provider_failure(
|
||||
context,
|
||||
task,
|
||||
TranslationProviderError::new(
|
||||
TranslationProviderFailureClass::InvalidRequest,
|
||||
error.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
match context.provider.translate(request.clone()).await {
|
||||
Ok(response) => {
|
||||
let results =
|
||||
match validate_provider_response(&request, response, context.provider_name) {
|
||||
Ok(results) => results,
|
||||
Err(error) => {
|
||||
record_provider_failure(
|
||||
context,
|
||||
task,
|
||||
TranslationProviderError::new(
|
||||
TranslationProviderFailureClass::InvalidRequest,
|
||||
error.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
let task_units = task_index_units(task, context.index)?;
|
||||
let mut results = BTreeMap::new();
|
||||
let mut provider_units = Vec::new();
|
||||
for unit in &task_units {
|
||||
if let Some(translation_memory) = context.translation_memory {
|
||||
let source_context = translation_memory_context(
|
||||
&unit.destination,
|
||||
unit.archive_entry.as_deref(),
|
||||
unit.serialized_file.as_deref(),
|
||||
unit.path_id,
|
||||
unit.class_id,
|
||||
unit.field_path.as_deref(),
|
||||
unit.format.as_deref(),
|
||||
unit.asset_name.as_deref(),
|
||||
unit.text_source_kind.as_deref(),
|
||||
&unit.context,
|
||||
);
|
||||
match translation_memory
|
||||
.find_matches(&unit.source_text, &source_context, 1)
|
||||
.await
|
||||
{
|
||||
Ok(matches) => {
|
||||
if let Some(found) = matches.into_iter().find(|item| item.can_auto_reuse) {
|
||||
context
|
||||
.stats
|
||||
.translation_memory_hit_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
results.insert(
|
||||
unit.id.clone(),
|
||||
translation_memory_result(task, unit, &found.entry),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
context
|
||||
.repository
|
||||
.complete_claim(
|
||||
&task.task.task_id,
|
||||
context.worker_id,
|
||||
&task.provider_run_id.clone().unwrap_or_default(),
|
||||
context.provider_name,
|
||||
}
|
||||
Err(error) => {
|
||||
record_translation_memory_failure(
|
||||
context,
|
||||
format!(
|
||||
"任务 {} TextUnit {} 查询失败:{}",
|
||||
task.task.task_id, unit.id, error
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
provider_units.push(*unit);
|
||||
}
|
||||
|
||||
if !provider_units.is_empty() {
|
||||
context
|
||||
.stats
|
||||
.provider_unit_count
|
||||
.fetch_add(provider_units.len(), Ordering::Relaxed);
|
||||
let request = match provider_request(task, &provider_units) {
|
||||
Ok(request) => request,
|
||||
Err(error) => {
|
||||
record_provider_failure(
|
||||
context,
|
||||
task,
|
||||
TranslationProviderError::new(
|
||||
TranslationProviderFailureClass::InvalidRequest,
|
||||
error.to_string(),
|
||||
),
|
||||
&results,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?;
|
||||
context
|
||||
.stats
|
||||
.completed_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(error) => {
|
||||
record_provider_failure(context, task, error).await?;
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
match context.provider.translate(request.clone()).await {
|
||||
Ok(response) => {
|
||||
let provider_results =
|
||||
match validate_provider_response(&request, response, context.provider_name) {
|
||||
Ok(results) => results,
|
||||
Err(error) => {
|
||||
record_provider_failure(
|
||||
context,
|
||||
task,
|
||||
TranslationProviderError::new(
|
||||
TranslationProviderFailureClass::InvalidRequest,
|
||||
error.to_string(),
|
||||
),
|
||||
&results,
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
for result in &provider_results {
|
||||
results.insert(result.unit_id.clone(), result.clone());
|
||||
if let Some(unit) = provider_units.iter().find(|unit| unit.id == result.unit_id)
|
||||
{
|
||||
let source_context = translation_memory_context(
|
||||
&unit.destination,
|
||||
unit.archive_entry.as_deref(),
|
||||
unit.serialized_file.as_deref(),
|
||||
unit.path_id,
|
||||
unit.class_id,
|
||||
unit.field_path.as_deref(),
|
||||
unit.format.as_deref(),
|
||||
unit.asset_name.as_deref(),
|
||||
unit.text_source_kind.as_deref(),
|
||||
&unit.context,
|
||||
);
|
||||
let draft = TranslationMemoryDraft {
|
||||
source_text: result.source_text.clone(),
|
||||
source_context,
|
||||
translated_text: result.translated_text.clone(),
|
||||
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||||
official_release_id: task.task.official_release_id.clone(),
|
||||
source_trace: translation_memory_trace(task, unit),
|
||||
provider: Some(context.provider_name.to_string()),
|
||||
provider_run_id: Some(request.provider_run_id.clone()),
|
||||
observed_unix_seconds: unix_seconds_now(),
|
||||
};
|
||||
if let Some(translation_memory) = context.translation_memory {
|
||||
if let Err(error) = translation_memory.upsert_candidate(draft).await {
|
||||
record_translation_memory_failure(
|
||||
context,
|
||||
format!(
|
||||
"任务 {} TextUnit {} 写入候选失败:{}",
|
||||
task.task.task_id, unit.id, error
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
record_provider_failure(context, task, error, &results).await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let ordered_results = task_units
|
||||
.iter()
|
||||
.filter_map(|unit| results.get(&unit.id).cloned())
|
||||
.collect::<Vec<_>>();
|
||||
if ordered_results.len() != task_units.len() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"任务 {} 的译文结果不完整:expected={} actual={}",
|
||||
task.task.task_id,
|
||||
task_units.len(),
|
||||
ordered_results.len()
|
||||
));
|
||||
}
|
||||
context
|
||||
.repository
|
||||
.complete_claim(
|
||||
&task.task.task_id,
|
||||
context.worker_id,
|
||||
&task.provider_run_id.clone().unwrap_or_default(),
|
||||
context.provider_name,
|
||||
&ordered_results,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?;
|
||||
context
|
||||
.stats
|
||||
.completed_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn provider_request(
|
||||
fn task_index_units<'a>(
|
||||
task: &PersistedTranslationTask,
|
||||
index: &crate::official_parse::OfficialTextUnitIndex,
|
||||
) -> anyhow::Result<TranslationProviderRequest> {
|
||||
index: &'a crate::official_parse::OfficialTextUnitIndex,
|
||||
) -> anyhow::Result<Vec<&'a OfficialTextUnitIndexUnit>> {
|
||||
let parse_entry_key = task
|
||||
.task
|
||||
.parse_entry_key
|
||||
@@ -832,7 +1053,6 @@ fn provider_request(
|
||||
&& unit.destination == task.task.destination
|
||||
&& unit.archive_entry == task.task.archive_entry
|
||||
})
|
||||
.map(|unit| provider_unit(task, unit))
|
||||
.collect::<Vec<_>>();
|
||||
if units.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -840,6 +1060,13 @@ fn provider_request(
|
||||
task.task.task_id
|
||||
));
|
||||
}
|
||||
Ok(units)
|
||||
}
|
||||
|
||||
fn provider_request(
|
||||
task: &PersistedTranslationTask,
|
||||
index_units: &[&OfficialTextUnitIndexUnit],
|
||||
) -> anyhow::Result<TranslationProviderRequest> {
|
||||
let provider_run_id = task
|
||||
.provider_run_id
|
||||
.clone()
|
||||
@@ -850,7 +1077,10 @@ fn provider_request(
|
||||
task_id: task.task.task_id.clone(),
|
||||
destination: task.task.destination.clone(),
|
||||
archive_entry: task.task.archive_entry.clone(),
|
||||
units,
|
||||
units: index_units
|
||||
.iter()
|
||||
.map(|unit| provider_unit(task, unit))
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -914,10 +1144,18 @@ fn validate_provider_response(
|
||||
result.unit_id
|
||||
));
|
||||
}
|
||||
if result.translated_text.trim().is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"provider 返回空 translated_text:{}",
|
||||
result.unit_id
|
||||
));
|
||||
}
|
||||
results.push(TranslationTaskUnitResult {
|
||||
unit_id: result.unit_id,
|
||||
source_text: result.source_text,
|
||||
translated_text: result.translated_text,
|
||||
source_kind: TranslationTaskResultSourceKind::Provider,
|
||||
translation_memory_record_id: None,
|
||||
provider: provider_name.to_string(),
|
||||
provider_run_id: request.provider_run_id.clone(),
|
||||
translated_unix_seconds: unix_seconds_now(),
|
||||
@@ -933,25 +1171,81 @@ fn validate_provider_response(
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn translation_memory_result(
|
||||
task: &PersistedTranslationTask,
|
||||
unit: &OfficialTextUnitIndexUnit,
|
||||
entry: &bat_core::domain::TranslationMemoryEntry,
|
||||
) -> TranslationTaskUnitResult {
|
||||
TranslationTaskUnitResult {
|
||||
unit_id: unit.id.clone(),
|
||||
source_text: unit.source_text.clone(),
|
||||
translated_text: entry.translated_text.clone(),
|
||||
source_kind: TranslationTaskResultSourceKind::TranslationMemory,
|
||||
translation_memory_record_id: Some(entry.record_id.clone()),
|
||||
provider: "translation_memory".to_string(),
|
||||
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
||||
translated_unix_seconds: unix_seconds_now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn translation_memory_trace(
|
||||
task: &PersistedTranslationTask,
|
||||
unit: &OfficialTextUnitIndexUnit,
|
||||
) -> TranslationMemorySourceTrace {
|
||||
TranslationMemorySourceTrace {
|
||||
official_release_id: task.task.official_release_id.clone(),
|
||||
unit_id: Some(unit.id.clone()),
|
||||
task_id: Some(task.task.task_id.clone()),
|
||||
destination: Some(unit.destination.clone()),
|
||||
archive_entry: unit.archive_entry.clone(),
|
||||
serialized_file: unit.serialized_file.clone(),
|
||||
path_id: unit.path_id,
|
||||
class_id: unit.class_id,
|
||||
field_path: unit.field_path.clone(),
|
||||
format: unit.format.clone(),
|
||||
asset_name: unit.asset_name.clone(),
|
||||
text_source_kind: unit.text_source_kind.clone(),
|
||||
source_url: Some(unit.source_url.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn record_translation_memory_failure(
|
||||
context: &WorkerTaskContext<'_>,
|
||||
message: String,
|
||||
) -> anyhow::Result<()> {
|
||||
context
|
||||
.stats
|
||||
.translation_memory_failures
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
|
||||
.push(message);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_provider_failure(
|
||||
context: &WorkerTaskContext<'_>,
|
||||
task: &PersistedTranslationTask,
|
||||
error: TranslationProviderError,
|
||||
partial_results: &BTreeMap<String, TranslationTaskUnitResult>,
|
||||
) -> anyhow::Result<()> {
|
||||
let retryable = error.retryable && task.attempt_count < context.max_attempts;
|
||||
let next_attempt =
|
||||
retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs()));
|
||||
let partial_results = partial_results.values().cloned().collect::<Vec<_>>();
|
||||
context
|
||||
.repository
|
||||
.fail_claim(TranslationTaskFailure {
|
||||
task_id: task.task.task_id.clone(),
|
||||
worker_id: context.worker_id.to_string(),
|
||||
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
||||
failure_class: error.class.as_str().to_string(),
|
||||
failure_reason: error.message.clone(),
|
||||
retryable,
|
||||
next_attempt_unix_seconds: next_attempt,
|
||||
})
|
||||
.fail_claim_with_results(
|
||||
TranslationTaskFailure {
|
||||
task_id: task.task.task_id.clone(),
|
||||
worker_id: context.worker_id.to_string(),
|
||||
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
||||
failure_class: error.class.as_str().to_string(),
|
||||
failure_reason: error.message.clone(),
|
||||
retryable,
|
||||
next_attempt_unix_seconds: next_attempt,
|
||||
},
|
||||
&partial_results,
|
||||
)
|
||||
.await
|
||||
.map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败:{failure}"))?;
|
||||
context.stats.failed_count.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -1161,6 +1455,277 @@ mod tests {
|
||||
assert_eq!(task.translation_results[0].translated_text, "translated-0");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_honors_cancellation_before_claiming_tasks() {
|
||||
let (temp, queue) = fixture_root();
|
||||
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||
crate::official_parse::write_textunit_index_at(temp.path(), &index(temp.path())).unwrap();
|
||||
let config = TranslationWorkerConfig {
|
||||
concurrency: 2,
|
||||
..TranslationWorkerConfig::default()
|
||||
};
|
||||
|
||||
let result =
|
||||
run_translation_worker_at_with_cancellation(temp.path(), &config, Arc::new(|| true))
|
||||
.await;
|
||||
|
||||
assert!(result.unwrap_err().to_string().contains("已取消"));
|
||||
let repository = SqliteTranslationTaskRepository::open(
|
||||
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||
assert_eq!(
|
||||
task.task_status,
|
||||
crate::translation_tasks::TranslationTaskStatus::Queued
|
||||
);
|
||||
assert_eq!(task.attempt_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_reuses_trusted_tm_for_part_of_a_task_and_calls_provider_for_the_rest() {
|
||||
let (temp, queue) = fixture_root();
|
||||
let textunit_index = index(temp.path());
|
||||
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
|
||||
|
||||
let translation_memory_path = temp.path().join("translation-memory.sqlite");
|
||||
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let unit = &textunit_index.units[0];
|
||||
let source_context = translation_memory_context(
|
||||
&unit.destination,
|
||||
unit.archive_entry.as_deref(),
|
||||
unit.serialized_file.as_deref(),
|
||||
unit.path_id,
|
||||
unit.class_id,
|
||||
unit.field_path.as_deref(),
|
||||
unit.format.as_deref(),
|
||||
unit.asset_name.as_deref(),
|
||||
unit.text_source_kind.as_deref(),
|
||||
&unit.context,
|
||||
);
|
||||
let candidate = translation_memory
|
||||
.upsert_candidate(TranslationMemoryDraft {
|
||||
source_text: unit.source_text.clone(),
|
||||
source_context,
|
||||
translated_text: "trusted-from-tm".to_string(),
|
||||
translation_source_kind: TranslationMemorySourceKind::Manual,
|
||||
official_release_id: queue.official_release_id.clone(),
|
||||
source_trace: TranslationMemorySourceTrace {
|
||||
official_release_id: queue.official_release_id.clone(),
|
||||
unit_id: Some(unit.id.clone()),
|
||||
task_id: Some(queue.tasks[0].task_id.clone()),
|
||||
destination: Some(unit.destination.clone()),
|
||||
archive_entry: unit.archive_entry.clone(),
|
||||
serialized_file: unit.serialized_file.clone(),
|
||||
path_id: unit.path_id,
|
||||
class_id: unit.class_id,
|
||||
field_path: unit.field_path.clone(),
|
||||
format: unit.format.clone(),
|
||||
asset_name: unit.asset_name.clone(),
|
||||
text_source_kind: unit.text_source_kind.clone(),
|
||||
source_url: Some(unit.source_url.clone()),
|
||||
},
|
||||
provider: None,
|
||||
provider_run_id: None,
|
||||
observed_unix_seconds: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let trusted = translation_memory
|
||||
.confirm(
|
||||
&candidate.record_id,
|
||||
"test-reviewer",
|
||||
Some("accepted".to_string()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
trusted.trust_status,
|
||||
bat_core::domain::TranslationMemoryTrustStatus::Trusted
|
||||
);
|
||||
|
||||
let fixture = temp.path().join("partial-mock.json");
|
||||
std::fs::write(
|
||||
&fixture,
|
||||
serde_json::to_vec(&serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"translations": {
|
||||
"direct:bundle#unit:1": "translated-by-provider"
|
||||
}
|
||||
}))
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let config = TranslationWorkerConfig {
|
||||
fixture_path: Some(fixture),
|
||||
translation_memory_path: Some(translation_memory_path),
|
||||
concurrency: 1,
|
||||
retry_backoff: Duration::ZERO,
|
||||
..TranslationWorkerConfig::default()
|
||||
};
|
||||
let report = run_translation_worker_at(temp.path(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report.translation_memory_hit_count, 1);
|
||||
assert_eq!(report.provider_unit_count, 1);
|
||||
assert_eq!(report.completed_count, 1);
|
||||
|
||||
let repository = SqliteTranslationTaskRepository::open(
|
||||
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||
assert_eq!(task.translation_results.len(), 2);
|
||||
assert_eq!(
|
||||
task.translation_results[0].translated_text,
|
||||
"trusted-from-tm"
|
||||
);
|
||||
assert_eq!(
|
||||
task.translation_results[0].source_kind,
|
||||
TranslationTaskResultSourceKind::TranslationMemory
|
||||
);
|
||||
assert_eq!(
|
||||
task.translation_results[0]
|
||||
.translation_memory_record_id
|
||||
.as_deref(),
|
||||
Some(trusted.record_id.as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
task.translation_results[1].translated_text,
|
||||
"translated-by-provider"
|
||||
);
|
||||
assert_eq!(
|
||||
task.translation_results[1].source_kind,
|
||||
TranslationTaskResultSourceKind::Provider
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_retains_tm_hits_when_provider_fails_for_remaining_units() {
|
||||
let (temp, queue) = fixture_root();
|
||||
let textunit_index = index(temp.path());
|
||||
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
|
||||
|
||||
let translation_memory_path = temp.path().join("translation-memory.sqlite");
|
||||
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let unit = &textunit_index.units[0];
|
||||
let source_context = translation_memory_context(
|
||||
&unit.destination,
|
||||
unit.archive_entry.as_deref(),
|
||||
unit.serialized_file.as_deref(),
|
||||
unit.path_id,
|
||||
unit.class_id,
|
||||
unit.field_path.as_deref(),
|
||||
unit.format.as_deref(),
|
||||
unit.asset_name.as_deref(),
|
||||
unit.text_source_kind.as_deref(),
|
||||
&unit.context,
|
||||
);
|
||||
let candidate = translation_memory
|
||||
.upsert_candidate(TranslationMemoryDraft {
|
||||
source_text: unit.source_text.clone(),
|
||||
source_context,
|
||||
translated_text: "trusted-from-tm".to_string(),
|
||||
translation_source_kind: TranslationMemorySourceKind::Manual,
|
||||
official_release_id: queue.official_release_id.clone(),
|
||||
source_trace: TranslationMemorySourceTrace {
|
||||
official_release_id: queue.official_release_id.clone(),
|
||||
unit_id: Some(unit.id.clone()),
|
||||
task_id: Some(queue.tasks[0].task_id.clone()),
|
||||
destination: Some(unit.destination.clone()),
|
||||
archive_entry: unit.archive_entry.clone(),
|
||||
serialized_file: unit.serialized_file.clone(),
|
||||
path_id: unit.path_id,
|
||||
class_id: unit.class_id,
|
||||
field_path: unit.field_path.clone(),
|
||||
format: unit.format.clone(),
|
||||
asset_name: unit.asset_name.clone(),
|
||||
text_source_kind: unit.text_source_kind.clone(),
|
||||
source_url: Some(unit.source_url.clone()),
|
||||
},
|
||||
provider: None,
|
||||
provider_run_id: None,
|
||||
observed_unix_seconds: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let trusted = translation_memory
|
||||
.confirm(
|
||||
&candidate.record_id,
|
||||
"test-reviewer",
|
||||
Some("accepted".to_string()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let fixture = temp.path().join("provider-failure.json");
|
||||
std::fs::write(
|
||||
&fixture,
|
||||
serde_json::to_vec(&serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"failures": {
|
||||
"textunit/release-1/bundle": {
|
||||
"class": "rate_limited",
|
||||
"message": "fixture throttled",
|
||||
"retryable": false
|
||||
}
|
||||
}
|
||||
}))
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let config = TranslationWorkerConfig {
|
||||
fixture_path: Some(fixture),
|
||||
translation_memory_path: Some(translation_memory_path),
|
||||
concurrency: 1,
|
||||
max_attempts: 1,
|
||||
retry_backoff: Duration::ZERO,
|
||||
..TranslationWorkerConfig::default()
|
||||
};
|
||||
let report = run_translation_worker_at(temp.path(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report.translation_memory_hit_count, 1);
|
||||
assert_eq!(report.provider_unit_count, 1);
|
||||
assert_eq!(report.failed_count, 1);
|
||||
|
||||
let repository = SqliteTranslationTaskRepository::open(
|
||||
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||
assert_eq!(
|
||||
task.task_status,
|
||||
crate::translation_tasks::TranslationTaskStatus::Failed
|
||||
);
|
||||
assert_eq!(task.translation_results.len(), 1);
|
||||
assert_eq!(
|
||||
task.translation_results[0]
|
||||
.translation_memory_record_id
|
||||
.as_deref(),
|
||||
Some(trusted.record_id.as_str())
|
||||
);
|
||||
|
||||
let handoff = crate::translation_tasks::build_translation_handoff(&queue, &[task]);
|
||||
assert_eq!(
|
||||
handoff.units[0].status,
|
||||
crate::translation_tasks::TranslationUnitStatus::Translated
|
||||
);
|
||||
assert_eq!(
|
||||
handoff.job.status,
|
||||
crate::translation_tasks::TranslationJobStatus::Failed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() {
|
||||
let (temp, queue) = fixture_root();
|
||||
|
||||
@@ -9,7 +9,7 @@ use crate::path_security::{
|
||||
use crate::{
|
||||
LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch,
|
||||
LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository,
|
||||
TranslationTaskStatus, TranslationTaskUnitResult,
|
||||
TranslationTaskResultSourceKind, TranslationTaskStatus, TranslationTaskUnitResult,
|
||||
};
|
||||
use bat_assetbundle::{
|
||||
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
|
||||
@@ -72,6 +72,12 @@ pub struct TranslationWorkbenchEntry {
|
||||
/// Provider run that produced this translation, when imported from worker output.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// Source of the worker result (`provider`, `manual`, or `translation_memory`).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_source_kind: Option<String>,
|
||||
/// Trusted Translation Memory record used for this translation, when applicable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translation_memory_record_id: Option<String>,
|
||||
/// Worker completion time for provider-produced text.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub translated_unix_seconds: Option<u64>,
|
||||
@@ -616,6 +622,8 @@ fn localized_patch_metadata(entry: &TranslationWorkbenchEntry) -> LocalizedPatch
|
||||
.to_string(),
|
||||
translation_provider: entry.translation_provider.clone(),
|
||||
provider_run_id: entry.provider_run_id.clone(),
|
||||
translation_source_kind: entry.translation_source_kind.clone(),
|
||||
translation_memory_record_id: entry.translation_memory_record_id.clone(),
|
||||
review_status: entry
|
||||
.review_status
|
||||
.clone()
|
||||
@@ -887,18 +895,30 @@ fn workbench_entry_from_worker_result(
|
||||
) -> TranslationWorkbenchEntry {
|
||||
let mut entry = TranslationWorkbenchEntry::from_index(unit);
|
||||
entry.translated_text = Some(result.translated_text.clone());
|
||||
entry.translation_provider = task
|
||||
.provider
|
||||
.clone()
|
||||
.or_else(|| Some(result.provider.clone()))
|
||||
.filter(|provider| !provider.trim().is_empty());
|
||||
entry.translation_provider = match result.source_kind {
|
||||
TranslationTaskResultSourceKind::TranslationMemory => None,
|
||||
TranslationTaskResultSourceKind::Provider | TranslationTaskResultSourceKind::Manual => task
|
||||
.provider
|
||||
.clone()
|
||||
.or_else(|| Some(result.provider.clone()))
|
||||
.filter(|provider| !provider.trim().is_empty()),
|
||||
};
|
||||
entry.provider_run_id = task
|
||||
.provider_run_id
|
||||
.clone()
|
||||
.or_else(|| Some(result.provider_run_id.clone()))
|
||||
.filter(|provider_run_id| !provider_run_id.trim().is_empty());
|
||||
entry.translation_source_kind = Some(result.source_kind.as_str().to_string());
|
||||
entry.translation_memory_record_id = result.translation_memory_record_id.clone();
|
||||
entry.translated_unix_seconds = Some(result.translated_unix_seconds);
|
||||
entry.review_status = Some("provider_completed".to_string());
|
||||
entry.review_status = Some(
|
||||
match result.source_kind {
|
||||
TranslationTaskResultSourceKind::Provider => "provider_completed",
|
||||
TranslationTaskResultSourceKind::Manual => "manual_submitted",
|
||||
TranslationTaskResultSourceKind::TranslationMemory => "translation_memory_reused",
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
entry
|
||||
}
|
||||
|
||||
@@ -938,6 +958,8 @@ impl TranslationWorkbenchEntry {
|
||||
translated_text: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
translated_unix_seconds: None,
|
||||
review_status: None,
|
||||
format: unit.format.clone(),
|
||||
@@ -975,6 +997,8 @@ mod tests {
|
||||
translated_text: None,
|
||||
translation_provider: None,
|
||||
provider_run_id: None,
|
||||
translation_source_kind: None,
|
||||
translation_memory_record_id: None,
|
||||
translated_unix_seconds: None,
|
||||
review_status: None,
|
||||
format: Some("plain".to_string()),
|
||||
|
||||
Reference in New Issue
Block a user