mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 07:24:55 +08:00
feat(translation): add Rust Translation Memory and config migration
This commit is contained in:
@@ -4,6 +4,7 @@ pub mod game_client;
|
|||||||
pub mod game_version;
|
pub mod game_version;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
pub mod translation;
|
pub mod translation;
|
||||||
|
pub mod translation_memory;
|
||||||
|
|
||||||
pub use game_client::{ClientStatus, GameClient, GameRegion};
|
pub use game_client::{ClientStatus, GameClient, GameRegion};
|
||||||
pub use game_version::{GameVersion, UnityVersion};
|
pub use game_version::{GameVersion, UnityVersion};
|
||||||
@@ -14,3 +15,8 @@ pub use translation::{
|
|||||||
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
|
ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText,
|
||||||
TranslationStatus,
|
TranslationStatus,
|
||||||
};
|
};
|
||||||
|
pub use translation_memory::{
|
||||||
|
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||||
|
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||||||
|
TranslationMemorySourceTrace, TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||||||
|
};
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
//! Translation Memory 领域对象。
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
/// TM 记录的来源类型。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationMemorySourceKind {
|
||||||
|
/// 来自 provider 输出。
|
||||||
|
Provider,
|
||||||
|
/// 来自人工确认。
|
||||||
|
Manual,
|
||||||
|
/// 来自外部导入。
|
||||||
|
Imported,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationMemorySourceKind {
|
||||||
|
/// 返回稳定的持久化标签。
|
||||||
|
pub const fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Provider => "provider",
|
||||||
|
Self::Manual => "manual",
|
||||||
|
Self::Imported => "imported",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TM 记录的可信状态。
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationMemoryTrustStatus {
|
||||||
|
/// 候选记录,不能自动复用。
|
||||||
|
Candidate,
|
||||||
|
/// 已确认可信,可在强匹配时自动复用。
|
||||||
|
Trusted,
|
||||||
|
/// 已被后续记录取代。
|
||||||
|
Superseded,
|
||||||
|
/// 已明确拒绝。
|
||||||
|
Rejected,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationMemoryTrustStatus {
|
||||||
|
/// 返回稳定的持久化标签。
|
||||||
|
pub const fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Candidate => "candidate",
|
||||||
|
Self::Trusted => "trusted",
|
||||||
|
Self::Superseded => "superseded",
|
||||||
|
Self::Rejected => "rejected",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TM 查询结果的匹配类型。
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationMemoryMatchKind {
|
||||||
|
/// 原始 source 和上下文都完全匹配,且记录可信,可自动复用。
|
||||||
|
StrongExact,
|
||||||
|
/// 原始 source 完全匹配,但上下文不同或不足,不能自动复用。
|
||||||
|
CandidateExact,
|
||||||
|
/// 原始 source 匹配,但上下文不兼容,不能自动复用。
|
||||||
|
SourceOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationMemoryMatchKind {
|
||||||
|
/// 返回稳定的查询结果标签。
|
||||||
|
pub const fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::StrongExact => "strong_exact",
|
||||||
|
Self::CandidateExact => "candidate_exact",
|
||||||
|
Self::SourceOnly => "source_only",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 稳定的上下文键值。
|
||||||
|
pub type TranslationMemoryContext = BTreeMap<String, String>;
|
||||||
|
|
||||||
|
/// TM 记录的 TextUnit / provider 溯源信息。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TranslationMemorySourceTrace {
|
||||||
|
/// 源官方 release ID。
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// 来源 TextUnit ID。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub unit_id: Option<String>,
|
||||||
|
/// 来源任务 ID。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub task_id: Option<String>,
|
||||||
|
/// 源资源 destination。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub destination: Option<String>,
|
||||||
|
/// 源 ZIP/archive entry。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub archive_entry: Option<String>,
|
||||||
|
/// Unity serialized file。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub serialized_file: Option<String>,
|
||||||
|
/// Unity object path ID。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub path_id: Option<i64>,
|
||||||
|
/// Unity class ID。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub class_id: Option<i32>,
|
||||||
|
/// TypeTree 字段路径。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub field_path: Option<String>,
|
||||||
|
/// TextUnit format。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub format: Option<String>,
|
||||||
|
/// TextAsset 名称。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub asset_name: Option<String>,
|
||||||
|
/// TextUnit 来源类型。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub text_source_kind: Option<String>,
|
||||||
|
/// 源 URL。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub source_url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TM 记录的候选输入。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TranslationMemoryDraft {
|
||||||
|
/// 原始 source text。
|
||||||
|
pub source_text: String,
|
||||||
|
/// 稳定上下文。
|
||||||
|
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||||
|
pub source_context: TranslationMemoryContext,
|
||||||
|
/// 译文。
|
||||||
|
pub translated_text: String,
|
||||||
|
/// 译文来源类型。
|
||||||
|
pub translation_source_kind: TranslationMemorySourceKind,
|
||||||
|
/// 源官方 release。
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// 原始 TextUnit / provider 溯源。
|
||||||
|
pub source_trace: TranslationMemorySourceTrace,
|
||||||
|
/// provider。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub provider: Option<String>,
|
||||||
|
/// provider run。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub provider_run_id: Option<String>,
|
||||||
|
/// 创建时间。
|
||||||
|
pub observed_unix_seconds: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 持久化 TM 记录。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TranslationMemoryEntry {
|
||||||
|
/// 稳定记录 ID。
|
||||||
|
pub record_id: String,
|
||||||
|
/// 原始 source text。
|
||||||
|
pub source_text: String,
|
||||||
|
/// source text hash。
|
||||||
|
pub source_hash: String,
|
||||||
|
/// 保守归一化后的 source text,仅用于辅助查询。
|
||||||
|
pub normalized_source_text: String,
|
||||||
|
/// 稳定上下文。
|
||||||
|
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||||
|
pub source_context: TranslationMemoryContext,
|
||||||
|
/// 上下文 hash。
|
||||||
|
pub source_context_hash: String,
|
||||||
|
/// 译文。
|
||||||
|
pub translated_text: String,
|
||||||
|
/// 译文来源类型。
|
||||||
|
pub translation_source_kind: TranslationMemorySourceKind,
|
||||||
|
/// 当前可信状态。
|
||||||
|
pub trust_status: TranslationMemoryTrustStatus,
|
||||||
|
/// 源官方 release。
|
||||||
|
pub official_release_id: String,
|
||||||
|
/// 原始 TextUnit / provider 溯源。
|
||||||
|
pub source_trace: TranslationMemorySourceTrace,
|
||||||
|
/// provider。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub provider: Option<String>,
|
||||||
|
/// provider run。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub provider_run_id: Option<String>,
|
||||||
|
/// 创建时间。
|
||||||
|
pub created_unix_seconds: u64,
|
||||||
|
/// 更新时间。
|
||||||
|
pub updated_unix_seconds: u64,
|
||||||
|
/// 可信确认时间。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub trusted_unix_seconds: Option<u64>,
|
||||||
|
/// 可信确认人。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub trusted_by: Option<String>,
|
||||||
|
/// 可信确认说明。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub trusted_reason: Option<String>,
|
||||||
|
/// 该记录替代了哪条记录。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub supersedes_record_id: Option<String>,
|
||||||
|
/// 该记录被哪条记录替代。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub superseded_by_record_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TM 查询结果。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TranslationMemoryMatch {
|
||||||
|
/// 记录本体。
|
||||||
|
pub entry: TranslationMemoryEntry,
|
||||||
|
/// 匹配类型。
|
||||||
|
pub match_kind: TranslationMemoryMatchKind,
|
||||||
|
/// 是否允许自动复用。
|
||||||
|
pub can_auto_reuse: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TM 仓储摘要。
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct TranslationMemorySummary {
|
||||||
|
/// schema 版本。
|
||||||
|
pub schema_version: u32,
|
||||||
|
/// 记录总数。
|
||||||
|
pub record_count: u64,
|
||||||
|
/// 可信记录数。
|
||||||
|
pub trusted_count: u64,
|
||||||
|
/// 候选记录数。
|
||||||
|
pub candidate_count: u64,
|
||||||
|
/// 已替代记录数。
|
||||||
|
pub superseded_count: u64,
|
||||||
|
/// 已拒绝记录数。
|
||||||
|
pub rejected_count: u64,
|
||||||
|
}
|
||||||
@@ -4,8 +4,10 @@
|
|||||||
|
|
||||||
pub mod cas_repository;
|
pub mod cas_repository;
|
||||||
pub mod resource_repository;
|
pub mod resource_repository;
|
||||||
|
pub mod translation_memory_repository;
|
||||||
pub mod translation_repository;
|
pub mod translation_repository;
|
||||||
|
|
||||||
pub use cas_repository::CasRepository;
|
pub use cas_repository::CasRepository;
|
||||||
pub use resource_repository::ResourceRepository;
|
pub use resource_repository::ResourceRepository;
|
||||||
|
pub use translation_memory_repository::TranslationMemoryRepository;
|
||||||
pub use translation_repository::TranslationRepository;
|
pub use translation_repository::TranslationRepository;
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
//! Translation Memory 仓储契约。
|
||||||
|
|
||||||
|
use crate::domain::{
|
||||||
|
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||||
|
TranslationMemoryMatch, TranslationMemorySummary,
|
||||||
|
};
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
/// 跨 official release 持久化的 Translation Memory 仓储。
|
||||||
|
///
|
||||||
|
/// 该契约只描述 V1 的精确查询和明确人工确认。仓储实现不得把
|
||||||
|
/// `TranslationTaskStatus::Completed` 或 provider 成功隐式解释为 trusted。
|
||||||
|
#[async_trait]
|
||||||
|
pub trait TranslationMemoryRepository: Send + Sync {
|
||||||
|
/// 保存一条 provider/manual/imported 译文候选。
|
||||||
|
///
|
||||||
|
/// 相同 source、上下文、译文、来源 release 和来源类型的重复写入必须幂等;
|
||||||
|
/// 已 trusted 的记录不得被普通候选静默覆盖。
|
||||||
|
async fn upsert_candidate(
|
||||||
|
&self,
|
||||||
|
draft: TranslationMemoryDraft,
|
||||||
|
) -> crate::Result<TranslationMemoryEntry>;
|
||||||
|
|
||||||
|
/// 按原始 source text 和上下文查询精确匹配。
|
||||||
|
///
|
||||||
|
/// 实现可以返回 source 归一化后但原文不同的辅助候选,但这类结果不能自动复用。
|
||||||
|
async fn find_matches(
|
||||||
|
&self,
|
||||||
|
source_text: &str,
|
||||||
|
source_context: &TranslationMemoryContext,
|
||||||
|
limit: usize,
|
||||||
|
) -> crate::Result<Vec<TranslationMemoryMatch>>;
|
||||||
|
|
||||||
|
/// 显式确认一条记录为 trusted。
|
||||||
|
async fn confirm(
|
||||||
|
&self,
|
||||||
|
record_id: &str,
|
||||||
|
reviewer: &str,
|
||||||
|
reason: Option<String>,
|
||||||
|
) -> crate::Result<TranslationMemoryEntry>;
|
||||||
|
|
||||||
|
/// 按稳定记录 ID 读取一条 TM 记录。
|
||||||
|
async fn find(&self, record_id: &str) -> crate::Result<TranslationMemoryEntry>;
|
||||||
|
|
||||||
|
/// 读取数据库和记录统计。
|
||||||
|
async fn summary(&self) -> crate::Result<TranslationMemorySummary>;
|
||||||
|
}
|
||||||
+653
-208
File diff suppressed because it is too large
Load Diff
@@ -17,9 +17,11 @@ fn parse_with_env(values: &[&str], env: &[(&str, &str)]) -> anyhow::Result<CliOp
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||||
.collect();
|
.collect();
|
||||||
parse_args_with_env(values.iter().map(|value| value.to_string()), move |key| {
|
parse_args_with_env(
|
||||||
map.get(key).cloned()
|
values.iter().map(|value| value.to_string()),
|
||||||
})
|
move |key| map.get(key).cloned(),
|
||||||
|
None,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -105,7 +107,7 @@ fn env_watch_daemon_only_affect_bare_run() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn env_values_do_not_break_status_and_reload_guard() {
|
fn env_values_do_not_break_status_and_reload_guard() {
|
||||||
// .env 提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。
|
// 配置文件/环境变量提供的代理/工具/输出目录不算"显式同步参数",status 应照常可用。
|
||||||
let options = parse_with_env(
|
let options = parse_with_env(
|
||||||
&["bat", "status"],
|
&["bat", "status"],
|
||||||
&[
|
&[
|
||||||
@@ -222,6 +224,148 @@ fn translation_worker_command_options_are_validated() {
|
|||||||
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
|
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn translation_memory_commands_parse_and_validate() {
|
||||||
|
let summary = parse(&["bat", "i18n", "memory", "summary"]).unwrap();
|
||||||
|
assert_eq!(summary.command, CliCommand::TranslationMemorySummary);
|
||||||
|
let query = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"tm",
|
||||||
|
"query",
|
||||||
|
"--tm-source-text",
|
||||||
|
"Hello",
|
||||||
|
"--tm-context-json",
|
||||||
|
r#"{"destination":"story.bundle"}"#,
|
||||||
|
"--limit",
|
||||||
|
"5",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(query.command, CliCommand::TranslationMemoryQuery);
|
||||||
|
assert_eq!(
|
||||||
|
query.translation_memory_source_text.as_deref(),
|
||||||
|
Some("Hello")
|
||||||
|
);
|
||||||
|
let confirm = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"confirm",
|
||||||
|
"--tm-record-id",
|
||||||
|
"tm-record",
|
||||||
|
"--tm-reviewer",
|
||||||
|
"reviewer",
|
||||||
|
"--tm-reason",
|
||||||
|
"accepted",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(confirm.command, CliCommand::TranslationMemoryConfirm);
|
||||||
|
assert!(parse(&["bat", "i18n", "memory", "query"]).is_err());
|
||||||
|
assert!(parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"confirm",
|
||||||
|
"--tm-record-id",
|
||||||
|
"tm-record"
|
||||||
|
])
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restart_reload_accept_translation_worker_startup_options() {
|
||||||
|
let restart = parse(&[
|
||||||
|
"bat",
|
||||||
|
"restart",
|
||||||
|
"--translation-provider",
|
||||||
|
"mock",
|
||||||
|
"--translation-memory-path",
|
||||||
|
"/tmp/tm.sqlite",
|
||||||
|
"--worker-concurrency",
|
||||||
|
"2",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(restart.command, CliCommand::Restart);
|
||||||
|
assert!(restart.translation_worker_option_explicit);
|
||||||
|
assert_eq!(
|
||||||
|
restart.translation_memory_path,
|
||||||
|
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||||
|
);
|
||||||
|
assert_eq!(restart.worker_concurrency, 2);
|
||||||
|
assert!(restart.config.auto_discover);
|
||||||
|
|
||||||
|
let reload = parse(&["bat", "reload", "--worker-id", "reload-worker"]).unwrap();
|
||||||
|
assert_eq!(reload.command, CliCommand::Reload);
|
||||||
|
assert!(reload.translation_worker_option_explicit);
|
||||||
|
assert_eq!(reload.worker_id.as_deref(), Some("reload-worker"));
|
||||||
|
assert!(reload.config.auto_discover);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn translation_memory_command_options_do_not_apply_to_other_commands() {
|
||||||
|
let error = parse(&["bat", "restart", "--tm-source-text", "Hello"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("查询/confirm"));
|
||||||
|
|
||||||
|
let error = parse(&["bat", "status", "--tm-record-id", "tm-record"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("查询/confirm"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn translation_memory_subcommands_reject_irrelevant_options() {
|
||||||
|
let error = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"summary",
|
||||||
|
"--tm-source-text",
|
||||||
|
"Hello",
|
||||||
|
])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("summary"));
|
||||||
|
let error = parse(&["bat", "i18n", "memory", "summary", "--limit", "5"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("summary"));
|
||||||
|
|
||||||
|
let error = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"query",
|
||||||
|
"--tm-source-text",
|
||||||
|
"Hello",
|
||||||
|
"--tm-record-id",
|
||||||
|
"tm-record",
|
||||||
|
])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("confirm 参数"));
|
||||||
|
let error = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"query",
|
||||||
|
"--tm-source-text",
|
||||||
|
"Hello",
|
||||||
|
"--offset",
|
||||||
|
"1",
|
||||||
|
])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("查询过滤"));
|
||||||
|
|
||||||
|
let error = parse(&[
|
||||||
|
"bat",
|
||||||
|
"i18n",
|
||||||
|
"memory",
|
||||||
|
"confirm",
|
||||||
|
"--tm-record-id",
|
||||||
|
"tm-record",
|
||||||
|
"--tm-reviewer",
|
||||||
|
"reviewer",
|
||||||
|
"--tm-context-json",
|
||||||
|
"{}",
|
||||||
|
])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("query 参数"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn translation_worker_env_defaults_apply() {
|
fn translation_worker_env_defaults_apply() {
|
||||||
let options = parse_with_env(
|
let options = parse_with_env(
|
||||||
@@ -229,6 +373,7 @@ fn translation_worker_env_defaults_apply() {
|
|||||||
&[
|
&[
|
||||||
("BAT_TRANSLATION_PROVIDER", "mock"),
|
("BAT_TRANSLATION_PROVIDER", "mock"),
|
||||||
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
|
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
|
||||||
|
("BAT_TRANSLATION_MEMORY_PATH", "/tmp/tm.sqlite"),
|
||||||
("BAT_TRANSLATION_CONCURRENCY", "16"),
|
("BAT_TRANSLATION_CONCURRENCY", "16"),
|
||||||
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
|
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
|
||||||
("BAT_TRANSLATION_LEASE_SECONDS", "120"),
|
("BAT_TRANSLATION_LEASE_SECONDS", "120"),
|
||||||
@@ -244,6 +389,10 @@ fn translation_worker_env_defaults_apply() {
|
|||||||
options.translation_fixture,
|
options.translation_fixture,
|
||||||
Some(PathBuf::from("/tmp/fixture.json"))
|
Some(PathBuf::from("/tmp/fixture.json"))
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
options.translation_memory_path,
|
||||||
|
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||||
|
);
|
||||||
assert_eq!(options.worker_concurrency, 16);
|
assert_eq!(options.worker_concurrency, 16);
|
||||||
assert_eq!(options.worker_max_attempts, 5);
|
assert_eq!(options.worker_max_attempts, 5);
|
||||||
assert_eq!(options.worker_lease_seconds, 120);
|
assert_eq!(options.worker_lease_seconds, 120);
|
||||||
@@ -610,6 +759,8 @@ fn translation_workbench_commands_read_update_and_clear_entries() {
|
|||||||
translated_text: None,
|
translated_text: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
translated_unix_seconds: None,
|
translated_unix_seconds: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
format: Some("plain".to_string()),
|
format: Some("plain".to_string()),
|
||||||
@@ -1030,46 +1181,72 @@ fn cli_download_concurrency_is_preserved_for_daemon_child() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_env_line_handles_quotes_and_rejects_bad_keys() {
|
fn config_file_is_applied_before_env_and_cli() {
|
||||||
assert_eq!(
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
parse_env_line("KEY=value"),
|
let config_path = temp.path().join(super::config_file::CONFIG_FILE_NAME);
|
||||||
Some(("KEY".to_string(), "value".to_string()))
|
std::fs::write(
|
||||||
);
|
&config_path,
|
||||||
assert_eq!(
|
r#"
|
||||||
parse_env_line("KEY=\"quoted value\""),
|
[runtime]
|
||||||
Some(("KEY".to_string(), "quoted value".to_string()))
|
state_dir = '/srv/state'
|
||||||
);
|
output_format = 'json'
|
||||||
assert_eq!(
|
|
||||||
parse_env_line("KEY='single'"),
|
|
||||||
Some(("KEY".to_string(), "single".to_string()))
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
parse_env_line("BAT_OUTPUT = ./x"),
|
|
||||||
Some(("BAT_OUTPUT".to_string(), "./x".to_string()))
|
|
||||||
);
|
|
||||||
assert_eq!(parse_env_line("no_equals_sign"), None);
|
|
||||||
assert_eq!(parse_env_line("1BAD=x"), None);
|
|
||||||
assert_eq!(parse_env_line("BAD KEY=x"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
[resource]
|
||||||
fn env_template_is_parseable_and_bootstrap_ready() {
|
output_root = '/srv/from-config'
|
||||||
// 模板每个非注释行必须可解析;无参启动所需的最小配置默认启用。
|
auto_discover = true
|
||||||
let mut keys = Vec::new();
|
|
||||||
for line in ENV_TEMPLATE.lines() {
|
[localized]
|
||||||
let line = line.trim();
|
output_root = '/srv/from-config-localized'
|
||||||
if line.is_empty() || line.starts_with('#') {
|
|
||||||
continue;
|
[network]
|
||||||
|
proxy = 'none'
|
||||||
|
download_concurrency = 12
|
||||||
|
|
||||||
|
[translation.worker]
|
||||||
|
translation_memory_path = '/srv/config-tm.sqlite'
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
let mut permissions = std::fs::metadata(&config_path).unwrap().permissions();
|
||||||
|
permissions.set_mode(0o600);
|
||||||
|
std::fs::set_permissions(&config_path, permissions).unwrap();
|
||||||
}
|
}
|
||||||
let (key, _) = parse_env_line(line).unwrap_or_else(|| panic!("模板行必须可解析:{line}"));
|
let config = super::config_file::load_from_binary_dir(temp.path())
|
||||||
keys.push(key);
|
.unwrap()
|
||||||
}
|
.unwrap();
|
||||||
assert!(keys.contains(&"BAT_OUTPUT".to_string()));
|
let options = parse_args_with_env(
|
||||||
assert!(keys.contains(&"BAT_LOCALIZED_OUTPUT".to_string()));
|
vec![
|
||||||
assert!(keys.contains(&"BAT_IMPORT_REPOSITORY".to_string()));
|
"bat".to_string(),
|
||||||
assert!(keys.contains(&"BAT_IMPORT_CAS_ROOT".to_string()));
|
"--daemon".to_string(),
|
||||||
assert!(keys.contains(&"BAT_IMPORT_RESOURCE_DB".to_string()));
|
"--output".to_string(),
|
||||||
assert!(keys.contains(&"BAT_AUTO_DISCOVER".to_string()));
|
"/cli/output".to_string(),
|
||||||
|
"--translation-memory-path".to_string(),
|
||||||
|
"/cli/tm.sqlite".to_string(),
|
||||||
|
],
|
||||||
|
|key| match key {
|
||||||
|
"BAT_OUTPUT" => Some("/env/output".to_string()),
|
||||||
|
"BAT_TRANSLATION_MEMORY_PATH" => Some("/env/tm.sqlite".to_string()),
|
||||||
|
_ => None,
|
||||||
|
},
|
||||||
|
Some(&config),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(options.state_dir, PathBuf::from("/srv/state"));
|
||||||
|
assert_eq!(options.output_format, OutputFormat::Json);
|
||||||
|
assert_eq!(options.config.output_root, PathBuf::from("/cli/output"));
|
||||||
|
assert_eq!(
|
||||||
|
options.config.localized_output_root,
|
||||||
|
PathBuf::from("/srv/from-config-localized")
|
||||||
|
);
|
||||||
|
assert_eq!(options.config.download_concurrency, 12);
|
||||||
|
assert_eq!(options.config.curl_proxy.mode(), &CurlProxyMode::Disabled);
|
||||||
|
assert_eq!(
|
||||||
|
options.translation_memory_path,
|
||||||
|
Some(PathBuf::from("/cli/tm.sqlite"))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1326,6 +1503,14 @@ fn rejects_proxy_with_unsupported_scheme() {
|
|||||||
assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok());
|
assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn proxy_parse_errors_redact_credentials() {
|
||||||
|
let error = parse(&["bat", "--proxy", "http://user:secret@"]).unwrap_err();
|
||||||
|
let message = error.to_string();
|
||||||
|
assert!(!message.contains("secret"));
|
||||||
|
assert!(message.contains("<redacted>"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() {
|
fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() {
|
||||||
let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap();
|
let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap();
|
||||||
@@ -1997,6 +2182,54 @@ fn daemon_child_args_preserve_sync_options() {
|
|||||||
assert!(args.contains(&"--no-banner".to_string()));
|
assert!(args.contains(&"--no-banner".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn daemon_child_args_preserve_translation_worker_configuration() {
|
||||||
|
let options = parse(&[
|
||||||
|
"bat",
|
||||||
|
"--daemon",
|
||||||
|
"--output",
|
||||||
|
"/tmp/daemon-output",
|
||||||
|
"--translation-provider",
|
||||||
|
"mock",
|
||||||
|
"--translation-fixture",
|
||||||
|
"/tmp/provider.json",
|
||||||
|
"--translation-memory-path",
|
||||||
|
"/tmp/tm.sqlite",
|
||||||
|
"--worker-concurrency",
|
||||||
|
"4",
|
||||||
|
"--worker-max-attempts",
|
||||||
|
"5",
|
||||||
|
"--worker-lease-seconds",
|
||||||
|
"60",
|
||||||
|
"--worker-retry-backoff-seconds",
|
||||||
|
"2",
|
||||||
|
"--worker-max-tasks",
|
||||||
|
"3",
|
||||||
|
"--worker-id",
|
||||||
|
"daemon-worker",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
let args = daemon_child_args(&options);
|
||||||
|
let mut child_args = vec!["bat".to_string()];
|
||||||
|
child_args.extend(args);
|
||||||
|
let child = parse_args_from(child_args).unwrap();
|
||||||
|
assert_eq!(child.translation_provider.as_deref(), Some("mock"));
|
||||||
|
assert_eq!(
|
||||||
|
child.translation_fixture,
|
||||||
|
Some(PathBuf::from("/tmp/provider.json"))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
child.translation_memory_path,
|
||||||
|
Some(PathBuf::from("/tmp/tm.sqlite"))
|
||||||
|
);
|
||||||
|
assert_eq!(child.worker_concurrency, 4);
|
||||||
|
assert_eq!(child.worker_max_attempts, 5);
|
||||||
|
assert_eq!(child.worker_lease_seconds, 60);
|
||||||
|
assert_eq!(child.worker_retry_backoff, Duration::from_secs(2));
|
||||||
|
assert_eq!(child.worker_max_tasks, Some(3));
|
||||||
|
assert_eq!(child.worker_id.as_deref(), Some("daemon-worker"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn curl_proxy_url_extracts_only_url_mode() {
|
fn curl_proxy_url_extracts_only_url_mode() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -2184,6 +2417,7 @@ fn test_task_context_with_config(base_config: OfficialUpdateConfig) -> DaemonTas
|
|||||||
registry: TaskRegistry::new(),
|
registry: TaskRegistry::new(),
|
||||||
queue,
|
queue,
|
||||||
base_config,
|
base_config,
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
sync_lock: Arc::new(Mutex::new(())),
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
restart_controller: test_restart_controller,
|
restart_controller: test_restart_controller,
|
||||||
}
|
}
|
||||||
@@ -2354,6 +2588,7 @@ fn dispatch_daemon_doctor_returns_report() {
|
|||||||
registry: TaskRegistry::new(),
|
registry: TaskRegistry::new(),
|
||||||
queue,
|
queue,
|
||||||
base_config,
|
base_config,
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
sync_lock: Arc::new(Mutex::new(())),
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
restart_controller: test_restart_controller,
|
restart_controller: test_restart_controller,
|
||||||
};
|
};
|
||||||
@@ -2374,6 +2609,112 @@ fn dispatch_daemon_doctor_returns_report() {
|
|||||||
assert!(checks.iter().any(|check| check["name"] == "daemon_rpc"));
|
assert!(checks.iter().any(|check| check["name"] == "daemon_rpc"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dispatch_translation_memory_summary_reports_missing_database_without_creating_it() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let output_root = temp.path().join("output");
|
||||||
|
let state_dir = temp.path().join("state");
|
||||||
|
let (queue, _rx) = mpsc::channel::<TaskJob>();
|
||||||
|
let context = DaemonTaskContext {
|
||||||
|
registry: TaskRegistry::new(),
|
||||||
|
queue,
|
||||||
|
base_config: OfficialUpdateConfig {
|
||||||
|
output_root: output_root.clone(),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
|
restart_controller: test_restart_controller,
|
||||||
|
};
|
||||||
|
|
||||||
|
let envelope = dispatch_rpc_method(
|
||||||
|
&rpc_request("translation.memory.summary", None),
|
||||||
|
&state_dir,
|
||||||
|
&new_daemon_control(),
|
||||||
|
&context,
|
||||||
|
"req-tm-summary-1".to_string(),
|
||||||
|
);
|
||||||
|
let value = serde_json::to_value(envelope).unwrap();
|
||||||
|
assert_eq!(value["ok"], true);
|
||||||
|
assert_eq!(value["data"]["available"], false);
|
||||||
|
assert_eq!(value["data"]["reason"], "database_missing");
|
||||||
|
assert!(!output_root.join("translation-memory.sqlite").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dispatch_translation_memory_summary_defaults_to_worker_config_path() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let output_root = temp.path().join("output");
|
||||||
|
let state_dir = temp.path().join("state");
|
||||||
|
let configured_tm_path = temp.path().join("configured-tm.sqlite");
|
||||||
|
let (queue, _rx) = mpsc::channel::<TaskJob>();
|
||||||
|
let context = DaemonTaskContext {
|
||||||
|
registry: TaskRegistry::new(),
|
||||||
|
queue,
|
||||||
|
base_config: OfficialUpdateConfig {
|
||||||
|
output_root,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
translation_worker_config: TranslationWorkerConfig {
|
||||||
|
translation_memory_path: Some(configured_tm_path.clone()),
|
||||||
|
..TranslationWorkerConfig::default()
|
||||||
|
},
|
||||||
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
|
restart_controller: test_restart_controller,
|
||||||
|
};
|
||||||
|
|
||||||
|
let envelope = dispatch_rpc_method(
|
||||||
|
&rpc_request("translation.memory.summary", None),
|
||||||
|
&state_dir,
|
||||||
|
&new_daemon_control(),
|
||||||
|
&context,
|
||||||
|
"req-tm-summary-configured-1".to_string(),
|
||||||
|
);
|
||||||
|
let value = serde_json::to_value(envelope).unwrap();
|
||||||
|
assert_eq!(value["ok"], true);
|
||||||
|
assert_eq!(value["data"]["available"], false);
|
||||||
|
assert_eq!(
|
||||||
|
value["data"]["path"].as_str(),
|
||||||
|
Some(configured_tm_path.to_str().unwrap())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dispatch_translation_memory_rejects_invalid_params_with_stable_error_code() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let context = test_task_context_with_config(OfficialUpdateConfig {
|
||||||
|
output_root: temp.path().join("output"),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
for (method, params) in [
|
||||||
|
("translation.memory.query", None),
|
||||||
|
(
|
||||||
|
"translation.memory.query",
|
||||||
|
Some(serde_json::json!({ "source_text": "Hello", "limit": "1" })),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"translation.memory.summary",
|
||||||
|
Some(serde_json::json!({ "translation_memory_path": 42 })),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"translation.memory.confirm",
|
||||||
|
Some(serde_json::json!({ "record_id": "tm-record", "reviewer": 42 })),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let envelope = dispatch_rpc_method(
|
||||||
|
&rpc_request(method, params),
|
||||||
|
temp.path(),
|
||||||
|
&new_daemon_control(),
|
||||||
|
&context,
|
||||||
|
format!("req-invalid-{method}"),
|
||||||
|
);
|
||||||
|
let value = serde_json::to_value(envelope).unwrap();
|
||||||
|
assert_eq!(value["ok"], false, "method={method}");
|
||||||
|
assert_eq!(value["error"]["code"], "BAT-ERR-700002", "method={method}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn dispatch_resource_sync_enqueues_task() {
|
fn dispatch_resource_sync_enqueues_task() {
|
||||||
let temp = tempfile::TempDir::new().unwrap();
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
@@ -2384,6 +2725,7 @@ fn dispatch_resource_sync_enqueues_task() {
|
|||||||
registry: TaskRegistry::new(),
|
registry: TaskRegistry::new(),
|
||||||
queue,
|
queue,
|
||||||
base_config: OfficialUpdateConfig::default(),
|
base_config: OfficialUpdateConfig::default(),
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
sync_lock: Arc::new(Mutex::new(())),
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
restart_controller: test_restart_controller,
|
restart_controller: test_restart_controller,
|
||||||
};
|
};
|
||||||
@@ -2445,6 +2787,7 @@ fn dispatch_resource_repair_enqueues_repair_task() {
|
|||||||
registry: TaskRegistry::new(),
|
registry: TaskRegistry::new(),
|
||||||
queue,
|
queue,
|
||||||
base_config,
|
base_config,
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
sync_lock: Arc::new(Mutex::new(())),
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
restart_controller: test_restart_controller,
|
restart_controller: test_restart_controller,
|
||||||
};
|
};
|
||||||
@@ -3737,6 +4080,7 @@ fn dispatch_catalog_refresh_enqueues_task() {
|
|||||||
registry: TaskRegistry::new(),
|
registry: TaskRegistry::new(),
|
||||||
queue,
|
queue,
|
||||||
base_config: OfficialUpdateConfig::default(),
|
base_config: OfficialUpdateConfig::default(),
|
||||||
|
translation_worker_config: TranslationWorkerConfig::default(),
|
||||||
sync_lock: Arc::new(Mutex::new(())),
|
sync_lock: Arc::new(Mutex::new(())),
|
||||||
restart_controller: test_restart_controller,
|
restart_controller: test_restart_controller,
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -111,6 +111,13 @@ impl HumanReport for bat_infrastructure::TranslationWorkerReport {
|
|||||||
print_field("失败任务", self.failed_count);
|
print_field("失败任务", self.failed_count);
|
||||||
print_field("已安排重试", self.retry_scheduled_count);
|
print_field("已安排重试", self.retry_scheduled_count);
|
||||||
print_field("剩余任务", self.remaining_count);
|
print_field("剩余任务", self.remaining_count);
|
||||||
|
print_path_field("Translation Memory", &self.translation_memory_path);
|
||||||
|
print_field("TM 可用", format_bool(self.translation_memory_available));
|
||||||
|
print_field("TM 命中 TextUnit", self.translation_memory_hit_count);
|
||||||
|
print_field("Provider TextUnit", self.provider_unit_count);
|
||||||
|
for failure in &self.translation_memory_failures {
|
||||||
|
println!(" - TM: {failure}");
|
||||||
|
}
|
||||||
for failure in &self.failures {
|
for failure in &self.failures {
|
||||||
println!(
|
println!(
|
||||||
" - {} [{}] retryable={} {}",
|
" - {} [{}] retryable={} {}",
|
||||||
|
|||||||
@@ -519,6 +519,8 @@ pub(super) struct DaemonTaskContext {
|
|||||||
pub(super) registry: TaskRegistry,
|
pub(super) registry: TaskRegistry,
|
||||||
pub(super) queue: mpsc::Sender<TaskJob>,
|
pub(super) queue: mpsc::Sender<TaskJob>,
|
||||||
pub(super) base_config: OfficialUpdateConfig,
|
pub(super) base_config: OfficialUpdateConfig,
|
||||||
|
/// daemon 中未显式传入参数的 translation worker 默认配置。
|
||||||
|
pub(super) translation_worker_config: TranslationWorkerConfig,
|
||||||
/// 串行化会读取或修改已发布资源状态的 daemon 操作。
|
/// 串行化会读取或修改已发布资源状态的 daemon 操作。
|
||||||
pub(super) sync_lock: Arc<Mutex<()>>,
|
pub(super) sync_lock: Arc<Mutex<()>>,
|
||||||
pub(super) restart_controller: DaemonRestartController,
|
pub(super) restart_controller: DaemonRestartController,
|
||||||
@@ -562,11 +564,15 @@ pub(super) fn run_task_worker(
|
|||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()?;
|
.build()?;
|
||||||
|
let cancel_check = Arc::clone(&cancel);
|
||||||
runtime
|
runtime
|
||||||
.block_on(bat_infrastructure::run_translation_worker_at(
|
.block_on(
|
||||||
|
bat_infrastructure::run_translation_worker_at_with_cancellation(
|
||||||
&resource_root,
|
&resource_root,
|
||||||
worker_config,
|
worker_config,
|
||||||
))
|
Arc::new(move || cancel_check.load(Ordering::Relaxed)),
|
||||||
|
),
|
||||||
|
)
|
||||||
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
|
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
@@ -608,7 +614,8 @@ pub(super) fn run_task_worker(
|
|||||||
record.result = Some(report);
|
record.result = Some(report);
|
||||||
}),
|
}),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let cancelled = cancel.load(Ordering::Relaxed);
|
let cancelled =
|
||||||
|
cancel.load(Ordering::Relaxed) || daemon_control_stop_requested(Some(&control));
|
||||||
// 下载失败携带类型化 DownloadError(含准确网络域码);其余归 internal。
|
// 下载失败携带类型化 DownloadError(含准确网络域码);其余归 internal。
|
||||||
let code = error
|
let code = error
|
||||||
.downcast_ref::<bat_infrastructure::DownloadError>()
|
.downcast_ref::<bat_infrastructure::DownloadError>()
|
||||||
|
|||||||
@@ -71,28 +71,24 @@ pub(super) fn print_startup_banner() {
|
|||||||
eprintln!("{STARTUP_BANNER}");
|
eprintln!("{STARTUP_BANNER}");
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn print_env_template_created(path: &Path) {
|
pub(super) fn print_config_template_created(path: &Path) {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"已生成配置模板 {}(编辑其中的 BAT_* 配置后,直接运行 `bat` 即可按 .env 启动)",
|
"已生成配置模板 {}(编辑 `config.toml`,`config.toml.example` 不会被程序自动读取)",
|
||||||
path.display()
|
path.display()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn print_env_template_warning(path: &Path, error: impl std::fmt::Display) {
|
pub(super) fn print_config_template_warning(path: &Path, error: impl std::fmt::Display) {
|
||||||
eprintln!("警告:生成 .env 配置模板失败 {}:{error}", path.display());
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn print_env_read_warning(path: &Path, error: impl std::fmt::Display) {
|
|
||||||
eprintln!("警告:读取 .env 失败 {}:{error}", path.display());
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn print_env_parse_warning(line_number: usize, raw_line: &str) {
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"警告:.env 第 {} 行无法解析,已忽略:{raw_line}",
|
"警告:生成 config.toml.example 模板失败 {}:{error}",
|
||||||
line_number
|
path.display()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn print_deprecated_env_file_warning() {
|
||||||
|
eprintln!("警告:BAT_SKIP_ENV_FILE 已废弃且不再影响启动,已忽略");
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(super) struct ProgressLogger {
|
pub(super) struct ProgressLogger {
|
||||||
enabled: bool,
|
enabled: bool,
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
|
use super::report_output::print_json_value;
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use bat_core::domain::TranslationMemoryContext;
|
||||||
|
use bat_core::repositories::TranslationMemoryRepository;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct TranslationTaskResultUpdateParam {
|
struct TranslationTaskResultUpdateParam {
|
||||||
@@ -332,6 +336,8 @@ fn build_manual_translation_results(
|
|||||||
unit_id: unit_id.to_string(),
|
unit_id: unit_id.to_string(),
|
||||||
source_text: param.source_text.clone(),
|
source_text: param.source_text.clone(),
|
||||||
translated_text: param.translated_text.clone(),
|
translated_text: param.translated_text.clone(),
|
||||||
|
source_kind: bat_infrastructure::TranslationTaskResultSourceKind::Manual,
|
||||||
|
translation_memory_record_id: None,
|
||||||
provider: provider.to_string(),
|
provider: provider.to_string(),
|
||||||
provider_run_id: provider_run_id.to_string(),
|
provider_run_id: provider_run_id.to_string(),
|
||||||
translated_unix_seconds,
|
translated_unix_seconds,
|
||||||
@@ -355,3 +361,241 @@ pub(super) fn translation_task_query_json(query: &OfficialTextUnitTaskQuery) ->
|
|||||||
"has_failure_reason": query.has_failure_reason,
|
"has_failure_reason": query.has_failure_reason,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn run_translation_memory_command(options: &CliOptions) -> anyhow::Result<()> {
|
||||||
|
let method = match options.command {
|
||||||
|
CliCommand::TranslationMemorySummary => RPC_METHOD_TRANSLATION_MEMORY_SUMMARY,
|
||||||
|
CliCommand::TranslationMemoryQuery => RPC_METHOD_TRANSLATION_MEMORY_QUERY,
|
||||||
|
CliCommand::TranslationMemoryConfirm => RPC_METHOD_TRANSLATION_MEMORY_CONFIRM,
|
||||||
|
_ => return Err(anyhow::anyhow!("不是 Translation Memory 命令")),
|
||||||
|
};
|
||||||
|
if daemon_rpc_available(&options.state_dir)
|
||||||
|
&& options.resource_root.is_none()
|
||||||
|
&& !options.output_explicit
|
||||||
|
{
|
||||||
|
let _control_lock = DaemonControlLock::acquire(&options.state_dir)?;
|
||||||
|
let report = daemon_rpc_call(
|
||||||
|
&options.state_dir,
|
||||||
|
method,
|
||||||
|
translation_memory_cli_params(options)?,
|
||||||
|
)?;
|
||||||
|
print_json_value(options.output_format, &report)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let path = translation_memory_cli_path(options)?;
|
||||||
|
let report = match options.command {
|
||||||
|
CliCommand::TranslationMemorySummary => build_translation_memory_summary_report(&path)?,
|
||||||
|
CliCommand::TranslationMemoryQuery => {
|
||||||
|
let source_text = options
|
||||||
|
.translation_memory_source_text
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
|
||||||
|
let context = parse_translation_memory_context(
|
||||||
|
options.translation_memory_context_json.as_deref(),
|
||||||
|
)?;
|
||||||
|
build_translation_memory_query_report(
|
||||||
|
&path,
|
||||||
|
source_text,
|
||||||
|
&context,
|
||||||
|
options.query_limit,
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
CliCommand::TranslationMemoryConfirm => {
|
||||||
|
let record_id = options
|
||||||
|
.translation_memory_record_id
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
|
||||||
|
let reviewer = options
|
||||||
|
.translation_memory_reviewer
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
|
||||||
|
build_translation_memory_confirm_report(
|
||||||
|
&path,
|
||||||
|
record_id,
|
||||||
|
reviewer,
|
||||||
|
options.translation_memory_reason.clone(),
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
print_json_value(options.output_format, &report)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn translation_memory_cli_path(options: &CliOptions) -> anyhow::Result<std::path::PathBuf> {
|
||||||
|
if let Some(path) = options.translation_memory_path.as_ref() {
|
||||||
|
return lexical_absolute(path).map_err(anyhow::Error::msg);
|
||||||
|
}
|
||||||
|
let resource_root = options
|
||||||
|
.resource_root
|
||||||
|
.as_deref()
|
||||||
|
.map(lexical_absolute)
|
||||||
|
.transpose()
|
||||||
|
.map_err(anyhow::Error::msg)?
|
||||||
|
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
||||||
|
Ok(bat_infrastructure::translation_memory_repository_path(
|
||||||
|
&resource_root,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn translation_memory_cli_params(
|
||||||
|
options: &CliOptions,
|
||||||
|
) -> anyhow::Result<Option<serde_json::Value>> {
|
||||||
|
let mut params = serde_json::Map::new();
|
||||||
|
if let Some(path) = options.translation_memory_path.as_ref() {
|
||||||
|
params.insert(
|
||||||
|
"translation_memory_path".to_string(),
|
||||||
|
serde_json::json!(path),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
match options.command {
|
||||||
|
CliCommand::TranslationMemorySummary => {}
|
||||||
|
CliCommand::TranslationMemoryQuery => {
|
||||||
|
let source_text = options
|
||||||
|
.translation_memory_source_text
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?;
|
||||||
|
let context = parse_translation_memory_context(
|
||||||
|
options.translation_memory_context_json.as_deref(),
|
||||||
|
)?;
|
||||||
|
params.insert("source_text".to_string(), serde_json::json!(source_text));
|
||||||
|
params.insert("source_context".to_string(), serde_json::json!(context));
|
||||||
|
params.insert("limit".to_string(), serde_json::json!(options.query_limit));
|
||||||
|
}
|
||||||
|
CliCommand::TranslationMemoryConfirm => {
|
||||||
|
let record_id = options
|
||||||
|
.translation_memory_record_id
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?;
|
||||||
|
let reviewer = options
|
||||||
|
.translation_memory_reviewer
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?;
|
||||||
|
params.insert("record_id".to_string(), serde_json::json!(record_id));
|
||||||
|
params.insert("reviewer".to_string(), serde_json::json!(reviewer));
|
||||||
|
if let Some(reason) = options.translation_memory_reason.as_deref() {
|
||||||
|
params.insert("reason".to_string(), serde_json::json!(reason));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
Ok(Some(serde_json::Value::Object(params)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_translation_memory_summary_report(
|
||||||
|
path: &std::path::Path,
|
||||||
|
) -> anyhow::Result<serde_json::Value> {
|
||||||
|
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||||
|
return Ok(serde_json::json!({
|
||||||
|
"available": false,
|
||||||
|
"path": path,
|
||||||
|
"reason": "database_missing",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()?;
|
||||||
|
let summary = runtime.block_on(async {
|
||||||
|
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
|
repository
|
||||||
|
.summary()
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||||
|
})?;
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"available": true,
|
||||||
|
"path": path,
|
||||||
|
"schema_version": summary.schema_version,
|
||||||
|
"summary": summary,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_translation_memory_query_report(
|
||||||
|
path: &std::path::Path,
|
||||||
|
source_text: &str,
|
||||||
|
source_context: &TranslationMemoryContext,
|
||||||
|
limit: usize,
|
||||||
|
) -> anyhow::Result<serde_json::Value> {
|
||||||
|
if source_text.trim().is_empty() {
|
||||||
|
return Err(anyhow::anyhow!("TM query 的 source_text 不能为空"));
|
||||||
|
}
|
||||||
|
if !(1..=1000).contains(&limit) {
|
||||||
|
return Err(anyhow::anyhow!("TM query 的 limit 必须在 1..=1000 范围内"));
|
||||||
|
}
|
||||||
|
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||||
|
return Ok(serde_json::json!({
|
||||||
|
"available": false,
|
||||||
|
"path": path,
|
||||||
|
"source_text": source_text,
|
||||||
|
"source_context": source_context,
|
||||||
|
"matches": [],
|
||||||
|
"reason": "database_missing",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()?;
|
||||||
|
let matches = runtime.block_on(async {
|
||||||
|
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
|
repository
|
||||||
|
.find_matches(source_text, source_context, limit)
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||||
|
})?;
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"available": true,
|
||||||
|
"path": path,
|
||||||
|
"source_text": source_text,
|
||||||
|
"source_context": source_context,
|
||||||
|
"matches": matches,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_translation_memory_confirm_report(
|
||||||
|
path: &std::path::Path,
|
||||||
|
record_id: &str,
|
||||||
|
reviewer: &str,
|
||||||
|
reason: Option<String>,
|
||||||
|
) -> anyhow::Result<serde_json::Value> {
|
||||||
|
if record_id.trim().is_empty() || reviewer.trim().is_empty() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"TM confirm 必须指定非空 record_id 和 reviewer"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"Translation Memory 数据库不存在:{}",
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()?;
|
||||||
|
let entry = runtime.block_on(async {
|
||||||
|
let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path)
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
|
repository
|
||||||
|
.confirm(record_id, reviewer, reason)
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))
|
||||||
|
})?;
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"available": true,
|
||||||
|
"path": path,
|
||||||
|
"entry": entry,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_translation_memory_context(
|
||||||
|
value: Option<&str>,
|
||||||
|
) -> anyhow::Result<TranslationMemoryContext> {
|
||||||
|
let Some(value) = value else {
|
||||||
|
return Ok(BTreeMap::new());
|
||||||
|
};
|
||||||
|
serde_json::from_str::<TranslationMemoryContext>(value)
|
||||||
|
.map_err(|error| anyhow::anyhow!("--tm-context-json 必须是 JSON object:{error}"))
|
||||||
|
}
|
||||||
|
|||||||
@@ -261,26 +261,10 @@ pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()>
|
|||||||
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
|
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
|
||||||
.transpose()?
|
.transpose()?
|
||||||
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
||||||
let provider = options
|
let config = super::translation_worker_config_from_options(
|
||||||
.translation_provider
|
options,
|
||||||
.as_deref()
|
&format!("bat-worker-{}", std::process::id()),
|
||||||
.unwrap_or(TranslationProviderKind::Mock.as_str());
|
)?;
|
||||||
let provider = TranslationProviderKind::parse(provider)
|
|
||||||
.ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效:{provider}"))?;
|
|
||||||
let config = TranslationWorkerConfig {
|
|
||||||
provider,
|
|
||||||
fixture_path: options.translation_fixture.clone(),
|
|
||||||
concurrency: options.worker_concurrency,
|
|
||||||
max_attempts: options.worker_max_attempts,
|
|
||||||
lease_seconds: options.worker_lease_seconds,
|
|
||||||
retry_backoff: options.worker_retry_backoff,
|
|
||||||
max_tasks: options.worker_max_tasks,
|
|
||||||
worker_id: options
|
|
||||||
.worker_id
|
|
||||||
.clone()
|
|
||||||
.unwrap_or_else(|| format!("bat-worker-{}", std::process::id())),
|
|
||||||
};
|
|
||||||
config.validate()?;
|
|
||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()?;
|
.build()?;
|
||||||
|
|||||||
+18
-12
@@ -29,6 +29,7 @@ pub mod patch_ops;
|
|||||||
pub mod path_security;
|
pub mod path_security;
|
||||||
pub mod release_flow;
|
pub mod release_flow;
|
||||||
pub mod resources;
|
pub mod resources;
|
||||||
|
pub mod translation_memory;
|
||||||
pub mod translation_tasks;
|
pub mod translation_tasks;
|
||||||
pub mod translation_worker;
|
pub mod translation_worker;
|
||||||
pub mod translation_workflow;
|
pub mod translation_workflow;
|
||||||
@@ -140,24 +141,29 @@ pub use path_security::{
|
|||||||
};
|
};
|
||||||
pub use release_flow::ReleaseFlowStatusCode;
|
pub use release_flow::ReleaseFlowStatusCode;
|
||||||
pub use resources::{InMemoryResourceRepository, SqliteResourceRepository};
|
pub use resources::{InMemoryResourceRepository, SqliteResourceRepository};
|
||||||
|
pub use translation_memory::{
|
||||||
|
translation_memory_context, translation_memory_repository_path,
|
||||||
|
SqliteTranslationMemoryRepository, TRANSLATION_MEMORY_REPOSITORY_FILE,
|
||||||
|
TRANSLATION_MEMORY_SCHEMA_COMPONENT, TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||||||
|
};
|
||||||
pub use translation_tasks::{
|
pub use translation_tasks::{
|
||||||
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
|
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
|
||||||
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
|
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
|
||||||
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
|
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
|
||||||
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus,
|
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskResultSourceKind,
|
||||||
TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus,
|
TranslationTaskStatus, TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit,
|
||||||
TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
|
TranslationUnitStatus, TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION,
|
||||||
TRANSLATION_TASK_SCHEMA_VERSION,
|
TRANSLATION_TASK_REPOSITORY_FILE, TRANSLATION_TASK_SCHEMA_VERSION,
|
||||||
};
|
};
|
||||||
pub use translation_worker::{
|
pub use translation_worker::{
|
||||||
run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider,
|
run_translation_worker_at, run_translation_worker_at_with_cancellation,
|
||||||
MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass,
|
run_translation_worker_with_provider, CrowdinProvider, MockTranslationProvider,
|
||||||
TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse,
|
TranslationProvider, TranslationProviderFailureClass, TranslationProviderKind,
|
||||||
TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig,
|
TranslationProviderRequest, TranslationProviderResponse, TranslationProviderUnit,
|
||||||
TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY,
|
TranslationProviderUnitResult, TranslationWorkerConfig, TranslationWorkerFailure,
|
||||||
DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS,
|
TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS,
|
||||||
DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY,
|
DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
||||||
MOCK_TRANSLATION_FIXTURE_VERSION,
|
MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, MOCK_TRANSLATION_FIXTURE_VERSION,
|
||||||
};
|
};
|
||||||
pub use translation_workflow::{
|
pub use translation_workflow::{
|
||||||
completed_worker_translation_workbench, export_completed_worker_translation_workbench,
|
completed_worker_translation_workbench, export_completed_worker_translation_workbench,
|
||||||
|
|||||||
@@ -91,6 +91,12 @@ pub struct LocalizedPatchOperationMetadata {
|
|||||||
/// Provider run ID that produced the text, if applicable.
|
/// Provider run ID that produced the text, if applicable.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub provider_run_id: Option<String>,
|
pub provider_run_id: Option<String>,
|
||||||
|
/// Source kind of the translation result.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_source_kind: Option<String>,
|
||||||
|
/// Trusted Translation Memory record used for the text, if applicable.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_memory_record_id: Option<String>,
|
||||||
/// Review state used by the publication input.
|
/// Review state used by the publication input.
|
||||||
pub review_status: String,
|
pub review_status: String,
|
||||||
}
|
}
|
||||||
@@ -331,6 +337,12 @@ pub struct LocalizedPatchOperation {
|
|||||||
/// Provider run ID that produced the text, if applicable.
|
/// Provider run ID that produced the text, if applicable.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub provider_run_id: Option<String>,
|
pub provider_run_id: Option<String>,
|
||||||
|
/// Source kind of the translation result.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_source_kind: Option<String>,
|
||||||
|
/// Trusted Translation Memory record used for the text, if applicable.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_memory_record_id: Option<String>,
|
||||||
/// Review state used by the publication input.
|
/// Review state used by the publication input.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub review_status: Option<String>,
|
pub review_status: Option<String>,
|
||||||
@@ -941,6 +953,8 @@ impl LocalizedPatchOperation {
|
|||||||
source_text_blake3: None,
|
source_text_blake3: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
},
|
},
|
||||||
metadata,
|
metadata,
|
||||||
@@ -966,6 +980,8 @@ impl LocalizedPatchOperation {
|
|||||||
source_text_blake3: None,
|
source_text_blake3: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
},
|
},
|
||||||
metadata,
|
metadata,
|
||||||
@@ -990,6 +1006,8 @@ impl LocalizedPatchOperation {
|
|||||||
source_text_blake3: None,
|
source_text_blake3: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
},
|
},
|
||||||
metadata,
|
metadata,
|
||||||
@@ -1005,6 +1023,8 @@ impl LocalizedPatchOperation {
|
|||||||
operation.source_text_blake3 = Some(metadata.source_text_blake3.clone());
|
operation.source_text_blake3 = Some(metadata.source_text_blake3.clone());
|
||||||
operation.translation_provider = metadata.translation_provider.clone();
|
operation.translation_provider = metadata.translation_provider.clone();
|
||||||
operation.provider_run_id = metadata.provider_run_id.clone();
|
operation.provider_run_id = metadata.provider_run_id.clone();
|
||||||
|
operation.translation_source_kind = metadata.translation_source_kind.clone();
|
||||||
|
operation.translation_memory_record_id = metadata.translation_memory_record_id.clone();
|
||||||
operation.review_status = Some(metadata.review_status.clone());
|
operation.review_status = Some(metadata.review_status.clone());
|
||||||
}
|
}
|
||||||
operation
|
operation
|
||||||
@@ -1029,6 +1049,8 @@ impl Default for LocalizedPatchOperation {
|
|||||||
source_text_blake3: None,
|
source_text_blake3: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1401,6 +1423,8 @@ mod tests {
|
|||||||
source_text_blake3: Some(blake3::hash(source).to_hex().to_string()),
|
source_text_blake3: Some(blake3::hash(source).to_hex().to_string()),
|
||||||
translation_provider: Some("mock".to_string()),
|
translation_provider: Some("mock".to_string()),
|
||||||
provider_run_id: Some("mock:unit-1:attempt-1".to_string()),
|
provider_run_id: Some("mock:unit-1:attempt-1".to_string()),
|
||||||
|
translation_source_kind: Some("provider".to_string()),
|
||||||
|
translation_memory_record_id: None,
|
||||||
review_status: Some("provider_completed".to_string()),
|
review_status: Some("provider_completed".to_string()),
|
||||||
}],
|
}],
|
||||||
}],
|
}],
|
||||||
|
|||||||
@@ -0,0 +1,859 @@
|
|||||||
|
//! 跨 official release 的 Translation Memory SQLite 仓储。
|
||||||
|
|
||||||
|
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use bat_core::domain::{
|
||||||
|
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||||
|
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||||||
|
TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||||||
|
};
|
||||||
|
use bat_core::repositories::TranslationMemoryRepository;
|
||||||
|
use bat_core::{Error, Result};
|
||||||
|
use serde::de::DeserializeOwned;
|
||||||
|
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
|
||||||
|
use sqlx::{Row, SqlitePool};
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::str::FromStr;
|
||||||
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
/// TM SQLite schema 版本。
|
||||||
|
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1;
|
||||||
|
/// TM schema migration component。
|
||||||
|
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
|
||||||
|
/// 默认 TM 数据库文件名。
|
||||||
|
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
|
||||||
|
|
||||||
|
/// SQLite-backed Translation Memory 仓储。
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SqliteTranslationMemoryRepository {
|
||||||
|
pub(crate) pool: SqlitePool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SqliteTranslationMemoryRepository {
|
||||||
|
/// 创建或打开 TM 数据库并执行迁移。
|
||||||
|
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
|
||||||
|
Self::open_with(path.as_ref(), true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 只打开已有 TM 数据库,不创建新文件。
|
||||||
|
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||||
|
Self::open_with(path.as_ref(), false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 根据 active release 根目录计算默认的跨 release TM 路径。
|
||||||
|
///
|
||||||
|
/// 正式 release 根目录形如 `<output>/versions/<id>`,因此默认结果为
|
||||||
|
/// `<output>/translation-memory.sqlite`,不会写入已发布版本目录。
|
||||||
|
pub fn repository_path(resource_root: &Path) -> PathBuf {
|
||||||
|
if resource_root
|
||||||
|
.parent()
|
||||||
|
.and_then(Path::file_name)
|
||||||
|
.is_some_and(|name| name == "versions")
|
||||||
|
{
|
||||||
|
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
|
||||||
|
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
|
||||||
|
let absolute = bat_infrastructure_absolute(path)?;
|
||||||
|
let parent = absolute.parent().ok_or_else(|| {
|
||||||
|
Error::InvalidArgument(format!("TM 数据库缺少父目录:{}", absolute.display()))
|
||||||
|
})?;
|
||||||
|
ensure_safe_tm_parent(parent)?;
|
||||||
|
if create_if_missing {
|
||||||
|
tokio::fs::create_dir_all(parent).await?;
|
||||||
|
ensure_safe_tm_parent(parent)?;
|
||||||
|
}
|
||||||
|
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(Error::InvalidArgument(format!(
|
||||||
|
"TM 数据库不能是 symlink:{}",
|
||||||
|
absolute.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if !metadata.is_file() {
|
||||||
|
return Err(Error::InvalidArgument(format!(
|
||||||
|
"TM 数据库不是普通文件:{}",
|
||||||
|
absolute.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
} else if !create_if_missing {
|
||||||
|
return Err(Error::NotFound(absolute.display().to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
|
||||||
|
.map_err(|error| Error::Other(error.into()))?
|
||||||
|
.create_if_missing(create_if_missing)
|
||||||
|
.journal_mode(SqliteJournalMode::Wal)
|
||||||
|
.busy_timeout(Duration::from_secs(30));
|
||||||
|
let pool = SqlitePoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_with(options)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
|
||||||
|
.map_err(Error::InvalidArgument)?;
|
||||||
|
let repository = Self { pool };
|
||||||
|
repository.init_schema().await?;
|
||||||
|
Ok(repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn init_schema(&self) -> Result<()> {
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
component TEXT PRIMARY KEY NOT NULL,
|
||||||
|
version INTEGER NOT NULL CHECK(version >= 1)
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
CREATE TABLE IF NOT EXISTS translation_memory (
|
||||||
|
record_id TEXT PRIMARY KEY NOT NULL,
|
||||||
|
source_text TEXT NOT NULL,
|
||||||
|
source_hash TEXT NOT NULL,
|
||||||
|
normalized_source_text TEXT NOT NULL,
|
||||||
|
source_context_json TEXT NOT NULL,
|
||||||
|
source_context_hash TEXT NOT NULL,
|
||||||
|
translated_text TEXT NOT NULL,
|
||||||
|
translation_source_kind TEXT NOT NULL,
|
||||||
|
trust_status TEXT NOT NULL,
|
||||||
|
official_release_id TEXT NOT NULL,
|
||||||
|
source_trace_json TEXT NOT NULL,
|
||||||
|
provider TEXT,
|
||||||
|
provider_run_id TEXT,
|
||||||
|
created_unix_seconds INTEGER NOT NULL,
|
||||||
|
updated_unix_seconds INTEGER NOT NULL,
|
||||||
|
trusted_unix_seconds INTEGER,
|
||||||
|
trusted_by TEXT,
|
||||||
|
trusted_reason TEXT,
|
||||||
|
supersedes_record_id TEXT,
|
||||||
|
superseded_by_record_id TEXT,
|
||||||
|
CHECK (length(source_text) > 0),
|
||||||
|
CHECK (length(source_hash) > 0),
|
||||||
|
CHECK (length(source_context_hash) > 0),
|
||||||
|
CHECK (length(official_release_id) > 0),
|
||||||
|
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
|
||||||
|
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
|
||||||
|
ON translation_memory(source_hash)",
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
|
||||||
|
ON translation_memory(normalized_source_text)",
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
|
||||||
|
ON translation_memory(source_hash, source_context_hash)",
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
|
||||||
|
let current: Option<i64> =
|
||||||
|
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
|
||||||
|
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||||
|
.fetch_optional(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
|
||||||
|
return Err(Error::InvalidArgument(format!(
|
||||||
|
"不支持的 Translation Memory schema 版本:{}",
|
||||||
|
current.unwrap_or_default()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO schema_migrations(component, version)
|
||||||
|
VALUES (?1, ?2)
|
||||||
|
ON CONFLICT(component) DO UPDATE SET version = excluded.version
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||||
|
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||||
|
source_context_json, source_context_hash, translated_text,
|
||||||
|
translation_source_kind, trust_status, official_release_id,
|
||||||
|
source_trace_json, provider, provider_run_id,
|
||||||
|
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||||
|
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||||
|
FROM translation_memory
|
||||||
|
WHERE record_id = ?1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(record_id)
|
||||||
|
.fetch_optional(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
row.map(row_to_entry).transpose()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
|
||||||
|
async fn upsert_candidate(
|
||||||
|
&self,
|
||||||
|
draft: TranslationMemoryDraft,
|
||||||
|
) -> Result<TranslationMemoryEntry> {
|
||||||
|
validate_draft(&draft)?;
|
||||||
|
let entry = entry_from_draft(draft)?;
|
||||||
|
if let Some(existing) = self.find_optional(&entry.record_id).await? {
|
||||||
|
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||||
|
return Ok(existing);
|
||||||
|
}
|
||||||
|
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||||
|
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE translation_memory
|
||||||
|
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
|
||||||
|
updated_unix_seconds = ?5
|
||||||
|
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(&entry.record_id)
|
||||||
|
.bind(source_trace_json)
|
||||||
|
.bind(&entry.provider)
|
||||||
|
.bind(&entry.provider_run_id)
|
||||||
|
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
return self.find(&entry.record_id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_context_json = serde_json::to_string(&entry.source_context)
|
||||||
|
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||||
|
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||||
|
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||||
|
let result = sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO translation_memory (
|
||||||
|
record_id, source_text, source_hash, normalized_source_text,
|
||||||
|
source_context_json, source_context_hash, translated_text,
|
||||||
|
translation_source_kind, trust_status, official_release_id,
|
||||||
|
source_trace_json, provider, provider_run_id,
|
||||||
|
created_unix_seconds, updated_unix_seconds,
|
||||||
|
trusted_unix_seconds, trusted_by, trusted_reason,
|
||||||
|
supersedes_record_id, superseded_by_record_id
|
||||||
|
)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
|
||||||
|
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
|
||||||
|
ON CONFLICT(record_id) DO UPDATE SET
|
||||||
|
source_trace_json = excluded.source_trace_json,
|
||||||
|
provider = excluded.provider,
|
||||||
|
provider_run_id = excluded.provider_run_id,
|
||||||
|
updated_unix_seconds = excluded.updated_unix_seconds
|
||||||
|
WHERE translation_memory.trust_status = 'candidate'
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(&entry.record_id)
|
||||||
|
.bind(&entry.source_text)
|
||||||
|
.bind(&entry.source_hash)
|
||||||
|
.bind(&entry.normalized_source_text)
|
||||||
|
.bind(source_context_json)
|
||||||
|
.bind(&entry.source_context_hash)
|
||||||
|
.bind(&entry.translated_text)
|
||||||
|
.bind(entry.translation_source_kind.as_str())
|
||||||
|
.bind(entry.trust_status.as_str())
|
||||||
|
.bind(&entry.official_release_id)
|
||||||
|
.bind(source_trace_json)
|
||||||
|
.bind(&entry.provider)
|
||||||
|
.bind(&entry.provider_run_id)
|
||||||
|
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
|
||||||
|
.bind(
|
||||||
|
entry
|
||||||
|
.trusted_unix_seconds
|
||||||
|
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
|
||||||
|
)
|
||||||
|
.bind(&entry.trusted_by)
|
||||||
|
.bind(&entry.trusted_reason)
|
||||||
|
.bind(&entry.supersedes_record_id)
|
||||||
|
.bind(&entry.superseded_by_record_id)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
if result.rows_affected() == 0 {
|
||||||
|
return self.find(&entry.record_id).await;
|
||||||
|
}
|
||||||
|
Ok(entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn find_matches(
|
||||||
|
&self,
|
||||||
|
source_text: &str,
|
||||||
|
source_context: &TranslationMemoryContext,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<TranslationMemoryMatch>> {
|
||||||
|
if source_text.is_empty() {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory 查询 source_text 不能为空".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if limit == 0 {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory 查询 limit 必须大于 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let source_hash = hash_text(source_text);
|
||||||
|
let normalized_source_text = normalize_source_text(source_text);
|
||||||
|
let rows = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||||
|
source_context_json, source_context_hash, translated_text,
|
||||||
|
translation_source_kind, trust_status, official_release_id,
|
||||||
|
source_trace_json, provider, provider_run_id,
|
||||||
|
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||||
|
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||||
|
FROM translation_memory
|
||||||
|
WHERE source_hash = ?1 OR normalized_source_text = ?2
|
||||||
|
ORDER BY updated_unix_seconds DESC, record_id ASC
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(source_hash)
|
||||||
|
.bind(&normalized_source_text)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
let mut matches = rows
|
||||||
|
.into_iter()
|
||||||
|
.map(row_to_entry)
|
||||||
|
.collect::<Result<Vec<_>>>()?
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|entry| {
|
||||||
|
let raw_exact = entry.source_text == source_text;
|
||||||
|
let normalized_exact = entry.normalized_source_text == normalized_source_text;
|
||||||
|
if !raw_exact && !normalized_exact {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let same_context = !source_context.is_empty()
|
||||||
|
&& !entry.source_context.is_empty()
|
||||||
|
&& entry.source_context == *source_context;
|
||||||
|
let strong = raw_exact
|
||||||
|
&& same_context
|
||||||
|
&& entry.trust_status == TranslationMemoryTrustStatus::Trusted;
|
||||||
|
let match_kind = if strong {
|
||||||
|
TranslationMemoryMatchKind::StrongExact
|
||||||
|
} else if raw_exact {
|
||||||
|
TranslationMemoryMatchKind::CandidateExact
|
||||||
|
} else {
|
||||||
|
TranslationMemoryMatchKind::SourceOnly
|
||||||
|
};
|
||||||
|
Some(TranslationMemoryMatch {
|
||||||
|
can_auto_reuse: strong,
|
||||||
|
entry,
|
||||||
|
match_kind,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
matches.sort_by(|left, right| {
|
||||||
|
match_rank(left)
|
||||||
|
.cmp(&match_rank(right))
|
||||||
|
.then_with(|| {
|
||||||
|
right
|
||||||
|
.entry
|
||||||
|
.updated_unix_seconds
|
||||||
|
.cmp(&left.entry.updated_unix_seconds)
|
||||||
|
})
|
||||||
|
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
|
||||||
|
});
|
||||||
|
matches.truncate(limit);
|
||||||
|
Ok(matches)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn confirm(
|
||||||
|
&self,
|
||||||
|
record_id: &str,
|
||||||
|
reviewer: &str,
|
||||||
|
reason: Option<String>,
|
||||||
|
) -> Result<TranslationMemoryEntry> {
|
||||||
|
if reviewer.trim().is_empty() {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory reviewer 不能为空".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let current = self.find(record_id).await?;
|
||||||
|
if current.trust_status == TranslationMemoryTrustStatus::Trusted {
|
||||||
|
return Ok(current);
|
||||||
|
}
|
||||||
|
if current.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||||
|
return Err(Error::InvalidArgument(format!(
|
||||||
|
"Translation Memory 记录 {} 当前状态为 {},不能确认",
|
||||||
|
record_id,
|
||||||
|
current.trust_status.as_str()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let now = unix_seconds_now();
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE translation_memory
|
||||||
|
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
|
||||||
|
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
|
||||||
|
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(record_id)
|
||||||
|
.bind(i64::try_from(now).unwrap_or(i64::MAX))
|
||||||
|
.bind(reviewer.trim())
|
||||||
|
.bind(reason.filter(|value| !value.trim().is_empty()))
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
self.find(record_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
|
||||||
|
self.find_optional(record_id)
|
||||||
|
.await?
|
||||||
|
.ok_or_else(|| Error::NotFound(record_id.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn summary(&self) -> Result<TranslationMemorySummary> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT COUNT(*) AS record_count,
|
||||||
|
SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count,
|
||||||
|
SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count,
|
||||||
|
SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count,
|
||||||
|
SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count
|
||||||
|
FROM translation_memory
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(db_error)?;
|
||||||
|
Ok(TranslationMemorySummary {
|
||||||
|
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||||||
|
record_count: row.try_get::<i64, _>("record_count").map_err(db_error)? as u64,
|
||||||
|
trusted_count: row.try_get::<i64, _>("trusted_count").map_err(db_error)? as u64,
|
||||||
|
candidate_count: row.try_get::<i64, _>("candidate_count").map_err(db_error)? as u64,
|
||||||
|
superseded_count: row
|
||||||
|
.try_get::<i64, _>("superseded_count")
|
||||||
|
.map_err(db_error)? as u64,
|
||||||
|
rejected_count: row.try_get::<i64, _>("rejected_count").map_err(db_error)? as u64,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 由 active official release 根目录计算默认 TM 数据库路径。
|
||||||
|
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
|
||||||
|
SqliteTranslationMemoryRepository::repository_path(resource_root)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 从 TextUnit 定位字段构建 TM 上下文。
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub fn translation_memory_context(
|
||||||
|
destination: &str,
|
||||||
|
archive_entry: Option<&str>,
|
||||||
|
serialized_file: Option<&str>,
|
||||||
|
path_id: Option<i64>,
|
||||||
|
class_id: Option<i32>,
|
||||||
|
field_path: Option<&str>,
|
||||||
|
format: Option<&str>,
|
||||||
|
asset_name: Option<&str>,
|
||||||
|
text_source_kind: Option<&str>,
|
||||||
|
parser_context: &TranslationMemoryContext,
|
||||||
|
) -> TranslationMemoryContext {
|
||||||
|
let mut context = TranslationMemoryContext::new();
|
||||||
|
context.insert("destination".to_string(), destination.to_string());
|
||||||
|
insert_optional(&mut context, "archive_entry", archive_entry);
|
||||||
|
insert_optional(&mut context, "serialized_file", serialized_file);
|
||||||
|
if let Some(value) = path_id {
|
||||||
|
context.insert("path_id".to_string(), value.to_string());
|
||||||
|
}
|
||||||
|
if let Some(value) = class_id {
|
||||||
|
context.insert("class_id".to_string(), value.to_string());
|
||||||
|
}
|
||||||
|
insert_optional(&mut context, "field_path", field_path);
|
||||||
|
insert_optional(&mut context, "format", format);
|
||||||
|
insert_optional(&mut context, "asset_name", asset_name);
|
||||||
|
insert_optional(&mut context, "text_source_kind", text_source_kind);
|
||||||
|
for (key, value) in parser_context {
|
||||||
|
context.insert(format!("context.{key}"), value.clone());
|
||||||
|
}
|
||||||
|
context
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
|
||||||
|
if let Some(value) = value.filter(|value| !value.is_empty()) {
|
||||||
|
context.insert(key.to_string(), value.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
|
||||||
|
let source_hash = hash_text(&draft.source_text);
|
||||||
|
let normalized_source_text = normalize_source_text(&draft.source_text);
|
||||||
|
let source_context_hash = hash_context(&draft.source_context)?;
|
||||||
|
let source_kind = draft.translation_source_kind;
|
||||||
|
let record_id = record_id(
|
||||||
|
&source_hash,
|
||||||
|
&source_context_hash,
|
||||||
|
&draft.translated_text,
|
||||||
|
&draft.official_release_id,
|
||||||
|
&source_kind,
|
||||||
|
);
|
||||||
|
let observed = draft.observed_unix_seconds;
|
||||||
|
Ok(TranslationMemoryEntry {
|
||||||
|
record_id,
|
||||||
|
source_text: draft.source_text,
|
||||||
|
source_hash,
|
||||||
|
normalized_source_text,
|
||||||
|
source_context: draft.source_context,
|
||||||
|
source_context_hash,
|
||||||
|
translated_text: draft.translated_text,
|
||||||
|
translation_source_kind: source_kind,
|
||||||
|
trust_status: TranslationMemoryTrustStatus::Candidate,
|
||||||
|
official_release_id: draft.official_release_id,
|
||||||
|
source_trace: draft.source_trace,
|
||||||
|
provider: draft.provider,
|
||||||
|
provider_run_id: draft.provider_run_id,
|
||||||
|
created_unix_seconds: observed,
|
||||||
|
updated_unix_seconds: observed,
|
||||||
|
trusted_unix_seconds: None,
|
||||||
|
trusted_by: None,
|
||||||
|
trusted_reason: None,
|
||||||
|
supersedes_record_id: None,
|
||||||
|
superseded_by_record_id: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
|
||||||
|
if draft.source_text.is_empty() {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory source_text 不能为空".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if draft.translated_text.trim().is_empty() {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory translated_text 不能为空".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if draft.official_release_id.trim().is_empty()
|
||||||
|
|| draft.source_trace.official_release_id.trim().is_empty()
|
||||||
|
{
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory official_release_id 不能为空".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if draft.official_release_id != draft.source_trace.official_release_id {
|
||||||
|
return Err(Error::InvalidArgument(
|
||||||
|
"Translation Memory draft 的 release provenance 不一致".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
|
||||||
|
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
|
||||||
|
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
|
||||||
|
Ok(TranslationMemoryEntry {
|
||||||
|
record_id: row.try_get("record_id").map_err(db_error)?,
|
||||||
|
source_text: row.try_get("source_text").map_err(db_error)?,
|
||||||
|
source_hash: row.try_get("source_hash").map_err(db_error)?,
|
||||||
|
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
|
||||||
|
source_context,
|
||||||
|
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
|
||||||
|
translated_text: row.try_get("translated_text").map_err(db_error)?,
|
||||||
|
translation_source_kind: parse_source_kind(
|
||||||
|
row.try_get::<String, _>("translation_source_kind")
|
||||||
|
.map_err(db_error)?
|
||||||
|
.as_str(),
|
||||||
|
)?,
|
||||||
|
trust_status: parse_trust_status(
|
||||||
|
row.try_get::<String, _>("trust_status")
|
||||||
|
.map_err(db_error)?
|
||||||
|
.as_str(),
|
||||||
|
)?,
|
||||||
|
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
|
||||||
|
source_trace,
|
||||||
|
provider: row.try_get("provider").map_err(db_error)?,
|
||||||
|
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
|
||||||
|
created_unix_seconds: i64_to_u64(
|
||||||
|
row.try_get("created_unix_seconds").map_err(db_error)?,
|
||||||
|
"created",
|
||||||
|
)?,
|
||||||
|
updated_unix_seconds: i64_to_u64(
|
||||||
|
row.try_get("updated_unix_seconds").map_err(db_error)?,
|
||||||
|
"updated",
|
||||||
|
)?,
|
||||||
|
trusted_unix_seconds: optional_i64_to_u64(
|
||||||
|
row.try_get("trusted_unix_seconds").map_err(db_error)?,
|
||||||
|
"trusted",
|
||||||
|
)?,
|
||||||
|
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
|
||||||
|
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
|
||||||
|
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
|
||||||
|
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
|
||||||
|
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
|
||||||
|
match value {
|
||||||
|
"provider" => Ok(TranslationMemorySourceKind::Provider),
|
||||||
|
"manual" => Ok(TranslationMemorySourceKind::Manual),
|
||||||
|
"imported" => Ok(TranslationMemorySourceKind::Imported),
|
||||||
|
_ => Err(Error::Serialization(format!(
|
||||||
|
"未知 Translation Memory source kind:{value}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
|
||||||
|
match value {
|
||||||
|
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
|
||||||
|
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
|
||||||
|
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
|
||||||
|
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
|
||||||
|
_ => Err(Error::Serialization(format!(
|
||||||
|
"未知 Translation Memory trust status:{value}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
|
||||||
|
match value.match_kind {
|
||||||
|
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
|
||||||
|
TranslationMemoryMatchKind::CandidateExact => 1,
|
||||||
|
TranslationMemoryMatchKind::SourceOnly => 2,
|
||||||
|
TranslationMemoryMatchKind::StrongExact => 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash_text(value: &str) -> String {
|
||||||
|
blake3::hash(value.as_bytes()).to_hex().to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_source_text(value: &str) -> String {
|
||||||
|
value.replace("\r\n", "\n").replace('\r', "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
|
||||||
|
let bytes =
|
||||||
|
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
|
||||||
|
Ok(blake3::hash(&bytes).to_hex().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_id(
|
||||||
|
source_hash: &str,
|
||||||
|
context_hash: &str,
|
||||||
|
translated_text: &str,
|
||||||
|
official_release_id: &str,
|
||||||
|
source_kind: &TranslationMemorySourceKind,
|
||||||
|
) -> String {
|
||||||
|
let mut key = Vec::new();
|
||||||
|
for value in [
|
||||||
|
source_hash,
|
||||||
|
context_hash,
|
||||||
|
translated_text,
|
||||||
|
official_release_id,
|
||||||
|
source_kind.as_str(),
|
||||||
|
] {
|
||||||
|
key.extend_from_slice(value.as_bytes());
|
||||||
|
key.push(0);
|
||||||
|
}
|
||||||
|
format!("tm-{}", blake3::hash(&key).to_hex())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_seconds_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
|
||||||
|
u64::try_from(value)
|
||||||
|
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
|
||||||
|
value.map(|value| i64_to_u64(value, label)).transpose()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn db_error(error: sqlx::Error) -> Error {
|
||||||
|
Error::Other(error.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
|
||||||
|
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
|
||||||
|
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
|
||||||
|
.map_err(Error::InvalidArgument)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use bat_core::domain::TranslationMemorySourceTrace;
|
||||||
|
|
||||||
|
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
|
||||||
|
let source_trace = TranslationMemorySourceTrace {
|
||||||
|
official_release_id: release.to_string(),
|
||||||
|
unit_id: Some(format!("{release}-unit")),
|
||||||
|
task_id: Some(format!("{release}-task")),
|
||||||
|
destination: Some("Bundles/story.bundle".to_string()),
|
||||||
|
archive_entry: None,
|
||||||
|
serialized_file: Some("CAB-story".to_string()),
|
||||||
|
path_id: Some(1),
|
||||||
|
class_id: Some(49),
|
||||||
|
field_path: Some("m_Text".to_string()),
|
||||||
|
format: Some("plain".to_string()),
|
||||||
|
asset_name: Some("Story".to_string()),
|
||||||
|
text_source_kind: Some("text_asset".to_string()),
|
||||||
|
source_url: Some("https://example.invalid/story".to_string()),
|
||||||
|
};
|
||||||
|
TranslationMemoryDraft {
|
||||||
|
source_text: source.to_string(),
|
||||||
|
source_context: translation_memory_context(
|
||||||
|
"Bundles/story.bundle",
|
||||||
|
None,
|
||||||
|
Some("CAB-story"),
|
||||||
|
Some(1),
|
||||||
|
Some(49),
|
||||||
|
Some("m_Text"),
|
||||||
|
Some("plain"),
|
||||||
|
Some("Story"),
|
||||||
|
Some("text_asset"),
|
||||||
|
&TranslationMemoryContext::new(),
|
||||||
|
),
|
||||||
|
translated_text: translated.to_string(),
|
||||||
|
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||||||
|
official_release_id: release.to_string(),
|
||||||
|
source_trace,
|
||||||
|
provider: Some("mock".to_string()),
|
||||||
|
provider_run_id: Some(format!("mock:{release}")),
|
||||||
|
observed_unix_seconds: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let repository = SqliteTranslationMemoryRepository::new(
|
||||||
|
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let entry = repository
|
||||||
|
.upsert_candidate(draft("release-1", "Hello", "你好"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
|
||||||
|
let trusted = repository
|
||||||
|
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
|
||||||
|
|
||||||
|
let query = draft("release-2", "Hello", "ignored");
|
||||||
|
let matches = repository
|
||||||
|
.find_matches("Hello", &query.source_context, 10)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(matches.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
matches[0].match_kind,
|
||||||
|
TranslationMemoryMatchKind::StrongExact
|
||||||
|
);
|
||||||
|
assert!(matches[0].can_auto_reuse);
|
||||||
|
assert_eq!(matches[0].entry.translated_text, "你好");
|
||||||
|
assert_eq!(matches[0].entry.official_release_id, "release-1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rejects_future_schema_version() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let path = temp.path().join("tm.sqlite");
|
||||||
|
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
|
||||||
|
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
|
||||||
|
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||||
|
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
|
||||||
|
.execute(&repository.pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
repository.pool.close().await;
|
||||||
|
|
||||||
|
let error = SqliteTranslationMemoryRepository::new(&path)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error
|
||||||
|
.to_string()
|
||||||
|
.contains("不支持的 Translation Memory schema"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let first = draft("release-1", "Hello", "你好");
|
||||||
|
repository.upsert_candidate(first.clone()).await.unwrap();
|
||||||
|
repository.upsert_candidate(first).await.unwrap();
|
||||||
|
assert_eq!(repository.summary().await.unwrap().record_count, 1);
|
||||||
|
|
||||||
|
let mut different = draft("release-2", "Hello", "你好");
|
||||||
|
different
|
||||||
|
.source_context
|
||||||
|
.insert("field_path".to_string(), "m_Other".to_string());
|
||||||
|
let matches = repository
|
||||||
|
.find_matches("Hello", &different.source_context, 10)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
matches[0].match_kind,
|
||||||
|
TranslationMemoryMatchKind::CandidateExact
|
||||||
|
);
|
||||||
|
assert!(!matches[0].can_auto_reuse);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rejects_empty_translation_candidates() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let error = repository
|
||||||
|
.upsert_candidate(draft("release-1", "Hello", " \n"))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("translated_text"));
|
||||||
|
assert_eq!(repository.summary().await.unwrap().record_count, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -161,6 +161,12 @@ pub struct TranslationTaskUnitResult {
|
|||||||
pub source_text: String,
|
pub source_text: String,
|
||||||
/// Provider-produced or human-supplied translation.
|
/// Provider-produced or human-supplied translation.
|
||||||
pub translated_text: String,
|
pub translated_text: String,
|
||||||
|
/// Result source kind.
|
||||||
|
#[serde(default)]
|
||||||
|
pub source_kind: TranslationTaskResultSourceKind,
|
||||||
|
/// Trusted Translation Memory record used for this result, when applicable.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_memory_record_id: Option<String>,
|
||||||
/// Provider identifier.
|
/// Provider identifier.
|
||||||
pub provider: String,
|
pub provider: String,
|
||||||
/// Provider run that produced this result.
|
/// Provider run that produced this result.
|
||||||
@@ -169,6 +175,30 @@ pub struct TranslationTaskUnitResult {
|
|||||||
pub translated_unix_seconds: u64,
|
pub translated_unix_seconds: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Source of one persisted TextUnit translation result.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TranslationTaskResultSourceKind {
|
||||||
|
/// Result returned by the configured provider.
|
||||||
|
#[default]
|
||||||
|
Provider,
|
||||||
|
/// Result submitted through the manual task update interface.
|
||||||
|
Manual,
|
||||||
|
/// Result reused from a trusted Translation Memory entry.
|
||||||
|
TranslationMemory,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TranslationTaskResultSourceKind {
|
||||||
|
/// Returns the stable JSON label.
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Provider => "provider",
|
||||||
|
Self::Manual => "manual",
|
||||||
|
Self::TranslationMemory => "translation_memory",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// One provider execution associated with one or more translation units.
|
/// One provider execution associated with one or more translation units.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ProviderRun {
|
pub struct ProviderRun {
|
||||||
@@ -247,12 +277,18 @@ pub fn build_translation_handoff(
|
|||||||
Vec::new(),
|
Vec::new(),
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
let unit_status = match state.0 {
|
let unit_status = if state.5.is_empty() {
|
||||||
|
match state.0 {
|
||||||
TranslationTaskStatus::Queued => TranslationUnitStatus::Queued,
|
TranslationTaskStatus::Queued => TranslationUnitStatus::Queued,
|
||||||
TranslationTaskStatus::Running => TranslationUnitStatus::Translating,
|
TranslationTaskStatus::Running => TranslationUnitStatus::Translating,
|
||||||
TranslationTaskStatus::Failed => TranslationUnitStatus::Failed,
|
TranslationTaskStatus::Failed => TranslationUnitStatus::Failed,
|
||||||
TranslationTaskStatus::Completed => TranslationUnitStatus::Translated,
|
TranslationTaskStatus::Completed => TranslationUnitStatus::Translated,
|
||||||
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped,
|
TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// A task can retain successful TM hits while the remaining provider
|
||||||
|
// units are failed or waiting for retry.
|
||||||
|
TranslationUnitStatus::Translated
|
||||||
};
|
};
|
||||||
let unit = TranslationUnit {
|
let unit = TranslationUnit {
|
||||||
unit_id: task.task_id.clone(),
|
unit_id: task.task_id.clone(),
|
||||||
@@ -1168,11 +1204,29 @@ impl SqliteTranslationTaskRepository {
|
|||||||
pub async fn fail_claim(
|
pub async fn fail_claim(
|
||||||
&self,
|
&self,
|
||||||
failure: TranslationTaskFailure,
|
failure: TranslationTaskFailure,
|
||||||
|
) -> Result<PersistedTranslationTask> {
|
||||||
|
self.fail_claim_with_results(failure, &[]).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records a provider failure while retaining any already-resolved TextUnit
|
||||||
|
/// results, such as trusted Translation Memory hits.
|
||||||
|
pub async fn fail_claim_with_results(
|
||||||
|
&self,
|
||||||
|
failure: TranslationTaskFailure,
|
||||||
|
translation_results: &[TranslationTaskUnitResult],
|
||||||
) -> Result<PersistedTranslationTask> {
|
) -> Result<PersistedTranslationTask> {
|
||||||
let now = unix_seconds_now_i64();
|
let now = unix_seconds_now_i64();
|
||||||
let next_attempt = failure
|
let next_attempt = failure
|
||||||
.next_attempt_unix_seconds
|
.next_attempt_unix_seconds
|
||||||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX));
|
.map(|value| i64::try_from(value).unwrap_or(i64::MAX));
|
||||||
|
let translation_results_json = if translation_results.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(
|
||||||
|
serde_json::to_string(translation_results)
|
||||||
|
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?,
|
||||||
|
)
|
||||||
|
};
|
||||||
let result = sqlx::query(
|
let result = sqlx::query(
|
||||||
r#"
|
r#"
|
||||||
UPDATE translation_tasks
|
UPDATE translation_tasks
|
||||||
@@ -1183,11 +1237,12 @@ impl SqliteTranslationTaskRepository {
|
|||||||
lease_expires_unix_seconds = NULL,
|
lease_expires_unix_seconds = NULL,
|
||||||
failure_class = ?4,
|
failure_class = ?4,
|
||||||
failure_retryable = ?5,
|
failure_retryable = ?5,
|
||||||
next_attempt_unix_seconds = ?6
|
next_attempt_unix_seconds = ?6,
|
||||||
|
translation_results_json = COALESCE(?7, translation_results_json)
|
||||||
WHERE task_id = ?1
|
WHERE task_id = ?1
|
||||||
AND worker_status = 'running'
|
AND worker_status = 'running'
|
||||||
AND lease_owner = ?7
|
AND lease_owner = ?8
|
||||||
AND provider_run_id = ?8
|
AND provider_run_id = ?9
|
||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
.bind(&failure.task_id)
|
.bind(&failure.task_id)
|
||||||
@@ -1196,6 +1251,7 @@ impl SqliteTranslationTaskRepository {
|
|||||||
.bind(&failure.failure_class)
|
.bind(&failure.failure_class)
|
||||||
.bind(if failure.retryable { 1_i64 } else { 0_i64 })
|
.bind(if failure.retryable { 1_i64 } else { 0_i64 })
|
||||||
.bind(next_attempt)
|
.bind(next_attempt)
|
||||||
|
.bind(translation_results_json)
|
||||||
.bind(&failure.worker_id)
|
.bind(&failure.worker_id)
|
||||||
.bind(&failure.provider_run_id)
|
.bind(&failure.provider_run_id)
|
||||||
.execute(&self.pool)
|
.execute(&self.pool)
|
||||||
@@ -1747,6 +1803,8 @@ mod tests {
|
|||||||
unit_id: "unit-a".to_string(),
|
unit_id: "unit-a".to_string(),
|
||||||
source_text: "source".to_string(),
|
source_text: "source".to_string(),
|
||||||
translated_text: "manual translation".to_string(),
|
translated_text: "manual translation".to_string(),
|
||||||
|
source_kind: TranslationTaskResultSourceKind::Manual,
|
||||||
|
translation_memory_record_id: None,
|
||||||
provider: "manual".to_string(),
|
provider: "manual".to_string(),
|
||||||
provider_run_id: "manual-run-1".to_string(),
|
provider_run_id: "manual-run-1".to_string(),
|
||||||
translated_unix_seconds: 321,
|
translated_unix_seconds: 321,
|
||||||
@@ -1864,6 +1922,8 @@ mod tests {
|
|||||||
unit_id: "unit-a".to_string(),
|
unit_id: "unit-a".to_string(),
|
||||||
source_text: "source".to_string(),
|
source_text: "source".to_string(),
|
||||||
translated_text: "translated".to_string(),
|
translated_text: "translated".to_string(),
|
||||||
|
source_kind: TranslationTaskResultSourceKind::Provider,
|
||||||
|
translation_memory_record_id: None,
|
||||||
provider: "mock".to_string(),
|
provider: "mock".to_string(),
|
||||||
provider_run_id: second_run.clone(),
|
provider_run_id: second_run.clone(),
|
||||||
translated_unix_seconds: 1,
|
translated_unix_seconds: 1,
|
||||||
|
|||||||
@@ -2,15 +2,21 @@
|
|||||||
//!
|
//!
|
||||||
//! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态,不
|
//! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态,不
|
||||||
//! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract;状态、
|
//! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract;状态、
|
||||||
//! 租约和译文结果始终写入 `translation-tasks.sqlite`。
|
//! 租约和任务结果写入 release 级 `translation-tasks.sqlite`,跨 release 的
|
||||||
|
//! Translation Memory 写入项目级独立 SQLite 数据库。
|
||||||
|
|
||||||
use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit};
|
use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit};
|
||||||
use crate::official_textunit_queue::read_textunit_task_queue_at;
|
use crate::official_textunit_queue::read_textunit_task_queue_at;
|
||||||
|
use crate::translation_memory::{translation_memory_context, SqliteTranslationMemoryRepository};
|
||||||
use crate::translation_tasks::{
|
use crate::translation_tasks::{
|
||||||
PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure,
|
PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure,
|
||||||
TranslationTaskUnitResult,
|
TranslationTaskResultSourceKind, TranslationTaskUnitResult,
|
||||||
};
|
};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
use bat_core::domain::{
|
||||||
|
TranslationMemoryDraft, TranslationMemorySourceKind, TranslationMemorySourceTrace,
|
||||||
|
};
|
||||||
|
use bat_core::repositories::TranslationMemoryRepository;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::env;
|
use std::env;
|
||||||
@@ -89,6 +95,9 @@ pub struct TranslationWorkerConfig {
|
|||||||
pub max_tasks: Option<usize>,
|
pub max_tasks: Option<usize>,
|
||||||
/// worker 实例前缀,用于 lease 诊断。
|
/// worker 实例前缀,用于 lease 诊断。
|
||||||
pub worker_id: String,
|
pub worker_id: String,
|
||||||
|
/// Translation Memory SQLite path. `None` uses the output-root default.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_memory_path: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for TranslationWorkerConfig {
|
impl Default for TranslationWorkerConfig {
|
||||||
@@ -102,6 +111,7 @@ impl Default for TranslationWorkerConfig {
|
|||||||
retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
||||||
max_tasks: None,
|
max_tasks: None,
|
||||||
worker_id: format!("bat-worker-{}", std::process::id()),
|
worker_id: format!("bat-worker-{}", std::process::id()),
|
||||||
|
translation_memory_path: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -576,6 +586,16 @@ pub struct TranslationWorkerReport {
|
|||||||
pub remaining_count: usize,
|
pub remaining_count: usize,
|
||||||
/// 失败诊断。
|
/// 失败诊断。
|
||||||
pub failures: Vec<TranslationWorkerFailure>,
|
pub failures: Vec<TranslationWorkerFailure>,
|
||||||
|
/// Translation Memory database path used by this run.
|
||||||
|
pub translation_memory_path: PathBuf,
|
||||||
|
/// Whether the Translation Memory repository was available.
|
||||||
|
pub translation_memory_available: bool,
|
||||||
|
/// Number of TextUnits reused from trusted Translation Memory.
|
||||||
|
pub translation_memory_hit_count: usize,
|
||||||
|
/// Number of TextUnits sent to the provider after Translation Memory lookup.
|
||||||
|
pub provider_unit_count: usize,
|
||||||
|
/// Translation Memory diagnostics that did not invalidate provider work.
|
||||||
|
pub translation_memory_failures: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// worker 失败诊断。
|
/// worker 失败诊断。
|
||||||
@@ -599,7 +619,10 @@ struct WorkerStats {
|
|||||||
completed_count: AtomicUsize,
|
completed_count: AtomicUsize,
|
||||||
failed_count: AtomicUsize,
|
failed_count: AtomicUsize,
|
||||||
retry_scheduled_count: AtomicUsize,
|
retry_scheduled_count: AtomicUsize,
|
||||||
|
translation_memory_hit_count: AtomicUsize,
|
||||||
|
provider_unit_count: AtomicUsize,
|
||||||
failures: Mutex<Vec<TranslationWorkerFailure>>,
|
failures: Mutex<Vec<TranslationWorkerFailure>>,
|
||||||
|
translation_memory_failures: Mutex<Vec<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct WorkerTaskContext<'a> {
|
struct WorkerTaskContext<'a> {
|
||||||
@@ -611,12 +634,25 @@ struct WorkerTaskContext<'a> {
|
|||||||
max_attempts: u32,
|
max_attempts: u32,
|
||||||
retry_backoff: Duration,
|
retry_backoff: Duration,
|
||||||
stats: &'a WorkerStats,
|
stats: &'a WorkerStats,
|
||||||
|
translation_memory: Option<&'a dyn TranslationMemoryRepository>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 运行一个 provider worker 轮次。
|
/// 运行一个 provider worker 轮次。
|
||||||
pub async fn run_translation_worker_at(
|
pub async fn run_translation_worker_at(
|
||||||
resource_root: &Path,
|
resource_root: &Path,
|
||||||
config: &TranslationWorkerConfig,
|
config: &TranslationWorkerConfig,
|
||||||
|
) -> anyhow::Result<TranslationWorkerReport> {
|
||||||
|
run_translation_worker_at_with_cancellation(resource_root, config, Arc::new(|| false)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 运行一个可协作取消的 provider worker 轮次。
|
||||||
|
///
|
||||||
|
/// 取消只在 claim 循环边界检查;已经开始的 provider 请求会先完成,
|
||||||
|
/// 避免丢失 lease 结果或遗留未清理的外部子进程。
|
||||||
|
pub async fn run_translation_worker_at_with_cancellation(
|
||||||
|
resource_root: &Path,
|
||||||
|
config: &TranslationWorkerConfig,
|
||||||
|
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||||
) -> anyhow::Result<TranslationWorkerReport> {
|
) -> anyhow::Result<TranslationWorkerReport> {
|
||||||
config.validate()?;
|
config.validate()?;
|
||||||
let provider: Arc<dyn TranslationProvider> = match config.provider {
|
let provider: Arc<dyn TranslationProvider> = match config.provider {
|
||||||
@@ -625,7 +661,13 @@ pub async fn run_translation_worker_at(
|
|||||||
)?),
|
)?),
|
||||||
TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?),
|
TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?),
|
||||||
};
|
};
|
||||||
run_translation_worker_with_provider(resource_root, config, provider).await
|
run_translation_worker_with_provider_and_cancellation(
|
||||||
|
resource_root,
|
||||||
|
config,
|
||||||
|
provider,
|
||||||
|
should_cancel,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 使用指定 provider 运行 worker,供测试和插件宿主使用。
|
/// 使用指定 provider 运行 worker,供测试和插件宿主使用。
|
||||||
@@ -633,6 +675,21 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
resource_root: &Path,
|
resource_root: &Path,
|
||||||
config: &TranslationWorkerConfig,
|
config: &TranslationWorkerConfig,
|
||||||
provider: Arc<dyn TranslationProvider>,
|
provider: Arc<dyn TranslationProvider>,
|
||||||
|
) -> anyhow::Result<TranslationWorkerReport> {
|
||||||
|
run_translation_worker_with_provider_and_cancellation(
|
||||||
|
resource_root,
|
||||||
|
config,
|
||||||
|
provider,
|
||||||
|
Arc::new(|| false),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_translation_worker_with_provider_and_cancellation(
|
||||||
|
resource_root: &Path,
|
||||||
|
config: &TranslationWorkerConfig,
|
||||||
|
provider: Arc<dyn TranslationProvider>,
|
||||||
|
should_cancel: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||||
) -> anyhow::Result<TranslationWorkerReport> {
|
) -> anyhow::Result<TranslationWorkerReport> {
|
||||||
config.validate()?;
|
config.validate()?;
|
||||||
let queue = read_textunit_task_queue_at(resource_root)
|
let queue = read_textunit_task_queue_at(resource_root)
|
||||||
@@ -643,6 +700,21 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
.map_err(anyhow::Error::msg)?
|
.map_err(anyhow::Error::msg)?
|
||||||
.ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?,
|
.ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?,
|
||||||
);
|
);
|
||||||
|
let translation_memory_path = config
|
||||||
|
.translation_memory_path
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| SqliteTranslationMemoryRepository::repository_path(resource_root));
|
||||||
|
let (translation_memory, translation_memory_startup_failure) =
|
||||||
|
match SqliteTranslationMemoryRepository::new(&translation_memory_path).await {
|
||||||
|
Ok(repository) => (Some(Arc::new(repository)), None),
|
||||||
|
Err(error) => (
|
||||||
|
None,
|
||||||
|
Some(format!(
|
||||||
|
"打开 Translation Memory 数据库失败 {}:{error}",
|
||||||
|
translation_memory_path.display()
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
};
|
||||||
let repository = Arc::new(
|
let repository = Arc::new(
|
||||||
SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path(
|
SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path(
|
||||||
resource_root,
|
resource_root,
|
||||||
@@ -659,6 +731,13 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
.await
|
.await
|
||||||
.map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败:{error}"))?;
|
.map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败:{error}"))?;
|
||||||
let stats = Arc::new(WorkerStats::default());
|
let stats = Arc::new(WorkerStats::default());
|
||||||
|
if let Some(failure) = translation_memory_startup_failure {
|
||||||
|
stats
|
||||||
|
.translation_memory_failures
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
|
||||||
|
.push(failure);
|
||||||
|
}
|
||||||
let claimed_limit = Arc::new(AtomicUsize::new(0));
|
let claimed_limit = Arc::new(AtomicUsize::new(0));
|
||||||
let mut handles = Vec::with_capacity(config.concurrency);
|
let mut handles = Vec::with_capacity(config.concurrency);
|
||||||
|
|
||||||
@@ -674,8 +753,13 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
let max_attempts = config.max_attempts;
|
let max_attempts = config.max_attempts;
|
||||||
let lease_seconds = config.lease_seconds;
|
let lease_seconds = config.lease_seconds;
|
||||||
let retry_backoff = config.retry_backoff;
|
let retry_backoff = config.retry_backoff;
|
||||||
|
let translation_memory = translation_memory.clone();
|
||||||
|
let should_cancel = Arc::clone(&should_cancel);
|
||||||
handles.push(tokio::spawn(async move {
|
handles.push(tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
|
if should_cancel() {
|
||||||
|
return Err(anyhow::anyhow!("翻译 worker 已取消"));
|
||||||
|
}
|
||||||
if let Some(max_tasks) = max_tasks {
|
if let Some(max_tasks) = max_tasks {
|
||||||
let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel);
|
let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel);
|
||||||
if reservation >= max_tasks {
|
if reservation >= max_tasks {
|
||||||
@@ -704,6 +788,9 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
max_attempts,
|
max_attempts,
|
||||||
retry_backoff,
|
retry_backoff,
|
||||||
stats: &stats,
|
stats: &stats,
|
||||||
|
translation_memory: translation_memory
|
||||||
|
.as_deref()
|
||||||
|
.map(|repository| repository as &dyn TranslationMemoryRepository),
|
||||||
},
|
},
|
||||||
&task,
|
&task,
|
||||||
)
|
)
|
||||||
@@ -712,10 +799,24 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
Ok::<(), anyhow::Error>(())
|
Ok::<(), anyhow::Error>(())
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
let mut first_worker_error = None;
|
||||||
for handle in handles {
|
for handle in handles {
|
||||||
handle
|
match handle.await {
|
||||||
.await
|
Ok(Ok(())) => {}
|
||||||
.map_err(|error| anyhow::anyhow!("等待翻译 worker 失败:{error}"))??;
|
Ok(Err(error)) => {
|
||||||
|
if first_worker_error.is_none() {
|
||||||
|
first_worker_error = Some(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
if first_worker_error.is_none() {
|
||||||
|
first_worker_error = Some(anyhow::anyhow!("等待翻译 worker 失败:{error}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(error) = first_worker_error {
|
||||||
|
return Err(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
let remaining_count = repository
|
let remaining_count = repository
|
||||||
@@ -736,6 +837,11 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
.map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))?
|
.map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))?
|
||||||
.clone();
|
.clone();
|
||||||
let failed_count = stats.failed_count.load(Ordering::Relaxed);
|
let failed_count = stats.failed_count.load(Ordering::Relaxed);
|
||||||
|
let translation_memory_failures = stats
|
||||||
|
.translation_memory_failures
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| anyhow::anyhow!("读取 Translation Memory 诊断时 mutex poisoned"))?
|
||||||
|
.clone();
|
||||||
Ok(TranslationWorkerReport {
|
Ok(TranslationWorkerReport {
|
||||||
command: "translation-worker",
|
command: "translation-worker",
|
||||||
status: if failed_count == 0 {
|
status: if failed_count == 0 {
|
||||||
@@ -752,6 +858,11 @@ pub async fn run_translation_worker_with_provider(
|
|||||||
retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed),
|
retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed),
|
||||||
remaining_count,
|
remaining_count,
|
||||||
failures,
|
failures,
|
||||||
|
translation_memory_path,
|
||||||
|
translation_memory_available: translation_memory.is_some(),
|
||||||
|
translation_memory_hit_count: stats.translation_memory_hit_count.load(Ordering::Relaxed),
|
||||||
|
provider_unit_count: stats.provider_unit_count.load(Ordering::Relaxed),
|
||||||
|
translation_memory_failures,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -759,7 +870,60 @@ async fn process_claimed_task(
|
|||||||
context: &WorkerTaskContext<'_>,
|
context: &WorkerTaskContext<'_>,
|
||||||
task: &PersistedTranslationTask,
|
task: &PersistedTranslationTask,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let request = match provider_request(task, context.index) {
|
let task_units = task_index_units(task, context.index)?;
|
||||||
|
let mut results = BTreeMap::new();
|
||||||
|
let mut provider_units = Vec::new();
|
||||||
|
for unit in &task_units {
|
||||||
|
if let Some(translation_memory) = context.translation_memory {
|
||||||
|
let source_context = translation_memory_context(
|
||||||
|
&unit.destination,
|
||||||
|
unit.archive_entry.as_deref(),
|
||||||
|
unit.serialized_file.as_deref(),
|
||||||
|
unit.path_id,
|
||||||
|
unit.class_id,
|
||||||
|
unit.field_path.as_deref(),
|
||||||
|
unit.format.as_deref(),
|
||||||
|
unit.asset_name.as_deref(),
|
||||||
|
unit.text_source_kind.as_deref(),
|
||||||
|
&unit.context,
|
||||||
|
);
|
||||||
|
match translation_memory
|
||||||
|
.find_matches(&unit.source_text, &source_context, 1)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(matches) => {
|
||||||
|
if let Some(found) = matches.into_iter().find(|item| item.can_auto_reuse) {
|
||||||
|
context
|
||||||
|
.stats
|
||||||
|
.translation_memory_hit_count
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
results.insert(
|
||||||
|
unit.id.clone(),
|
||||||
|
translation_memory_result(task, unit, &found.entry),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
record_translation_memory_failure(
|
||||||
|
context,
|
||||||
|
format!(
|
||||||
|
"任务 {} TextUnit {} 查询失败:{}",
|
||||||
|
task.task.task_id, unit.id, error
|
||||||
|
),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
provider_units.push(*unit);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !provider_units.is_empty() {
|
||||||
|
context
|
||||||
|
.stats
|
||||||
|
.provider_unit_count
|
||||||
|
.fetch_add(provider_units.len(), Ordering::Relaxed);
|
||||||
|
let request = match provider_request(task, &provider_units) {
|
||||||
Ok(request) => request,
|
Ok(request) => request,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
record_provider_failure(
|
record_provider_failure(
|
||||||
@@ -769,6 +933,7 @@ async fn process_claimed_task(
|
|||||||
TranslationProviderFailureClass::InvalidRequest,
|
TranslationProviderFailureClass::InvalidRequest,
|
||||||
error.to_string(),
|
error.to_string(),
|
||||||
),
|
),
|
||||||
|
&results,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -776,7 +941,7 @@ async fn process_claimed_task(
|
|||||||
};
|
};
|
||||||
match context.provider.translate(request.clone()).await {
|
match context.provider.translate(request.clone()).await {
|
||||||
Ok(response) => {
|
Ok(response) => {
|
||||||
let results =
|
let provider_results =
|
||||||
match validate_provider_response(&request, response, context.provider_name) {
|
match validate_provider_response(&request, response, context.provider_name) {
|
||||||
Ok(results) => results,
|
Ok(results) => results,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
@@ -787,11 +952,72 @@ async fn process_claimed_task(
|
|||||||
TranslationProviderFailureClass::InvalidRequest,
|
TranslationProviderFailureClass::InvalidRequest,
|
||||||
error.to_string(),
|
error.to_string(),
|
||||||
),
|
),
|
||||||
|
&results,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
for result in &provider_results {
|
||||||
|
results.insert(result.unit_id.clone(), result.clone());
|
||||||
|
if let Some(unit) = provider_units.iter().find(|unit| unit.id == result.unit_id)
|
||||||
|
{
|
||||||
|
let source_context = translation_memory_context(
|
||||||
|
&unit.destination,
|
||||||
|
unit.archive_entry.as_deref(),
|
||||||
|
unit.serialized_file.as_deref(),
|
||||||
|
unit.path_id,
|
||||||
|
unit.class_id,
|
||||||
|
unit.field_path.as_deref(),
|
||||||
|
unit.format.as_deref(),
|
||||||
|
unit.asset_name.as_deref(),
|
||||||
|
unit.text_source_kind.as_deref(),
|
||||||
|
&unit.context,
|
||||||
|
);
|
||||||
|
let draft = TranslationMemoryDraft {
|
||||||
|
source_text: result.source_text.clone(),
|
||||||
|
source_context,
|
||||||
|
translated_text: result.translated_text.clone(),
|
||||||
|
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||||||
|
official_release_id: task.task.official_release_id.clone(),
|
||||||
|
source_trace: translation_memory_trace(task, unit),
|
||||||
|
provider: Some(context.provider_name.to_string()),
|
||||||
|
provider_run_id: Some(request.provider_run_id.clone()),
|
||||||
|
observed_unix_seconds: unix_seconds_now(),
|
||||||
|
};
|
||||||
|
if let Some(translation_memory) = context.translation_memory {
|
||||||
|
if let Err(error) = translation_memory.upsert_candidate(draft).await {
|
||||||
|
record_translation_memory_failure(
|
||||||
|
context,
|
||||||
|
format!(
|
||||||
|
"任务 {} TextUnit {} 写入候选失败:{}",
|
||||||
|
task.task.task_id, unit.id, error
|
||||||
|
),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
record_provider_failure(context, task, error, &results).await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let ordered_results = task_units
|
||||||
|
.iter()
|
||||||
|
.filter_map(|unit| results.get(&unit.id).cloned())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if ordered_results.len() != task_units.len() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"任务 {} 的译文结果不完整:expected={} actual={}",
|
||||||
|
task.task.task_id,
|
||||||
|
task_units.len(),
|
||||||
|
ordered_results.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
context
|
context
|
||||||
.repository
|
.repository
|
||||||
.complete_claim(
|
.complete_claim(
|
||||||
@@ -799,7 +1025,7 @@ async fn process_claimed_task(
|
|||||||
context.worker_id,
|
context.worker_id,
|
||||||
&task.provider_run_id.clone().unwrap_or_default(),
|
&task.provider_run_id.clone().unwrap_or_default(),
|
||||||
context.provider_name,
|
context.provider_name,
|
||||||
&results,
|
&ordered_results,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?;
|
.map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败:{error}"))?;
|
||||||
@@ -807,18 +1033,13 @@ async fn process_claimed_task(
|
|||||||
.stats
|
.stats
|
||||||
.completed_count
|
.completed_count
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
record_provider_failure(context, task, error).await?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn provider_request(
|
fn task_index_units<'a>(
|
||||||
task: &PersistedTranslationTask,
|
task: &PersistedTranslationTask,
|
||||||
index: &crate::official_parse::OfficialTextUnitIndex,
|
index: &'a crate::official_parse::OfficialTextUnitIndex,
|
||||||
) -> anyhow::Result<TranslationProviderRequest> {
|
) -> anyhow::Result<Vec<&'a OfficialTextUnitIndexUnit>> {
|
||||||
let parse_entry_key = task
|
let parse_entry_key = task
|
||||||
.task
|
.task
|
||||||
.parse_entry_key
|
.parse_entry_key
|
||||||
@@ -832,7 +1053,6 @@ fn provider_request(
|
|||||||
&& unit.destination == task.task.destination
|
&& unit.destination == task.task.destination
|
||||||
&& unit.archive_entry == task.task.archive_entry
|
&& unit.archive_entry == task.task.archive_entry
|
||||||
})
|
})
|
||||||
.map(|unit| provider_unit(task, unit))
|
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
if units.is_empty() {
|
if units.is_empty() {
|
||||||
return Err(anyhow::anyhow!(
|
return Err(anyhow::anyhow!(
|
||||||
@@ -840,6 +1060,13 @@ fn provider_request(
|
|||||||
task.task.task_id
|
task.task.task_id
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
Ok(units)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_request(
|
||||||
|
task: &PersistedTranslationTask,
|
||||||
|
index_units: &[&OfficialTextUnitIndexUnit],
|
||||||
|
) -> anyhow::Result<TranslationProviderRequest> {
|
||||||
let provider_run_id = task
|
let provider_run_id = task
|
||||||
.provider_run_id
|
.provider_run_id
|
||||||
.clone()
|
.clone()
|
||||||
@@ -850,7 +1077,10 @@ fn provider_request(
|
|||||||
task_id: task.task.task_id.clone(),
|
task_id: task.task.task_id.clone(),
|
||||||
destination: task.task.destination.clone(),
|
destination: task.task.destination.clone(),
|
||||||
archive_entry: task.task.archive_entry.clone(),
|
archive_entry: task.task.archive_entry.clone(),
|
||||||
units,
|
units: index_units
|
||||||
|
.iter()
|
||||||
|
.map(|unit| provider_unit(task, unit))
|
||||||
|
.collect(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -914,10 +1144,18 @@ fn validate_provider_response(
|
|||||||
result.unit_id
|
result.unit_id
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if result.translated_text.trim().is_empty() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"provider 返回空 translated_text:{}",
|
||||||
|
result.unit_id
|
||||||
|
));
|
||||||
|
}
|
||||||
results.push(TranslationTaskUnitResult {
|
results.push(TranslationTaskUnitResult {
|
||||||
unit_id: result.unit_id,
|
unit_id: result.unit_id,
|
||||||
source_text: result.source_text,
|
source_text: result.source_text,
|
||||||
translated_text: result.translated_text,
|
translated_text: result.translated_text,
|
||||||
|
source_kind: TranslationTaskResultSourceKind::Provider,
|
||||||
|
translation_memory_record_id: None,
|
||||||
provider: provider_name.to_string(),
|
provider: provider_name.to_string(),
|
||||||
provider_run_id: request.provider_run_id.clone(),
|
provider_run_id: request.provider_run_id.clone(),
|
||||||
translated_unix_seconds: unix_seconds_now(),
|
translated_unix_seconds: unix_seconds_now(),
|
||||||
@@ -933,17 +1171,71 @@ fn validate_provider_response(
|
|||||||
Ok(results)
|
Ok(results)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn translation_memory_result(
|
||||||
|
task: &PersistedTranslationTask,
|
||||||
|
unit: &OfficialTextUnitIndexUnit,
|
||||||
|
entry: &bat_core::domain::TranslationMemoryEntry,
|
||||||
|
) -> TranslationTaskUnitResult {
|
||||||
|
TranslationTaskUnitResult {
|
||||||
|
unit_id: unit.id.clone(),
|
||||||
|
source_text: unit.source_text.clone(),
|
||||||
|
translated_text: entry.translated_text.clone(),
|
||||||
|
source_kind: TranslationTaskResultSourceKind::TranslationMemory,
|
||||||
|
translation_memory_record_id: Some(entry.record_id.clone()),
|
||||||
|
provider: "translation_memory".to_string(),
|
||||||
|
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
||||||
|
translated_unix_seconds: unix_seconds_now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn translation_memory_trace(
|
||||||
|
task: &PersistedTranslationTask,
|
||||||
|
unit: &OfficialTextUnitIndexUnit,
|
||||||
|
) -> TranslationMemorySourceTrace {
|
||||||
|
TranslationMemorySourceTrace {
|
||||||
|
official_release_id: task.task.official_release_id.clone(),
|
||||||
|
unit_id: Some(unit.id.clone()),
|
||||||
|
task_id: Some(task.task.task_id.clone()),
|
||||||
|
destination: Some(unit.destination.clone()),
|
||||||
|
archive_entry: unit.archive_entry.clone(),
|
||||||
|
serialized_file: unit.serialized_file.clone(),
|
||||||
|
path_id: unit.path_id,
|
||||||
|
class_id: unit.class_id,
|
||||||
|
field_path: unit.field_path.clone(),
|
||||||
|
format: unit.format.clone(),
|
||||||
|
asset_name: unit.asset_name.clone(),
|
||||||
|
text_source_kind: unit.text_source_kind.clone(),
|
||||||
|
source_url: Some(unit.source_url.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_translation_memory_failure(
|
||||||
|
context: &WorkerTaskContext<'_>,
|
||||||
|
message: String,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
context
|
||||||
|
.stats
|
||||||
|
.translation_memory_failures
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))?
|
||||||
|
.push(message);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn record_provider_failure(
|
async fn record_provider_failure(
|
||||||
context: &WorkerTaskContext<'_>,
|
context: &WorkerTaskContext<'_>,
|
||||||
task: &PersistedTranslationTask,
|
task: &PersistedTranslationTask,
|
||||||
error: TranslationProviderError,
|
error: TranslationProviderError,
|
||||||
|
partial_results: &BTreeMap<String, TranslationTaskUnitResult>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let retryable = error.retryable && task.attempt_count < context.max_attempts;
|
let retryable = error.retryable && task.attempt_count < context.max_attempts;
|
||||||
let next_attempt =
|
let next_attempt =
|
||||||
retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs()));
|
retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs()));
|
||||||
|
let partial_results = partial_results.values().cloned().collect::<Vec<_>>();
|
||||||
context
|
context
|
||||||
.repository
|
.repository
|
||||||
.fail_claim(TranslationTaskFailure {
|
.fail_claim_with_results(
|
||||||
|
TranslationTaskFailure {
|
||||||
task_id: task.task.task_id.clone(),
|
task_id: task.task.task_id.clone(),
|
||||||
worker_id: context.worker_id.to_string(),
|
worker_id: context.worker_id.to_string(),
|
||||||
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
provider_run_id: task.provider_run_id.clone().unwrap_or_default(),
|
||||||
@@ -951,7 +1243,9 @@ async fn record_provider_failure(
|
|||||||
failure_reason: error.message.clone(),
|
failure_reason: error.message.clone(),
|
||||||
retryable,
|
retryable,
|
||||||
next_attempt_unix_seconds: next_attempt,
|
next_attempt_unix_seconds: next_attempt,
|
||||||
})
|
},
|
||||||
|
&partial_results,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败:{failure}"))?;
|
.map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败:{failure}"))?;
|
||||||
context.stats.failed_count.fetch_add(1, Ordering::Relaxed);
|
context.stats.failed_count.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -1161,6 +1455,277 @@ mod tests {
|
|||||||
assert_eq!(task.translation_results[0].translated_text, "translated-0");
|
assert_eq!(task.translation_results[0].translated_text, "translated-0");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_honors_cancellation_before_claiming_tasks() {
|
||||||
|
let (temp, queue) = fixture_root();
|
||||||
|
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||||
|
crate::official_parse::write_textunit_index_at(temp.path(), &index(temp.path())).unwrap();
|
||||||
|
let config = TranslationWorkerConfig {
|
||||||
|
concurrency: 2,
|
||||||
|
..TranslationWorkerConfig::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let result =
|
||||||
|
run_translation_worker_at_with_cancellation(temp.path(), &config, Arc::new(|| true))
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.unwrap_err().to_string().contains("已取消"));
|
||||||
|
let repository = SqliteTranslationTaskRepository::open(
|
||||||
|
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
task.task_status,
|
||||||
|
crate::translation_tasks::TranslationTaskStatus::Queued
|
||||||
|
);
|
||||||
|
assert_eq!(task.attempt_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_reuses_trusted_tm_for_part_of_a_task_and_calls_provider_for_the_rest() {
|
||||||
|
let (temp, queue) = fixture_root();
|
||||||
|
let textunit_index = index(temp.path());
|
||||||
|
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||||
|
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
|
||||||
|
|
||||||
|
let translation_memory_path = temp.path().join("translation-memory.sqlite");
|
||||||
|
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let unit = &textunit_index.units[0];
|
||||||
|
let source_context = translation_memory_context(
|
||||||
|
&unit.destination,
|
||||||
|
unit.archive_entry.as_deref(),
|
||||||
|
unit.serialized_file.as_deref(),
|
||||||
|
unit.path_id,
|
||||||
|
unit.class_id,
|
||||||
|
unit.field_path.as_deref(),
|
||||||
|
unit.format.as_deref(),
|
||||||
|
unit.asset_name.as_deref(),
|
||||||
|
unit.text_source_kind.as_deref(),
|
||||||
|
&unit.context,
|
||||||
|
);
|
||||||
|
let candidate = translation_memory
|
||||||
|
.upsert_candidate(TranslationMemoryDraft {
|
||||||
|
source_text: unit.source_text.clone(),
|
||||||
|
source_context,
|
||||||
|
translated_text: "trusted-from-tm".to_string(),
|
||||||
|
translation_source_kind: TranslationMemorySourceKind::Manual,
|
||||||
|
official_release_id: queue.official_release_id.clone(),
|
||||||
|
source_trace: TranslationMemorySourceTrace {
|
||||||
|
official_release_id: queue.official_release_id.clone(),
|
||||||
|
unit_id: Some(unit.id.clone()),
|
||||||
|
task_id: Some(queue.tasks[0].task_id.clone()),
|
||||||
|
destination: Some(unit.destination.clone()),
|
||||||
|
archive_entry: unit.archive_entry.clone(),
|
||||||
|
serialized_file: unit.serialized_file.clone(),
|
||||||
|
path_id: unit.path_id,
|
||||||
|
class_id: unit.class_id,
|
||||||
|
field_path: unit.field_path.clone(),
|
||||||
|
format: unit.format.clone(),
|
||||||
|
asset_name: unit.asset_name.clone(),
|
||||||
|
text_source_kind: unit.text_source_kind.clone(),
|
||||||
|
source_url: Some(unit.source_url.clone()),
|
||||||
|
},
|
||||||
|
provider: None,
|
||||||
|
provider_run_id: None,
|
||||||
|
observed_unix_seconds: 1,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let trusted = translation_memory
|
||||||
|
.confirm(
|
||||||
|
&candidate.record_id,
|
||||||
|
"test-reviewer",
|
||||||
|
Some("accepted".to_string()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
trusted.trust_status,
|
||||||
|
bat_core::domain::TranslationMemoryTrustStatus::Trusted
|
||||||
|
);
|
||||||
|
|
||||||
|
let fixture = temp.path().join("partial-mock.json");
|
||||||
|
std::fs::write(
|
||||||
|
&fixture,
|
||||||
|
serde_json::to_vec(&serde_json::json!({
|
||||||
|
"schema_version": 1,
|
||||||
|
"translations": {
|
||||||
|
"direct:bundle#unit:1": "translated-by-provider"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let config = TranslationWorkerConfig {
|
||||||
|
fixture_path: Some(fixture),
|
||||||
|
translation_memory_path: Some(translation_memory_path),
|
||||||
|
concurrency: 1,
|
||||||
|
retry_backoff: Duration::ZERO,
|
||||||
|
..TranslationWorkerConfig::default()
|
||||||
|
};
|
||||||
|
let report = run_translation_worker_at(temp.path(), &config)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(report.translation_memory_hit_count, 1);
|
||||||
|
assert_eq!(report.provider_unit_count, 1);
|
||||||
|
assert_eq!(report.completed_count, 1);
|
||||||
|
|
||||||
|
let repository = SqliteTranslationTaskRepository::open(
|
||||||
|
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||||
|
assert_eq!(task.translation_results.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[0].translated_text,
|
||||||
|
"trusted-from-tm"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[0].source_kind,
|
||||||
|
TranslationTaskResultSourceKind::TranslationMemory
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[0]
|
||||||
|
.translation_memory_record_id
|
||||||
|
.as_deref(),
|
||||||
|
Some(trusted.record_id.as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[1].translated_text,
|
||||||
|
"translated-by-provider"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[1].source_kind,
|
||||||
|
TranslationTaskResultSourceKind::Provider
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_retains_tm_hits_when_provider_fails_for_remaining_units() {
|
||||||
|
let (temp, queue) = fixture_root();
|
||||||
|
let textunit_index = index(temp.path());
|
||||||
|
crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap();
|
||||||
|
crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap();
|
||||||
|
|
||||||
|
let translation_memory_path = temp.path().join("translation-memory.sqlite");
|
||||||
|
let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let unit = &textunit_index.units[0];
|
||||||
|
let source_context = translation_memory_context(
|
||||||
|
&unit.destination,
|
||||||
|
unit.archive_entry.as_deref(),
|
||||||
|
unit.serialized_file.as_deref(),
|
||||||
|
unit.path_id,
|
||||||
|
unit.class_id,
|
||||||
|
unit.field_path.as_deref(),
|
||||||
|
unit.format.as_deref(),
|
||||||
|
unit.asset_name.as_deref(),
|
||||||
|
unit.text_source_kind.as_deref(),
|
||||||
|
&unit.context,
|
||||||
|
);
|
||||||
|
let candidate = translation_memory
|
||||||
|
.upsert_candidate(TranslationMemoryDraft {
|
||||||
|
source_text: unit.source_text.clone(),
|
||||||
|
source_context,
|
||||||
|
translated_text: "trusted-from-tm".to_string(),
|
||||||
|
translation_source_kind: TranslationMemorySourceKind::Manual,
|
||||||
|
official_release_id: queue.official_release_id.clone(),
|
||||||
|
source_trace: TranslationMemorySourceTrace {
|
||||||
|
official_release_id: queue.official_release_id.clone(),
|
||||||
|
unit_id: Some(unit.id.clone()),
|
||||||
|
task_id: Some(queue.tasks[0].task_id.clone()),
|
||||||
|
destination: Some(unit.destination.clone()),
|
||||||
|
archive_entry: unit.archive_entry.clone(),
|
||||||
|
serialized_file: unit.serialized_file.clone(),
|
||||||
|
path_id: unit.path_id,
|
||||||
|
class_id: unit.class_id,
|
||||||
|
field_path: unit.field_path.clone(),
|
||||||
|
format: unit.format.clone(),
|
||||||
|
asset_name: unit.asset_name.clone(),
|
||||||
|
text_source_kind: unit.text_source_kind.clone(),
|
||||||
|
source_url: Some(unit.source_url.clone()),
|
||||||
|
},
|
||||||
|
provider: None,
|
||||||
|
provider_run_id: None,
|
||||||
|
observed_unix_seconds: 1,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let trusted = translation_memory
|
||||||
|
.confirm(
|
||||||
|
&candidate.record_id,
|
||||||
|
"test-reviewer",
|
||||||
|
Some("accepted".to_string()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let fixture = temp.path().join("provider-failure.json");
|
||||||
|
std::fs::write(
|
||||||
|
&fixture,
|
||||||
|
serde_json::to_vec(&serde_json::json!({
|
||||||
|
"schema_version": 1,
|
||||||
|
"failures": {
|
||||||
|
"textunit/release-1/bundle": {
|
||||||
|
"class": "rate_limited",
|
||||||
|
"message": "fixture throttled",
|
||||||
|
"retryable": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let config = TranslationWorkerConfig {
|
||||||
|
fixture_path: Some(fixture),
|
||||||
|
translation_memory_path: Some(translation_memory_path),
|
||||||
|
concurrency: 1,
|
||||||
|
max_attempts: 1,
|
||||||
|
retry_backoff: Duration::ZERO,
|
||||||
|
..TranslationWorkerConfig::default()
|
||||||
|
};
|
||||||
|
let report = run_translation_worker_at(temp.path(), &config)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(report.translation_memory_hit_count, 1);
|
||||||
|
assert_eq!(report.provider_unit_count, 1);
|
||||||
|
assert_eq!(report.failed_count, 1);
|
||||||
|
|
||||||
|
let repository = SqliteTranslationTaskRepository::open(
|
||||||
|
SqliteTranslationTaskRepository::repository_path(temp.path()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let task = repository.find(&queue.tasks[0].task_id).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
task.task_status,
|
||||||
|
crate::translation_tasks::TranslationTaskStatus::Failed
|
||||||
|
);
|
||||||
|
assert_eq!(task.translation_results.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
task.translation_results[0]
|
||||||
|
.translation_memory_record_id
|
||||||
|
.as_deref(),
|
||||||
|
Some(trusted.record_id.as_str())
|
||||||
|
);
|
||||||
|
|
||||||
|
let handoff = crate::translation_tasks::build_translation_handoff(&queue, &[task]);
|
||||||
|
assert_eq!(
|
||||||
|
handoff.units[0].status,
|
||||||
|
crate::translation_tasks::TranslationUnitStatus::Translated
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
handoff.job.status,
|
||||||
|
crate::translation_tasks::TranslationJobStatus::Failed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() {
|
async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() {
|
||||||
let (temp, queue) = fixture_root();
|
let (temp, queue) = fixture_root();
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use crate::path_security::{
|
|||||||
use crate::{
|
use crate::{
|
||||||
LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch,
|
LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch,
|
||||||
LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository,
|
LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository,
|
||||||
TranslationTaskStatus, TranslationTaskUnitResult,
|
TranslationTaskResultSourceKind, TranslationTaskStatus, TranslationTaskUnitResult,
|
||||||
};
|
};
|
||||||
use bat_assetbundle::{
|
use bat_assetbundle::{
|
||||||
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
|
patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch,
|
||||||
@@ -72,6 +72,12 @@ pub struct TranslationWorkbenchEntry {
|
|||||||
/// Provider run that produced this translation, when imported from worker output.
|
/// Provider run that produced this translation, when imported from worker output.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub provider_run_id: Option<String>,
|
pub provider_run_id: Option<String>,
|
||||||
|
/// Source of the worker result (`provider`, `manual`, or `translation_memory`).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_source_kind: Option<String>,
|
||||||
|
/// Trusted Translation Memory record used for this translation, when applicable.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub translation_memory_record_id: Option<String>,
|
||||||
/// Worker completion time for provider-produced text.
|
/// Worker completion time for provider-produced text.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub translated_unix_seconds: Option<u64>,
|
pub translated_unix_seconds: Option<u64>,
|
||||||
@@ -616,6 +622,8 @@ fn localized_patch_metadata(entry: &TranslationWorkbenchEntry) -> LocalizedPatch
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
translation_provider: entry.translation_provider.clone(),
|
translation_provider: entry.translation_provider.clone(),
|
||||||
provider_run_id: entry.provider_run_id.clone(),
|
provider_run_id: entry.provider_run_id.clone(),
|
||||||
|
translation_source_kind: entry.translation_source_kind.clone(),
|
||||||
|
translation_memory_record_id: entry.translation_memory_record_id.clone(),
|
||||||
review_status: entry
|
review_status: entry
|
||||||
.review_status
|
.review_status
|
||||||
.clone()
|
.clone()
|
||||||
@@ -887,18 +895,30 @@ fn workbench_entry_from_worker_result(
|
|||||||
) -> TranslationWorkbenchEntry {
|
) -> TranslationWorkbenchEntry {
|
||||||
let mut entry = TranslationWorkbenchEntry::from_index(unit);
|
let mut entry = TranslationWorkbenchEntry::from_index(unit);
|
||||||
entry.translated_text = Some(result.translated_text.clone());
|
entry.translated_text = Some(result.translated_text.clone());
|
||||||
entry.translation_provider = task
|
entry.translation_provider = match result.source_kind {
|
||||||
|
TranslationTaskResultSourceKind::TranslationMemory => None,
|
||||||
|
TranslationTaskResultSourceKind::Provider | TranslationTaskResultSourceKind::Manual => task
|
||||||
.provider
|
.provider
|
||||||
.clone()
|
.clone()
|
||||||
.or_else(|| Some(result.provider.clone()))
|
.or_else(|| Some(result.provider.clone()))
|
||||||
.filter(|provider| !provider.trim().is_empty());
|
.filter(|provider| !provider.trim().is_empty()),
|
||||||
|
};
|
||||||
entry.provider_run_id = task
|
entry.provider_run_id = task
|
||||||
.provider_run_id
|
.provider_run_id
|
||||||
.clone()
|
.clone()
|
||||||
.or_else(|| Some(result.provider_run_id.clone()))
|
.or_else(|| Some(result.provider_run_id.clone()))
|
||||||
.filter(|provider_run_id| !provider_run_id.trim().is_empty());
|
.filter(|provider_run_id| !provider_run_id.trim().is_empty());
|
||||||
|
entry.translation_source_kind = Some(result.source_kind.as_str().to_string());
|
||||||
|
entry.translation_memory_record_id = result.translation_memory_record_id.clone();
|
||||||
entry.translated_unix_seconds = Some(result.translated_unix_seconds);
|
entry.translated_unix_seconds = Some(result.translated_unix_seconds);
|
||||||
entry.review_status = Some("provider_completed".to_string());
|
entry.review_status = Some(
|
||||||
|
match result.source_kind {
|
||||||
|
TranslationTaskResultSourceKind::Provider => "provider_completed",
|
||||||
|
TranslationTaskResultSourceKind::Manual => "manual_submitted",
|
||||||
|
TranslationTaskResultSourceKind::TranslationMemory => "translation_memory_reused",
|
||||||
|
}
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
entry
|
entry
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -938,6 +958,8 @@ impl TranslationWorkbenchEntry {
|
|||||||
translated_text: None,
|
translated_text: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
translated_unix_seconds: None,
|
translated_unix_seconds: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
format: unit.format.clone(),
|
format: unit.format.clone(),
|
||||||
@@ -975,6 +997,8 @@ mod tests {
|
|||||||
translated_text: None,
|
translated_text: None,
|
||||||
translation_provider: None,
|
translation_provider: None,
|
||||||
provider_run_id: None,
|
provider_run_id: None,
|
||||||
|
translation_source_kind: None,
|
||||||
|
translation_memory_record_id: None,
|
||||||
translated_unix_seconds: None,
|
translated_unix_seconds: None,
|
||||||
review_status: None,
|
review_status: None,
|
||||||
format: Some("plain".to_string()),
|
format: Some("plain".to_string()),
|
||||||
|
|||||||
Reference in New Issue
Block a user