From 7d6389806be06ebe868d38810b5f3031de035c90 Mon Sep 17 00:00:00 2001 From: Yuyi-Oak <1722157266@qq.com> Date: Sun, 6 Sep 2026 22:48:51 +0800 Subject: [PATCH] feat(translation): add Rust Translation Memory and config migration --- core/src/domain/mod.rs | 6 + core/src/domain/translation_memory.rs | 228 ++++ core/src/repositories/mod.rs | 2 + .../translation_memory_repository.rs | 47 + infrastructure/src/bin/bat/app.rs | 861 +++++++++--- infrastructure/src/bin/bat/app_tests.rs | 428 +++++- infrastructure/src/bin/bat/config_file.rs | 1182 +++++++++++++++++ infrastructure/src/bin/bat/report_output.rs | 7 + infrastructure/src/bin/bat/task_registry.rs | 17 +- infrastructure/src/bin/bat/terminal_output.rs | 22 +- .../src/bin/bat/translation_query.rs | 244 ++++ .../src/bin/bat/workflow_commands.rs | 24 +- infrastructure/src/lib.rs | 30 +- infrastructure/src/localized_patch.rs | 24 + infrastructure/src/translation_memory.rs | 859 ++++++++++++ infrastructure/src/translation_tasks.rs | 78 +- infrastructure/src/translation_worker.rs | 701 +++++++++- infrastructure/src/translation_workflow.rs | 38 +- 18 files changed, 4414 insertions(+), 384 deletions(-) create mode 100644 core/src/domain/translation_memory.rs create mode 100644 core/src/repositories/translation_memory_repository.rs create mode 100644 infrastructure/src/bin/bat/config_file.rs create mode 100644 infrastructure/src/translation_memory.rs diff --git a/core/src/domain/mod.rs b/core/src/domain/mod.rs index 41af38a..5ad4642 100644 --- a/core/src/domain/mod.rs +++ b/core/src/domain/mod.rs @@ -4,6 +4,7 @@ pub mod game_client; pub mod game_version; pub mod resource; pub mod translation; +pub mod translation_memory; pub use game_client::{ClientStatus, GameClient, GameRegion}; pub use game_version::{GameVersion, UnityVersion}; @@ -14,3 +15,8 @@ pub use translation::{ ExtractedText, SourceText, TextContext, TextMetadata, TextSource, TranslatedText, TranslationStatus, }; +pub use translation_memory::{ + TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry, + TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind, + TranslationMemorySourceTrace, TranslationMemorySummary, TranslationMemoryTrustStatus, +}; diff --git a/core/src/domain/translation_memory.rs b/core/src/domain/translation_memory.rs new file mode 100644 index 0000000..a51ccae --- /dev/null +++ b/core/src/domain/translation_memory.rs @@ -0,0 +1,228 @@ +//! Translation Memory 领域对象。 + +use std::collections::BTreeMap; + +/// TM 记录的来源类型。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationMemorySourceKind { + /// 来自 provider 输出。 + Provider, + /// 来自人工确认。 + Manual, + /// 来自外部导入。 + Imported, +} + +impl TranslationMemorySourceKind { + /// 返回稳定的持久化标签。 + pub const fn as_str(&self) -> &'static str { + match self { + Self::Provider => "provider", + Self::Manual => "manual", + Self::Imported => "imported", + } + } +} + +/// TM 记录的可信状态。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationMemoryTrustStatus { + /// 候选记录，不能自动复用。 + Candidate, + /// 已确认可信，可在强匹配时自动复用。 + Trusted, + /// 已被后续记录取代。 + Superseded, + /// 已明确拒绝。 + Rejected, +} + +impl TranslationMemoryTrustStatus { + /// 返回稳定的持久化标签。 + pub const fn as_str(&self) -> &'static str { + match self { + Self::Candidate => "candidate", + Self::Trusted => "trusted", + Self::Superseded => "superseded", + Self::Rejected => "rejected", + } + } +} + +/// TM 查询结果的匹配类型。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationMemoryMatchKind { + /// 原始 source 和上下文都完全匹配，且记录可信，可自动复用。 + StrongExact, + /// 原始 source 完全匹配，但上下文不同或不足，不能自动复用。 + CandidateExact, + /// 原始 source 匹配，但上下文不兼容，不能自动复用。 + SourceOnly, +} + +impl TranslationMemoryMatchKind { + /// 返回稳定的查询结果标签。 + pub const fn as_str(&self) -> &'static str { + match self { + Self::StrongExact => "strong_exact", + Self::CandidateExact => "candidate_exact", + Self::SourceOnly => "source_only", + } + } +} + +/// 稳定的上下文键值。 +pub type TranslationMemoryContext = BTreeMap; + +/// TM 记录的 TextUnit / provider 溯源信息。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TranslationMemorySourceTrace { + /// 源官方 release ID。 + pub official_release_id: String, + /// 来源 TextUnit ID。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unit_id: Option, + /// 来源任务 ID。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub task_id: Option, + /// 源资源 destination。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub destination: Option, + /// 源 ZIP/archive entry。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub archive_entry: Option, + /// Unity serialized file。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub serialized_file: Option, + /// Unity object path ID。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path_id: Option, + /// Unity class ID。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub class_id: Option, + /// TypeTree 字段路径。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub field_path: Option, + /// TextUnit format。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + /// TextAsset 名称。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset_name: Option, + /// TextUnit 来源类型。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_source_kind: Option, + /// 源 URL。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, +} + +/// TM 记录的候选输入。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TranslationMemoryDraft { + /// 原始 source text。 + pub source_text: String, + /// 稳定上下文。 + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub source_context: TranslationMemoryContext, + /// 译文。 + pub translated_text: String, + /// 译文来源类型。 + pub translation_source_kind: TranslationMemorySourceKind, + /// 源官方 release。 + pub official_release_id: String, + /// 原始 TextUnit / provider 溯源。 + pub source_trace: TranslationMemorySourceTrace, + /// provider。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// provider run。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_run_id: Option, + /// 创建时间。 + pub observed_unix_seconds: u64, +} + +/// 持久化 TM 记录。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TranslationMemoryEntry { + /// 稳定记录 ID。 + pub record_id: String, + /// 原始 source text。 + pub source_text: String, + /// source text hash。 + pub source_hash: String, + /// 保守归一化后的 source text，仅用于辅助查询。 + pub normalized_source_text: String, + /// 稳定上下文。 + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub source_context: TranslationMemoryContext, + /// 上下文 hash。 + pub source_context_hash: String, + /// 译文。 + pub translated_text: String, + /// 译文来源类型。 + pub translation_source_kind: TranslationMemorySourceKind, + /// 当前可信状态。 + pub trust_status: TranslationMemoryTrustStatus, + /// 源官方 release。 + pub official_release_id: String, + /// 原始 TextUnit / provider 溯源。 + pub source_trace: TranslationMemorySourceTrace, + /// provider。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// provider run。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_run_id: Option, + /// 创建时间。 + pub created_unix_seconds: u64, + /// 更新时间。 + pub updated_unix_seconds: u64, + /// 可信确认时间。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trusted_unix_seconds: Option, + /// 可信确认人。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trusted_by: Option, + /// 可信确认说明。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trusted_reason: Option, + /// 该记录替代了哪条记录。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supersedes_record_id: Option, + /// 该记录被哪条记录替代。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub superseded_by_record_id: Option, +} + +/// TM 查询结果。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TranslationMemoryMatch { + /// 记录本体。 + pub entry: TranslationMemoryEntry, + /// 匹配类型。 + pub match_kind: TranslationMemoryMatchKind, + /// 是否允许自动复用。 + pub can_auto_reuse: bool, +} + +/// TM 仓储摘要。 +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TranslationMemorySummary { + /// schema 版本。 + pub schema_version: u32, + /// 记录总数。 + pub record_count: u64, + /// 可信记录数。 + pub trusted_count: u64, + /// 候选记录数。 + pub candidate_count: u64, + /// 已替代记录数。 + pub superseded_count: u64, + /// 已拒绝记录数。 + pub rejected_count: u64, +} diff --git a/core/src/repositories/mod.rs b/core/src/repositories/mod.rs index 0a2d17f..a0802de 100644 --- a/core/src/repositories/mod.rs +++ b/core/src/repositories/mod.rs @@ -4,8 +4,10 @@ pub mod cas_repository; pub mod resource_repository; +pub mod translation_memory_repository; pub mod translation_repository; pub use cas_repository::CasRepository; pub use resource_repository::ResourceRepository; +pub use translation_memory_repository::TranslationMemoryRepository; pub use translation_repository::TranslationRepository; diff --git a/core/src/repositories/translation_memory_repository.rs b/core/src/repositories/translation_memory_repository.rs new file mode 100644 index 0000000..5a434fe --- /dev/null +++ b/core/src/repositories/translation_memory_repository.rs @@ -0,0 +1,47 @@ +//! Translation Memory 仓储契约。 + +use crate::domain::{ + TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry, + TranslationMemoryMatch, TranslationMemorySummary, +}; +use async_trait::async_trait; + +/// 跨 official release 持久化的 Translation Memory 仓储。 +/// +/// 该契约只描述 V1 的精确查询和明确人工确认。仓储实现不得把 +/// `TranslationTaskStatus::Completed` 或 provider 成功隐式解释为 trusted。 +#[async_trait] +pub trait TranslationMemoryRepository: Send + Sync { + /// 保存一条 provider/manual/imported 译文候选。 + /// + /// 相同 source、上下文、译文、来源 release 和来源类型的重复写入必须幂等； + /// 已 trusted 的记录不得被普通候选静默覆盖。 + async fn upsert_candidate( + &self, + draft: TranslationMemoryDraft, + ) -> crate::Result; + + /// 按原始 source text 和上下文查询精确匹配。 + /// + /// 实现可以返回 source 归一化后但原文不同的辅助候选，但这类结果不能自动复用。 + async fn find_matches( + &self, + source_text: &str, + source_context: &TranslationMemoryContext, + limit: usize, + ) -> crate::Result>; + + /// 显式确认一条记录为 trusted。 + async fn confirm( + &self, + record_id: &str, + reviewer: &str, + reason: Option, + ) -> crate::Result; + + /// 按稳定记录 ID 读取一条 TM 记录。 + async fn find(&self, record_id: &str) -> crate::Result; + + /// 读取数据库和记录统计。 + async fn summary(&self) -> crate::Result; +} diff --git a/infrastructure/src/bin/bat/app.rs b/infrastructure/src/bin/bat/app.rs index 36617c2..f3d54f3 100644 --- a/infrastructure/src/bin/bat/app.rs +++ b/infrastructure/src/bin/bat/app.rs @@ -52,6 +52,8 @@ use std::sync::{mpsc, Arc, Condvar, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +#[path = "config_file.rs"] +mod config_file; #[path = "patch_commands.rs"] mod patch_commands; #[path = "readonly_query.rs"] @@ -90,7 +92,9 @@ use terminal_output::RotatingStructuredLogger; #[cfg(test)] use terminal_output::STARTUP_BANNER; use translation_query::{ - build_translation_handoff_report, build_translation_tasks_report, textunit_query_json, + build_translation_handoff_report, build_translation_memory_confirm_report, + build_translation_memory_query_report, build_translation_memory_summary_report, + build_translation_tasks_report, run_translation_memory_command, textunit_query_json, update_translation_task_status_report, }; use workflow_commands::{ @@ -145,7 +149,6 @@ pub fn main() { } fn run() -> anyhow::Result { - bootstrap_env_file(); let options = parse_args()?; if matches!( options.command, @@ -220,6 +223,12 @@ fn run() -> anyhow::Result { run_translation_proofread(&options)?; Ok(0) } + CliCommand::TranslationMemorySummary + | CliCommand::TranslationMemoryQuery + | CliCommand::TranslationMemoryConfirm => { + run_translation_memory_command(&options)?; + Ok(0) + } CliCommand::TranslationWorker => { run_repeated_workflow(&options, "translation-worker", run_translation_worker)?; Ok(0) @@ -375,6 +384,14 @@ struct CliOptions { translation_from_worker: bool, translation_provider: Option, translation_fixture: Option, + translation_memory_path: Option, + translation_memory_option_explicit: bool, + translation_memory_command_option_explicit: bool, + translation_memory_source_text: Option, + translation_memory_context_json: Option, + translation_memory_record_id: Option, + translation_memory_reviewer: Option, + translation_memory_reason: Option, worker_concurrency: usize, worker_max_attempts: u32, worker_lease_seconds: u64, @@ -442,7 +459,7 @@ struct CliOptions { unityfs_replacement_value: Option, unityfs_expected_semantic_value: Option, write_patch_option_explicit: bool, - /// 环境变量（含 .env）应用后、命令行解析前的配置快照。 + /// `config.toml` 和环境变量应用后、命令行解析前的配置快照。 /// 工具/代理"是否命令行显式传入"的判断以它为基线。 env_baseline_config: OfficialUpdateConfig, } @@ -470,6 +487,14 @@ impl Default for CliOptions { translation_from_worker: false, translation_provider: None, translation_fixture: None, + translation_memory_path: None, + translation_memory_option_explicit: false, + translation_memory_command_option_explicit: false, + translation_memory_source_text: None, + translation_memory_context_json: None, + translation_memory_record_id: None, + translation_memory_reviewer: None, + translation_memory_reason: None, worker_concurrency: DEFAULT_TRANSLATION_CONCURRENCY, worker_max_attempts: DEFAULT_TRANSLATION_MAX_ATTEMPTS, worker_lease_seconds: DEFAULT_TRANSLATION_LEASE_SECONDS, @@ -562,6 +587,9 @@ enum CliCommand { TranslationTaskUpdate, TranslationWorker, TranslationProofread, + TranslationMemorySummary, + TranslationMemoryQuery, + TranslationMemoryConfirm, Repack, PublishLocalized, LocalizedRollback, @@ -619,6 +647,14 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> { let daemon_control = Some(new_daemon_control()); // 进程内同步锁：watch 循环与任务 worker 在跑同步前都获取它，互相等待而非撞文件锁失败。 let sync_lock = Arc::new(Mutex::new(())); + let daemon_translation_worker_config = if options.daemon_child { + Some(translation_worker_config_from_options( + &options, + &format!("bat-daemon-worker-{}", std::process::id()), + )?) + } else { + None + }; let (_task_worker, _task_context, _rpc_server) = if options.daemon_child { let control = daemon_control .as_ref() @@ -639,6 +675,7 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> { registry, queue: task_tx, base_config: options.config.clone(), + translation_worker_config: daemon_translation_worker_config.unwrap_or_default(), sync_lock: Arc::clone(&sync_lock), restart_controller: spawn_daemon_restart_controller, }; @@ -1067,6 +1104,9 @@ const RPC_METHOD_TRANSLATION_HANDOFF: &str = "translation.handoff"; const RPC_METHOD_TRANSLATION_TASK_UPDATE: &str = "translation.task.update"; const RPC_METHOD_TRANSLATION_PROOFREAD: &str = "translation.proofread"; const RPC_METHOD_TRANSLATION_WORKER_RUN: &str = "translation.worker.run"; +const RPC_METHOD_TRANSLATION_MEMORY_SUMMARY: &str = "translation.memory.summary"; +const RPC_METHOD_TRANSLATION_MEMORY_QUERY: &str = "translation.memory.query"; +const RPC_METHOD_TRANSLATION_MEMORY_CONFIRM: &str = "translation.memory.confirm"; const RPC_METHOD_LOCALIZED_STATUS: &str = "localized.status"; const RPC_METHOD_LOCALIZED_PUBLISH: &str = "localized.publish"; const RPC_METHOD_LOCALIZED_ROLLBACK: &str = "localized.rollback"; @@ -2079,8 +2119,47 @@ fn dispatch_rpc_method( .and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from)), ) } + RPC_METHOD_TRANSLATION_MEMORY_SUMMARY => translation_memory_rpc_envelope( + request_id, + translation_memory_summary_rpc_report( + state_dir, + &tasks.base_config.output_root, + tasks + .translation_worker_config + .translation_memory_path + .as_deref(), + request.params.as_ref(), + ), + ), + RPC_METHOD_TRANSLATION_MEMORY_QUERY => translation_memory_rpc_envelope( + request_id, + translation_memory_query_rpc_report( + state_dir, + &tasks.base_config.output_root, + tasks + .translation_worker_config + .translation_memory_path + .as_deref(), + request.params.as_ref(), + ), + ), + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM => translation_memory_rpc_envelope( + request_id, + translation_memory_confirm_rpc_report( + state_dir, + &tasks.base_config.output_root, + tasks + .translation_worker_config + .translation_memory_path + .as_deref(), + request.params.as_ref(), + ), + ), RPC_METHOD_TRANSLATION_WORKER_RUN => { - let config = match rpc_translation_worker_config(request.params.as_ref()) { + let config = match rpc_translation_worker_config_with_defaults( + request.params.as_ref(), + &tasks.translation_worker_config, + ) { Ok(config) => config, Err(error) => return rpc_envelope_error(request_id, error), }; @@ -3309,8 +3388,20 @@ fn normalize_optional_rpc_string(value: Option) -> Option { .filter(|value| !value.is_empty()) } +#[cfg(test)] fn rpc_translation_worker_config( params: Option<&serde_json::Value>, +) -> Result { + let defaults = TranslationWorkerConfig { + worker_id: "bat-rpc-worker".to_string(), + ..TranslationWorkerConfig::default() + }; + rpc_translation_worker_config_with_defaults(params, &defaults) +} + +fn rpc_translation_worker_config_with_defaults( + params: Option<&serde_json::Value>, + defaults: &TranslationWorkerConfig, ) -> Result { let empty = serde_json::json!({}); let params = params.unwrap_or(&empty); @@ -3326,7 +3417,7 @@ fn rpc_translation_worker_config( &["provider", "translation_provider"], "provider", )? - .unwrap_or_else(|| TranslationProviderKind::Mock.as_str().to_string()); + .unwrap_or_else(|| defaults.provider.as_str().to_string()); let provider = TranslationProviderKind::parse(&provider).ok_or_else(|| { ApiError::new( ErrorCode::RPC_INVALID_PARAMS, @@ -3343,7 +3434,7 @@ fn rpc_translation_worker_config( ], "concurrency", )? - .unwrap_or(DEFAULT_TRANSLATION_CONCURRENCY); + .unwrap_or(defaults.concurrency); let max_attempts = rpc_translation_worker_u32_param( params, &[ @@ -3353,7 +3444,7 @@ fn rpc_translation_worker_config( ], "max_attempts", )? - .unwrap_or(DEFAULT_TRANSLATION_MAX_ATTEMPTS); + .unwrap_or(defaults.max_attempts); let lease_seconds = rpc_translation_worker_u64_param( params, &[ @@ -3363,7 +3454,7 @@ fn rpc_translation_worker_config( ], "lease_seconds", )? - .unwrap_or(DEFAULT_TRANSLATION_LEASE_SECONDS); + .unwrap_or(defaults.lease_seconds); let retry_backoff_seconds = rpc_translation_worker_u64_param( params, &[ @@ -3373,12 +3464,13 @@ fn rpc_translation_worker_config( ], "retry_backoff_seconds", )? - .unwrap_or(DEFAULT_TRANSLATION_RETRY_BACKOFF.as_secs()); + .unwrap_or(defaults.retry_backoff.as_secs()); let max_tasks = rpc_translation_worker_usize_param( params, &["max_tasks", "worker_max_tasks", "translation_max_tasks"], "max_tasks", - )?; + )? + .or(defaults.max_tasks); let config = TranslationWorkerConfig { provider, fixture_path: rpc_translation_worker_string_param( @@ -3392,7 +3484,8 @@ fn rpc_translation_worker_config( ], "fixture_path", )? - .map(PathBuf::from), + .map(PathBuf::from) + .or_else(|| defaults.fixture_path.clone()), concurrency, max_attempts, lease_seconds, @@ -3403,7 +3496,14 @@ fn rpc_translation_worker_config( &["worker_id", "translation_worker_id"], "worker_id", )? - .unwrap_or_else(|| "bat-rpc-worker".to_string()), + .unwrap_or_else(|| defaults.worker_id.clone()), + translation_memory_path: rpc_translation_worker_string_param( + params, + &["translation_memory_path", "translation_memory", "tm_path"], + "translation_memory_path", + )? + .map(PathBuf::from) + .or_else(|| defaults.translation_memory_path.clone()), }; config.validate().map_err(|error| { ApiError::new( @@ -3415,6 +3515,280 @@ fn rpc_translation_worker_config( Ok(config) } +fn translation_memory_rpc_path( + state_dir: &Path, + output_root: &Path, + default_translation_memory_path: Option<&Path>, + params: Option<&serde_json::Value>, + method: &'static str, +) -> Result { + if let Some(path) = translation_memory_rpc_string_param( + params, + &["translation_memory_path", "translation_memory", "tm_path"], + method, + "translation_memory_path", + )? { + return lexical_absolute(Path::new(path)) + .map_err(|error| ApiError::new(ErrorCode::INTERNAL, method, error)); + } + if let Some(path) = default_translation_memory_path { + return lexical_absolute(path) + .map_err(|error| ApiError::new(ErrorCode::INTERNAL, method, error)); + } + let (_, version_state) = read_daemon_resource_state(state_dir) + .map_err(|error| ApiError::new(ErrorCode::INTERNAL, method, error.to_string()))?; + if let Some(record) = version_state + .as_ref() + .and_then(|state| state.current_completed_version.as_ref()) + { + return Ok(bat_infrastructure::translation_memory_repository_path( + &record.resource_root, + )); + } + Ok(bat_infrastructure::translation_memory_repository_path( + output_root, + )) +} + +fn translation_memory_rpc_envelope( + request_id: String, + result: Result, +) -> RpcEnvelope { + match result { + Ok(data) => rpc_envelope_ok(request_id, "ok", data), + Err(error) => rpc_envelope_error(request_id, error), + } +} + +fn translation_memory_rpc_params<'a>( + params: Option<&'a serde_json::Value>, + method: &'static str, +) -> Result>, ApiError> { + match params { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::Object(params)) => Ok(Some(params)), + Some(_) => Err(ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + method, + "params 必须是 JSON object", + )), + } +} + +fn translation_memory_rpc_value<'a>( + params: Option<&'a serde_json::Value>, + aliases: &[&str], + method: &'static str, +) -> Result, ApiError> { + let Some(params) = translation_memory_rpc_params(params, method)? else { + return Ok(None); + }; + for alias in aliases { + if let Some(value) = params.get(*alias) { + return Ok(Some(value)); + } + } + Ok(None) +} + +fn translation_memory_rpc_string_param<'a>( + params: Option<&'a serde_json::Value>, + aliases: &[&str], + method: &'static str, + label: &str, +) -> Result, ApiError> { + let Some(value) = translation_memory_rpc_value(params, aliases, method)? else { + return Ok(None); + }; + if value.is_null() { + return Ok(None); + } + let Some(value) = value.as_str() else { + return Err(ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + method, + format!("{label} 必须是字符串"), + )); + }; + Ok((!value.trim().is_empty()).then_some(value.trim())) +} + +fn translation_memory_rpc_context( + params: Option<&serde_json::Value>, +) -> Result { + let Some(value) = translation_memory_rpc_value( + params, + &["source_context", "context"], + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + )? + else { + return Ok(Default::default()); + }; + if value.is_null() { + return Ok(Default::default()); + } + if !value.is_object() { + return Err(ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + "source_context 必须是 JSON object", + )); + } + serde_json::from_value(value.clone()).map_err(|error| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + format!("source_context 无效：{error}"), + ) + }) +} + +fn translation_memory_rpc_limit(params: Option<&serde_json::Value>) -> Result { + let limit = match translation_memory_rpc_value( + params, + &["limit"], + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + )? { + None | Some(serde_json::Value::Null) => 100, + Some(value) => value.as_u64().ok_or_else(|| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + "limit 必须是非负整数 JSON number", + ) + })?, + }; + let limit = usize::try_from(limit).map_err(|error| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + format!("limit 无效：{error}"), + ) + })?; + if !(1..=1000).contains(&limit) { + return Err(ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + "limit 必须在 1..=1000 范围内", + )); + } + Ok(limit) +} + +fn translation_memory_summary_rpc_report( + state_dir: &Path, + output_root: &Path, + default_translation_memory_path: Option<&Path>, + params: Option<&serde_json::Value>, +) -> Result { + let path = translation_memory_rpc_path( + state_dir, + output_root, + default_translation_memory_path, + params, + RPC_METHOD_TRANSLATION_MEMORY_SUMMARY, + )?; + build_translation_memory_summary_report(&path).map_err(|error| { + ApiError::new( + ErrorCode::INTERNAL, + RPC_METHOD_TRANSLATION_MEMORY_SUMMARY, + error.to_string(), + ) + }) +} + +fn translation_memory_query_rpc_report( + state_dir: &Path, + output_root: &Path, + default_translation_memory_path: Option<&Path>, + params: Option<&serde_json::Value>, +) -> Result { + let source_text = translation_memory_rpc_string_param( + params, + &["source_text", "tm_source_text"], + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + "source_text", + )? + .ok_or_else(|| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + "translation.memory.query 缺少 source_text", + ) + })?; + let context = translation_memory_rpc_context(params)?; + let limit = translation_memory_rpc_limit(params)?; + let path = translation_memory_rpc_path( + state_dir, + output_root, + default_translation_memory_path, + params, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + )?; + build_translation_memory_query_report(&path, source_text, &context, limit).map_err(|error| { + ApiError::new( + ErrorCode::INTERNAL, + RPC_METHOD_TRANSLATION_MEMORY_QUERY, + error.to_string(), + ) + }) +} + +fn translation_memory_confirm_rpc_report( + state_dir: &Path, + output_root: &Path, + default_translation_memory_path: Option<&Path>, + params: Option<&serde_json::Value>, +) -> Result { + let record_id = translation_memory_rpc_string_param( + params, + &["record_id", "tm_record_id"], + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + "record_id", + )? + .ok_or_else(|| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + "translation.memory.confirm 缺少 record_id", + ) + })?; + let reviewer = translation_memory_rpc_string_param( + params, + &["reviewer", "tm_reviewer"], + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + "reviewer", + )? + .ok_or_else(|| { + ApiError::new( + ErrorCode::RPC_INVALID_PARAMS, + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + "translation.memory.confirm 缺少 reviewer", + ) + })?; + let reason = translation_memory_rpc_string_param( + params, + &["reason", "tm_reason"], + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + "reason", + )? + .map(str::to_string); + let path = translation_memory_rpc_path( + state_dir, + output_root, + default_translation_memory_path, + params, + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + )?; + build_translation_memory_confirm_report(&path, record_id, reviewer, reason).map_err(|error| { + ApiError::new( + ErrorCode::INTERNAL, + RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + error.to_string(), + ) + }) +} + fn rpc_translation_worker_string_param( params: &serde_json::Value, aliases: &[&str], @@ -4308,6 +4682,7 @@ fn run_daemon_restart(options: &CliOptions, command_name: &'static str) -> anyho let has_explicit_options = options.sync_option_explicit || options.output_explicit || options.proxy_option_explicit + || options.translation_worker_option_explicit || tools_are_non_default(&options.config, &options.env_baseline_config); if command_name == "reload" && !has_explicit_options && daemon_rpc_available(&options.state_dir) { @@ -5722,6 +6097,34 @@ fn daemon_child_args(options: &CliOptions) -> Vec { } else { args.push("--no-repair".to_string()); } + if let Some(provider) = options.translation_provider.as_deref() { + args.push("--translation-provider".to_string()); + args.push(provider.to_string()); + } + if let Some(fixture) = options.translation_fixture.as_ref() { + args.push("--translation-fixture".to_string()); + args.push(fixture.to_string_lossy().to_string()); + } + if let Some(path) = options.translation_memory_path.as_ref() { + args.push("--translation-memory-path".to_string()); + args.push(path.to_string_lossy().to_string()); + } + args.push("--worker-concurrency".to_string()); + args.push(options.worker_concurrency.to_string()); + args.push("--worker-max-attempts".to_string()); + args.push(options.worker_max_attempts.to_string()); + args.push("--worker-lease-seconds".to_string()); + args.push(options.worker_lease_seconds.to_string()); + args.push("--worker-retry-backoff-seconds".to_string()); + args.push(options.worker_retry_backoff.as_secs().to_string()); + if let Some(max_tasks) = options.worker_max_tasks { + args.push("--worker-max-tasks".to_string()); + args.push(max_tasks.to_string()); + } + if let Some(worker_id) = options.worker_id.as_deref() { + args.push("--worker-id".to_string()); + args.push(worker_id.to_string()); + } args.push("--state-dir".to_string()); args.push(options.state_dir.to_string_lossy().to_string()); args.push("--daemon-child".to_string()); @@ -5926,169 +6329,7 @@ fn should_print_status(status: OfficialUpdateStatus, quiet_up_to_date: bool) -> !(quiet_up_to_date && status == OfficialUpdateStatus::UpToDate) } -/// `.env` 配置文件名（位于 bat 二进制所在目录）。 -const ENV_FILE_NAME: &str = ".env"; - -/// 设为 `1` 时完全跳过 `.env` 的生成与加载（测试与特殊部署场景用）。 -const SKIP_ENV_FILE_VAR: &str = "BAT_SKIP_ENV_FILE"; - -/// 首次启动释放的 `.env` 配置模板。 -const ENV_TEMPLATE: &str = r#"# BlueArchive Toolkit 配置文件（bat 首次启动自动生成） -# -# 直接运行 `bat`（无参数）时会按本文件配置启动。 -# 优先级：命令行参数 > 进程环境变量 > 本文件 > 内置默认值。 -# 布尔值支持 1/0/true/false/yes/no/on/off；井号开头为注释。 -# 设 BAT_SKIP_ENV_FILE=1 可让 bat 完全忽略本文件。 - -# ---- 基本配置 ---- -# 官方原版资源发布根目录（默认 ./bat-resources，相对当前工作目录） -BAT_OUTPUT=./bat-resources -# 汉化产物输出根目录（默认 ./bat-localized，与官方原版资源分离） -BAT_LOCALIZED_OUTPUT=./bat-localized -# 启用官方 release 导入 CAS + ResourceRepository；默认关闭。 -BAT_IMPORT_REPOSITORY=0 -# 官方资源 CAS 目录；未设置时默认 /.cas -BAT_IMPORT_CAS_ROOT= -# 官方资源 SQLite 索引；未设置时默认 /resources.sqlite -BAT_IMPORT_RESOURCE_DB= -# 自动发现 app-version / connection-group / server-info（无参启动建议保持 1） -BAT_AUTO_DISCOVER=1 -# 后台状态目录（bat.sock / 日志 / 任务历史等；默认 /tmp/bat-pid） -#BAT_STATE_DIR=/tmp/bat-pid -# 启动即进入常驻模式：watch（前台常驻）或 daemon（后台自托管）。 -# 只对无子命令的 `bat` 生效；同时为 1 时 daemon 优先。 -#BAT_WATCH=0 -#BAT_DAEMON=0 -# 正常检查间隔与失败重试间隔（秒） -#BAT_INTERVAL_SECONDS=3600 -#BAT_ERROR_RETRY_SECONDS=60 -# ---- 网络 ---- -# 显式代理 URL（支持 http/https/socks4/socks4a/socks5/socks5h）。 -# 不设则自动检测 HTTPS_PROXY / ALL_PROXY / HTTP_PROXY（也可写在本文件里）。 -#BAT_PROXY=http://127.0.0.1:7897 -# 设为 1 时强制直连（忽略一切代理配置） -#BAT_NO_PROXY=0 - -# ---- 同步参数（通常保持自动发现，无需手动指定）---- -#BAT_APP_VERSION= -#BAT_CONNECTION_GROUP= -#BAT_LAUNCHER_VERSION= -# 逗号分隔：windows,android -#BAT_PLATFORMS=windows,android -#BAT_CURL=curl -# 最大并发下载数（默认 8；范围 1..=256） -#BAT_DOWNLOAD_CONCURRENCY=8 -#BAT_UNZIP=unzip - -# ---- 翻译 provider worker ---- -# provider：mock（本地 fixture）或 crowdin（读取 CROWDIN_* 环境变量） -#BAT_TRANSLATION_PROVIDER=mock -#BAT_TRANSLATION_FIXTURE= -#BAT_TRANSLATION_CONCURRENCY=8 -#BAT_TRANSLATION_MAX_ATTEMPTS=3 -#BAT_TRANSLATION_LEASE_SECONDS=300 -#BAT_TRANSLATION_RETRY_BACKOFF_SECONDS=5 -#BAT_TRANSLATION_MAX_TASKS= -#BAT_TRANSLATION_WORKER_ID= - -# ---- 输出 ---- -# 设为 1 时输出机器可读 JSON（默认人类可读） -#BAT_JSON=0 -# 远端与本地一致时是否静默（watch/daemon 模式默认 1） -#BAT_QUIET_UP_TO_DATE= - -# ---- Redis（预留，当前未接入）---- -# 任务历史当前持久化在 /bat-tasks.json； -# Redis 任务后端落地后以下配置才会生效。 -#BAT_REDIS_URL=redis://127.0.0.1:6379 -#BAT_REDIS_PASSWORD= -"#; - -/// `.env` 引导：首次启动时在二进制所在目录释放配置模板，之后每次启动把其中的 -/// 键加载为进程环境变量（不覆盖已存在的环境变量，保持"环境变量 > .env"优先级）。 -/// -/// 任何失败只在 stderr 警告、不中断启动——`.env` 是便利层，不是启动硬依赖。 -fn bootstrap_env_file() { - if env::var(SKIP_ENV_FILE_VAR).map(|value| value == "1") == Ok(true) { - return; - } - let Ok(exe_path) = env::current_exe() else { - return; - }; - let Some(exe_dir) = exe_path.parent() else { - return; - }; - let path = exe_dir.join(ENV_FILE_NAME); - if !path.exists() { - match write_env_template(&path) { - Ok(()) => terminal_output::print_env_template_created(&path), - Err(error) => { - terminal_output::print_env_template_warning(&path, error); - return; - } - } - } - match fs::read_to_string(&path) { - Ok(content) => apply_env_file(&content), - Err(error) => terminal_output::print_env_read_warning(&path, error), - } -} - -/// 以 `create_new` 原子创建模板文件，避免并发启动时互相覆盖；unix 下限制 `0600` -/// 权限（`.env` 可能保存代理凭据等敏感配置）。 -fn write_env_template(path: &Path) -> std::io::Result<()> { - let mut open_options = OpenOptions::new(); - open_options.write(true).create_new(true); - #[cfg(unix)] - open_options.mode(PRIVATE_FILE_MODE); - let mut file = open_options.open(path)?; - file.write_all(ENV_TEMPLATE.as_bytes()) -} - -/// 解析 `.env` 内容，把进程环境里尚不存在的键设为环境变量。 -fn apply_env_file(content: &str) { - for (line_number, raw_line) in content.lines().enumerate() { - let line = raw_line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((key, value)) = parse_env_line(line) else { - terminal_output::print_env_parse_warning(line_number + 1, raw_line); - continue; - }; - if env::var_os(&key).is_none() { - env::set_var(&key, value); - } - } -} - -/// 解析单行 `KEY=VALUE`。key 须为 `[A-Za-z_][A-Za-z0-9_]*`；值两侧的成对 -/// 单/双引号会剥除。不支持 `export` 前缀和多行值。 -fn parse_env_line(line: &str) -> Option<(String, String)> { - let (key, value) = line.split_once('=')?; - let key = key.trim(); - let valid_key = !key.is_empty() - && key.chars().enumerate().all(|(index, character)| { - character == '_' - || character.is_ascii_alphabetic() - || (index > 0 && character.is_ascii_digit()) - }); - if !valid_key { - return None; - } - let mut value = value.trim(); - if value.len() >= 2 { - let bytes = value.as_bytes(); - let quoted = (bytes[0] == b'"' && bytes[value.len() - 1] == b'"') - || (bytes[0] == b'\'' && bytes[value.len() - 1] == b'\''); - if quoted { - value = &value[1..value.len() - 1]; - } - } - Some((key.to_string(), value.to_string())) -} - -/// `.env`/环境变量提供的运行模式开关（延迟到命令确定后应用；值型配置 +/// `config.toml`/环境变量提供的运行模式开关（延迟到命令确定后应用；值型配置 /// 由 [`apply_bat_env_overrides`] 直接写入 options）。 struct EnvModeOverrides { watch: bool, @@ -6174,6 +6415,11 @@ fn apply_bat_env_overrides( if let Some(v) = value("BAT_TRANSLATION_FIXTURE") { options.translation_fixture = Some(PathBuf::from(v)); } + if let Some(v) = + value("BAT_TRANSLATION_MEMORY_PATH").or_else(|| value("BAT_TRANSLATION_MEMORY")) + { + options.translation_memory_path = Some(PathBuf::from(v)); + } if let Some(v) = value("BAT_TRANSLATION_CONCURRENCY") { options.worker_concurrency = parse_translation_worker_concurrency(&v, "环境变量 BAT_TRANSLATION_CONCURRENCY")?; @@ -6235,25 +6481,33 @@ fn apply_bat_env_overrides( } fn parse_args() -> anyhow::Result { - parse_args_with_env(env::args(), |key| env::var(key).ok()) + let config = config_file::load_from_current_exe()?; + parse_args_with_env(env::args(), |key| env::var(key).ok(), config.as_ref()) } /// 测试入口：不读环境变量，解析结果只由参数决定。 #[cfg(test)] fn parse_args_from(raw_args: impl IntoIterator) -> anyhow::Result { - parse_args_with_env(raw_args, |_| None) + parse_args_with_env(raw_args, |_| None, None) } fn parse_args_with_env( raw_args: impl IntoIterator, env_lookup: impl Fn(&str) -> Option, + config_file: Option<&config_file::BatConfigFile>, ) -> anyhow::Result { let mut args = raw_args.into_iter().peekable(); let binary = args.next().unwrap_or_else(|| "bat".to_string()); let mut options = CliOptions::default(); - // `BAT_*` 环境变量（含 .env 加载的）先作为默认值写入，不标记 explicit； - // 命令行参数随后解析，逐字段覆盖。工具/代理的"非默认"判断以本基线为准， - // 保证 status/stop/logs 在 .env 存在时不误判为显式传了同步参数。 + if let Some(config_file) = config_file { + config_file.apply_to_options(&mut options)?; + } + if env_lookup("BAT_SKIP_ENV_FILE").is_some() { + terminal_output::print_deprecated_env_file_warning(); + } + // `BAT_*` 环境变量先作为默认值写入，不标记 explicit；命令行参数随后解析， + // 逐字段覆盖。工具/代理的"非默认"判断以 config.toml + 环境变量后的基线为准， + // 保证 status/stop/logs 在用户已配置默认值时不误判为显式传了同步参数。 let env_modes = apply_bat_env_overrides(&mut options, &env_lookup)?; options.env_baseline_config = options.config.clone(); let mut mode_flag_from_cli = false; @@ -6442,6 +6696,33 @@ fn parse_args_with_env( Some(PathBuf::from(next_option_value(&mut args, &flag)?)); options.translation_worker_option_explicit = true; } + "--translation-memory" | "--translation-memory-path" => { + options.translation_memory_path = + Some(PathBuf::from(next_option_value(&mut args, &flag)?)); + options.translation_memory_option_explicit = true; + options.translation_worker_option_explicit = true; + } + "--tm-source-text" => { + options.translation_memory_source_text = Some(next_option_value(&mut args, &flag)?); + options.translation_memory_command_option_explicit = true; + } + "--tm-context-json" => { + options.translation_memory_context_json = + Some(next_option_value(&mut args, &flag)?); + options.translation_memory_command_option_explicit = true; + } + "--tm-record-id" => { + options.translation_memory_record_id = Some(next_option_value(&mut args, &flag)?); + options.translation_memory_command_option_explicit = true; + } + "--tm-reviewer" => { + options.translation_memory_reviewer = Some(next_option_value(&mut args, &flag)?); + options.translation_memory_command_option_explicit = true; + } + "--tm-reason" => { + options.translation_memory_reason = Some(next_option_value(&mut args, &flag)?); + options.translation_memory_command_option_explicit = true; + } "--worker-concurrency" | "--translation-concurrency" => { options.worker_concurrency = parse_translation_worker_concurrency( &next_option_value(&mut args, &flag)?, @@ -6925,7 +7206,7 @@ fn parse_args_with_env( } // `BAT_WATCH` / `BAT_DAEMON` 只影响无子命令的 Run（无参启动场景）； - // 命令行显式选择了运行模式或 dry-run 时让位（命令行优先于 .env）， + // 命令行显式选择了运行模式或 dry-run 时让位（命令行优先于配置文件/环境变量）， // status/verify 等子命令不受其影响。daemon 优先于 watch（daemon 自带 watch）。 if matches!(options.command, CliCommand::Run) && !mode_flag_from_cli && !options.config.dry_run { @@ -6943,9 +7224,50 @@ fn parse_args_with_env( } if options.command != CliCommand::TranslationWorker + && !matches!( + options.command, + CliCommand::TranslationMemorySummary + | CliCommand::TranslationMemoryQuery + | CliCommand::TranslationMemoryConfirm + | CliCommand::Restart + | CliCommand::Reload + ) + && !options.daemon + && !options.daemon_child && options.translation_worker_option_explicit { - return Err(anyhow::anyhow!("翻译 worker 参数只适用于 i18n worker run")); + return Err(anyhow::anyhow!( + "翻译 worker 参数只适用于 i18n worker run 或 daemon restart/reload" + )); + } + if options.translation_memory_option_explicit + && !matches!( + options.command, + CliCommand::TranslationWorker + | CliCommand::TranslationMemorySummary + | CliCommand::TranslationMemoryQuery + | CliCommand::TranslationMemoryConfirm + | CliCommand::Restart + | CliCommand::Reload + ) + && !options.daemon + && !options.daemon_child + { + return Err(anyhow::anyhow!( + "Translation Memory 路径参数只适用于 i18n worker/memory 命令或 daemon restart/reload" + )); + } + if options.translation_memory_command_option_explicit + && !matches!( + options.command, + CliCommand::TranslationMemorySummary + | CliCommand::TranslationMemoryQuery + | CliCommand::TranslationMemoryConfirm + ) + { + return Err(anyhow::anyhow!( + "Translation Memory 查询/confirm 参数只适用于 i18n memory 命令" + )); } if options.command != CliCommand::PublishLocalized && options.translation_from_worker { return Err(anyhow::anyhow!("--from-worker 只适用于 i18n publish")); @@ -6976,6 +7298,77 @@ fn parse_args_with_env( options.progress = false; options.banner = false; } + CliCommand::TranslationMemorySummary + | CliCommand::TranslationMemoryQuery + | CliCommand::TranslationMemoryConfirm => { + if options.watch || options.daemon || options.daemon_child { + return Err(anyhow::anyhow!("i18n memory 命令只支持单次执行或 RPC 调用")); + } + if options.config.force + || options.config.dry_run + || options.run_count.is_some() + || options.sync_option_explicit + || options.output_explicit + || options.proxy_option_explicit + || tools_are_non_default(&options.config, &options.env_baseline_config) + || options.translation_worker_option_explicit + && !options.translation_memory_option_explicit + { + return Err(anyhow::anyhow!( + "i18n memory 命令只接受 --state-dir、TM 参数和 --json/--human" + )); + } + if translation_memory_has_unsupported_query_filters(&options) { + return Err(anyhow::anyhow!( + "i18n memory 命令不接受 TextUnit/resource 查询过滤参数" + )); + } + match options.command { + CliCommand::TranslationMemorySummary => { + if options.translation_memory_command_option_explicit + || options.query_limit != 100 + { + return Err(anyhow::anyhow!( + "i18n memory summary 只接受 --translation-memory-path、--resource-root、--state-dir 和 --json/--human" + )); + } + } + CliCommand::TranslationMemoryQuery => { + if options.translation_memory_record_id.is_some() + || options.translation_memory_reviewer.is_some() + || options.translation_memory_reason.is_some() + { + return Err(anyhow::anyhow!("i18n memory query 不接受 confirm 参数")); + } + } + CliCommand::TranslationMemoryConfirm => { + if options.translation_memory_source_text.is_some() + || options.translation_memory_context_json.is_some() + || options.query_limit != 100 + { + return Err(anyhow::anyhow!("i18n memory confirm 不接受 query 参数")); + } + } + _ => unreachable!(), + } + if matches!(options.command, CliCommand::TranslationMemoryQuery) + && options.translation_memory_source_text.is_none() + { + return Err(anyhow::anyhow!( + "i18n memory query 必须指定 --tm-source-text" + )); + } + if matches!(options.command, CliCommand::TranslationMemoryConfirm) + && (options.translation_memory_record_id.is_none() + || options.translation_memory_reviewer.is_none()) + { + return Err(anyhow::anyhow!( + "i18n memory confirm 必须指定 --tm-record-id 和 --tm-reviewer" + )); + } + options.progress = false; + options.banner = false; + } CliCommand::PatchApply | CliCommand::UnityFsPatchTextAsset | CliCommand::UnityFsPatchStringField @@ -7234,29 +7627,7 @@ fn parse_args_with_env( "i18n worker run 只接受 --output、--resource-root、--state-dir、worker 参数和轮询参数" )); } - let provider = options - .translation_provider - .as_deref() - .unwrap_or(TranslationProviderKind::Mock.as_str()); - if TranslationProviderKind::parse(provider).is_none() { - return Err(anyhow::anyhow!( - "i18n worker run 的 provider 无效：{provider}" - )); - } - TranslationWorkerConfig { - provider: TranslationProviderKind::parse(provider).expect("provider 已在上方校验"), - fixture_path: options.translation_fixture.clone(), - concurrency: options.worker_concurrency, - max_attempts: options.worker_max_attempts, - lease_seconds: options.worker_lease_seconds, - retry_backoff: options.worker_retry_backoff, - max_tasks: options.worker_max_tasks, - worker_id: options - .worker_id - .clone() - .unwrap_or_else(|| "bat-worker-validation".to_string()), - } - .validate()?; + translation_worker_config_from_options(&options, "bat-worker-validation")?; } CliCommand::TranslationProofread => { if options.watch || options.daemon || options.daemon_child { @@ -7360,6 +7731,7 @@ fn parse_args_with_env( if (options.sync_option_explicit || options.output_explicit || options.proxy_option_explicit + || options.translation_worker_option_explicit || tools_are_non_default(&options.config, &options.env_baseline_config)) && !options.config.auto_discover && options.config.server_info_source.is_none() @@ -7541,6 +7913,9 @@ fn parse_translation_command( if action == "worker" { return parse_translation_worker_command(args, options); } + if action == "memory" || action == "tm" { + return parse_translation_memory_command(args, options); + } let command = match action.as_str() { "run" => CliCommand::Translate, "export" => CliCommand::Translate, @@ -7565,6 +7940,22 @@ fn parse_translation_command( Ok(()) } +fn parse_translation_memory_command( + args: &mut impl Iterator, + options: &mut CliOptions, +) -> anyhow::Result<()> { + let action = next_option_value(args, "translation memory")?; + let command = match action.as_str() { + "summary" | "status" => CliCommand::TranslationMemorySummary, + "query" | "find" => CliCommand::TranslationMemoryQuery, + "confirm" | "trust" => CliCommand::TranslationMemoryConfirm, + other => return Err(anyhow::anyhow!("未知 translation memory 二级命令：{other}")), + }; + ensure_command_not_set(options.command, &format!("translation memory {action}"))?; + options.command = command; + Ok(()) +} + fn parse_translation_worker_command( args: &mut impl Iterator, options: &mut CliOptions, @@ -7642,7 +8033,7 @@ fn ensure_command_not_set(command: CliCommand, next: &str) -> anyhow::Result<()> } } -/// 判断 curl/代理/unzip 是否偏离基线。基线是环境变量（含 .env）应用后的 +/// 判断 curl/代理/unzip 是否偏离基线。基线是 `config.toml` 和环境变量应用后的 /// 配置快照，因此只有命令行显式传入才算"非默认"。 fn tools_are_non_default(config: &OfficialUpdateConfig, baseline: &OfficialUpdateConfig) -> bool { config.curl_command != baseline.curl_command @@ -7650,6 +8041,56 @@ fn tools_are_non_default(config: &OfficialUpdateConfig, baseline: &OfficialUpdat || config.unzip_command != baseline.unzip_command } +fn translation_memory_has_unsupported_query_filters(options: &CliOptions) -> bool { + options.query_offset != 0 + || options.query_task_id.is_some() + || options.query_resource_type.is_some() + || options.query_hash.is_some() + || options.query_path_pattern.is_some() + || options.query_official_release_id.is_some() + || options.query_platform.is_some() + || options.query_destination.is_some() + || options.query_bundle_path.is_some() + || options.query_archive_entry.is_some() + || options.query_task_status.is_some() + || options.query_worker_status.is_some() + || options.query_parse_status.is_some() + || options.query_path_id.is_some() + || options.query_class_id.is_some() + || options.query_field_path.is_some() + || options.query_format.is_some() + || options.query_has_reason.is_some() + || options.query_has_failure_reason.is_some() +} + +fn translation_worker_config_from_options( + options: &CliOptions, + default_worker_id: &str, +) -> anyhow::Result { + let provider = options + .translation_provider + .as_deref() + .unwrap_or(TranslationProviderKind::Mock.as_str()); + let provider = TranslationProviderKind::parse(provider) + .ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效：{provider}"))?; + let config = TranslationWorkerConfig { + provider, + fixture_path: options.translation_fixture.clone(), + concurrency: options.worker_concurrency, + max_attempts: options.worker_max_attempts, + lease_seconds: options.worker_lease_seconds, + retry_backoff: options.worker_retry_backoff, + max_tasks: options.worker_max_tasks, + worker_id: options + .worker_id + .clone() + .unwrap_or_else(|| default_worker_id.to_string()), + translation_memory_path: options.translation_memory_path.clone(), + }; + config.validate()?; + Ok(config) +} + fn parse_download_concurrency(value: &str, source: &str) -> anyhow::Result { let parsed = value .parse::() @@ -7729,8 +8170,12 @@ fn validate_proxy_url(url: &str) -> anyhow::Result<()> { SUPPORTED_PROXY_SCHEMES.join("/") )); } - if rest.is_empty() { - return Err(anyhow::anyhow!("--proxy 缺少代理主机：{url}")); + let host = rest.rsplit_once('@').map_or(rest, |(_, host)| host); + if host.is_empty() { + return Err(anyhow::anyhow!( + "--proxy 缺少代理主机：{}", + redact_proxy_url(url) + )); } } Ok(()) diff --git a/infrastructure/src/bin/bat/app_tests.rs b/infrastructure/src/bin/bat/app_tests.rs index 3079f20..8af6f9c 100644 --- a/infrastructure/src/bin/bat/app_tests.rs +++ b/infrastructure/src/bin/bat/app_tests.rs @@ -17,9 +17,11 @@ fn parse_with_env(values: &[&str], env: &[(&str, &str)]) -> anyhow::Result Some("/env/output".to_string()), + "BAT_TRANSLATION_MEMORY_PATH" => Some("/env/tm.sqlite".to_string()), + _ => None, + }, + Some(&config), + ) + .unwrap(); + assert_eq!(options.state_dir, PathBuf::from("/srv/state")); + assert_eq!(options.output_format, OutputFormat::Json); + assert_eq!(options.config.output_root, PathBuf::from("/cli/output")); + assert_eq!( + options.config.localized_output_root, + PathBuf::from("/srv/from-config-localized") + ); + assert_eq!(options.config.download_concurrency, 12); + assert_eq!(options.config.curl_proxy.mode(), &CurlProxyMode::Disabled); + assert_eq!( + options.translation_memory_path, + Some(PathBuf::from("/cli/tm.sqlite")) + ); } #[test] @@ -1326,6 +1503,14 @@ fn rejects_proxy_with_unsupported_scheme() { assert!(parse(&["bat", "--proxy", "127.0.0.1:7890"]).is_ok()); } +#[test] +fn proxy_parse_errors_redact_credentials() { + let error = parse(&["bat", "--proxy", "http://user:secret@"]).unwrap_err(); + let message = error.to_string(); + assert!(!message.contains("secret")); + assert!(message.contains("")); +} + #[test] fn parses_watch_defaults_to_one_hour_and_quiet_up_to_date() { let options = parse(&["bat", "--auto-discover", "--watch", "--interval", "30m"]).unwrap(); @@ -1997,6 +2182,54 @@ fn daemon_child_args_preserve_sync_options() { assert!(args.contains(&"--no-banner".to_string())); } +#[test] +fn daemon_child_args_preserve_translation_worker_configuration() { + let options = parse(&[ + "bat", + "--daemon", + "--output", + "/tmp/daemon-output", + "--translation-provider", + "mock", + "--translation-fixture", + "/tmp/provider.json", + "--translation-memory-path", + "/tmp/tm.sqlite", + "--worker-concurrency", + "4", + "--worker-max-attempts", + "5", + "--worker-lease-seconds", + "60", + "--worker-retry-backoff-seconds", + "2", + "--worker-max-tasks", + "3", + "--worker-id", + "daemon-worker", + ]) + .unwrap(); + let args = daemon_child_args(&options); + let mut child_args = vec!["bat".to_string()]; + child_args.extend(args); + let child = parse_args_from(child_args).unwrap(); + assert_eq!(child.translation_provider.as_deref(), Some("mock")); + assert_eq!( + child.translation_fixture, + Some(PathBuf::from("/tmp/provider.json")) + ); + assert_eq!( + child.translation_memory_path, + Some(PathBuf::from("/tmp/tm.sqlite")) + ); + assert_eq!(child.worker_concurrency, 4); + assert_eq!(child.worker_max_attempts, 5); + assert_eq!(child.worker_lease_seconds, 60); + assert_eq!(child.worker_retry_backoff, Duration::from_secs(2)); + assert_eq!(child.worker_max_tasks, Some(3)); + assert_eq!(child.worker_id.as_deref(), Some("daemon-worker")); +} + #[test] fn curl_proxy_url_extracts_only_url_mode() { assert_eq!( @@ -2184,6 +2417,7 @@ fn test_task_context_with_config(base_config: OfficialUpdateConfig) -> DaemonTas registry: TaskRegistry::new(), queue, base_config, + translation_worker_config: TranslationWorkerConfig::default(), sync_lock: Arc::new(Mutex::new(())), restart_controller: test_restart_controller, } @@ -2354,6 +2588,7 @@ fn dispatch_daemon_doctor_returns_report() { registry: TaskRegistry::new(), queue, base_config, + translation_worker_config: TranslationWorkerConfig::default(), sync_lock: Arc::new(Mutex::new(())), restart_controller: test_restart_controller, }; @@ -2374,6 +2609,112 @@ fn dispatch_daemon_doctor_returns_report() { assert!(checks.iter().any(|check| check["name"] == "daemon_rpc")); } +#[test] +fn dispatch_translation_memory_summary_reports_missing_database_without_creating_it() { + let temp = tempfile::TempDir::new().unwrap(); + let output_root = temp.path().join("output"); + let state_dir = temp.path().join("state"); + let (queue, _rx) = mpsc::channel::(); + let context = DaemonTaskContext { + registry: TaskRegistry::new(), + queue, + base_config: OfficialUpdateConfig { + output_root: output_root.clone(), + ..Default::default() + }, + translation_worker_config: TranslationWorkerConfig::default(), + sync_lock: Arc::new(Mutex::new(())), + restart_controller: test_restart_controller, + }; + + let envelope = dispatch_rpc_method( + &rpc_request("translation.memory.summary", None), + &state_dir, + &new_daemon_control(), + &context, + "req-tm-summary-1".to_string(), + ); + let value = serde_json::to_value(envelope).unwrap(); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["available"], false); + assert_eq!(value["data"]["reason"], "database_missing"); + assert!(!output_root.join("translation-memory.sqlite").exists()); +} + +#[test] +fn dispatch_translation_memory_summary_defaults_to_worker_config_path() { + let temp = tempfile::TempDir::new().unwrap(); + let output_root = temp.path().join("output"); + let state_dir = temp.path().join("state"); + let configured_tm_path = temp.path().join("configured-tm.sqlite"); + let (queue, _rx) = mpsc::channel::(); + let context = DaemonTaskContext { + registry: TaskRegistry::new(), + queue, + base_config: OfficialUpdateConfig { + output_root, + ..Default::default() + }, + translation_worker_config: TranslationWorkerConfig { + translation_memory_path: Some(configured_tm_path.clone()), + ..TranslationWorkerConfig::default() + }, + sync_lock: Arc::new(Mutex::new(())), + restart_controller: test_restart_controller, + }; + + let envelope = dispatch_rpc_method( + &rpc_request("translation.memory.summary", None), + &state_dir, + &new_daemon_control(), + &context, + "req-tm-summary-configured-1".to_string(), + ); + let value = serde_json::to_value(envelope).unwrap(); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["available"], false); + assert_eq!( + value["data"]["path"].as_str(), + Some(configured_tm_path.to_str().unwrap()) + ); +} + +#[test] +fn dispatch_translation_memory_rejects_invalid_params_with_stable_error_code() { + let temp = tempfile::TempDir::new().unwrap(); + let context = test_task_context_with_config(OfficialUpdateConfig { + output_root: temp.path().join("output"), + ..Default::default() + }); + + for (method, params) in [ + ("translation.memory.query", None), + ( + "translation.memory.query", + Some(serde_json::json!({ "source_text": "Hello", "limit": "1" })), + ), + ( + "translation.memory.summary", + Some(serde_json::json!({ "translation_memory_path": 42 })), + ), + ( + "translation.memory.confirm", + Some(serde_json::json!({ "record_id": "tm-record", "reviewer": 42 })), + ), + ] { + let envelope = dispatch_rpc_method( + &rpc_request(method, params), + temp.path(), + &new_daemon_control(), + &context, + format!("req-invalid-{method}"), + ); + let value = serde_json::to_value(envelope).unwrap(); + assert_eq!(value["ok"], false, "method={method}"); + assert_eq!(value["error"]["code"], "BAT-ERR-700002", "method={method}"); + } +} + #[test] fn dispatch_resource_sync_enqueues_task() { let temp = tempfile::TempDir::new().unwrap(); @@ -2384,6 +2725,7 @@ fn dispatch_resource_sync_enqueues_task() { registry: TaskRegistry::new(), queue, base_config: OfficialUpdateConfig::default(), + translation_worker_config: TranslationWorkerConfig::default(), sync_lock: Arc::new(Mutex::new(())), restart_controller: test_restart_controller, }; @@ -2445,6 +2787,7 @@ fn dispatch_resource_repair_enqueues_repair_task() { registry: TaskRegistry::new(), queue, base_config, + translation_worker_config: TranslationWorkerConfig::default(), sync_lock: Arc::new(Mutex::new(())), restart_controller: test_restart_controller, }; @@ -3737,6 +4080,7 @@ fn dispatch_catalog_refresh_enqueues_task() { registry: TaskRegistry::new(), queue, base_config: OfficialUpdateConfig::default(), + translation_worker_config: TranslationWorkerConfig::default(), sync_lock: Arc::new(Mutex::new(())), restart_controller: test_restart_controller, }; diff --git a/infrastructure/src/bin/bat/config_file.rs b/infrastructure/src/bin/bat/config_file.rs new file mode 100644 index 0000000..8d0f57c --- /dev/null +++ b/infrastructure/src/bin/bat/config_file.rs @@ -0,0 +1,1182 @@ +use super::terminal_output; +use super::{ + parse_download_concurrency, parse_platforms, parse_positive_u32, parse_positive_u64, + parse_proxy_config, parse_translation_worker_concurrency, CliOptions, CurlProxyConfig, + OfficialServerInfoSource, OutputFormat, TranslationProviderKind, PRIVATE_FILE_MODE, +}; +use bat_adapters::official::yostar_jp::PatchPlatform; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub(crate) const CONFIG_FILE_NAME: &str = "config.toml"; +pub(crate) const CONFIG_EXAMPLE_FILE_NAME: &str = "config.toml.example"; + +const CONFIG_TEMPLATE: &str = r#"# BlueArchive Toolkit 配置文件（bat 首次启动自动生成） +# +# `config.toml` 位于 bat 二进制所在目录。存在时会被读取并生效。 +# 优先级：CLI > 进程环境变量 > config.toml > 内置默认值。 +# `config.toml.example` 只是模板，程序不会自动读取它作为实际配置。 +# +# 字符串建议使用单引号，路径/URL 更容易直接复制。 + +[runtime] +state_dir = '/tmp/bat-pid' +interval_seconds = 3600 +error_retry_seconds = 60 +quiet_up_to_date = false +output_format = 'human' +banner = true +progress = true +tail_lines = 200 + +[resource] +output_root = './bat-resources' +auto_discover = true +app_version = '' +connection_group = '' +launcher_version = '1.7.2' +platforms = ['windows', 'android'] +snapshot_path = '' +dry_run = false +plan = false +force = false +audit_local = true +repair = true + +[resource.server_info] +kind = 'none' +value = '' + +[localized] +output_root = './bat-localized' + +[repository] +import_repository = false +import_cas_root = '' +import_resource_repository_path = '' + +[network] +curl_command = 'curl' +proxy = 'auto' +unzip_command = 'unzip' +download_concurrency = 8 + +[translation.worker] +provider = 'mock' +fixture = '' +translation_memory_path = '' +concurrency = 8 +max_attempts = 3 +lease_seconds = 300 +retry_backoff_seconds = 5 +max_tasks = '' +worker_id = '' +"#; + +#[derive(Debug, Clone, Default)] +pub(crate) struct BatConfigFile { + runtime: RuntimeSection, + resource: ResourceSection, + localized: LocalizedSection, + repository: RepositorySection, + network: NetworkSection, + translation: TranslationSection, +} + +#[derive(Debug, Clone, Default)] +struct RuntimeSection { + state_dir: Option, + interval_seconds: Option, + error_retry_seconds: Option, + quiet_up_to_date: Option, + output_format: Option, + banner: Option, + progress: Option, + tail_lines: Option, +} + +#[derive(Debug, Clone, Default)] +struct ResourceSection { + output_root: Option, + auto_discover: Option, + app_version: Option, + connection_group: Option, + launcher_version: Option, + platforms: Option>, + snapshot_path: Option, + dry_run: Option, + plan: Option, + force: Option, + audit_local: Option, + repair: Option, + server_info: Option, +} + +#[derive(Debug, Clone, Default)] +struct ServerInfoSection { + kind: Option, + value: Option, +} + +#[derive(Debug, Clone, Default)] +struct LocalizedSection { + output_root: Option, +} + +#[derive(Debug, Clone, Default)] +struct RepositorySection { + import_repository: Option, + import_cas_root: Option, + import_resource_repository_path: Option, +} + +#[derive(Debug, Clone, Default)] +struct NetworkSection { + curl_command: Option, + proxy: Option, + unzip_command: Option, + download_concurrency: Option, +} + +#[derive(Debug, Clone, Default)] +struct TranslationSection { + worker: TranslationWorkerSection, +} + +#[derive(Debug, Clone, Default)] +struct TranslationWorkerSection { + provider: Option, + fixture: Option, + translation_memory_path: Option, + concurrency: Option, + max_attempts: Option, + lease_seconds: Option, + retry_backoff_seconds: Option, + max_tasks: Option, + worker_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SectionPath { + Runtime, + Resource, + ResourceServerInfo, + Localized, + Repository, + Network, + TranslationWorker, +} + +pub(crate) fn load_from_current_exe() -> anyhow::Result> { + let exe_path = + env::current_exe().map_err(|error| anyhow::anyhow!("读取当前可执行文件失败：{error}"))?; + let Some(exe_dir) = exe_path.parent() else { + return Ok(None); + }; + load_from_binary_dir(exe_dir) +} + +pub(crate) fn load_from_binary_dir(exe_dir: &Path) -> anyhow::Result> { + let config_path = exe_dir.join(CONFIG_FILE_NAME); + let example_path = exe_dir.join(CONFIG_EXAMPLE_FILE_NAME); + ensure_private_config_file(&config_path)?; + let Some(bytes) = + super::read_file_no_symlink(&config_path, "bat config.toml").map_err(anyhow::Error::msg)? + else { + maybe_create_example(&example_path)?; + return Ok(None); + }; + let content = std::str::from_utf8(&bytes) + .map_err(|error| anyhow::anyhow!("config.toml 不是有效 UTF-8：{error}"))?; + let config = parse_config_document(content).map_err(|error| { + anyhow::anyhow!("解析 config.toml 失败 {}：{error}", config_path.display()) + })?; + Ok(Some(config)) +} + +/// `config.toml` may contain proxy credentials, so do not read a +/// group/world-readable file on Unix. +fn ensure_private_config_file(path: &Path) -> anyhow::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let Ok(metadata) = fs::symlink_metadata(path) else { + return Ok(()); + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Ok(()); + } + let mode = metadata.permissions().mode() & 0o777; + if mode & 0o077 != 0 { + return Err(anyhow::anyhow!( + "config.toml 可能包含代理凭据，权限必须为 0600 或更严格：{}（当前 {:03o}）", + path.display(), + mode + )); + } + } + #[cfg(not(unix))] + { + let _ = path; + } + Ok(()) +} + +fn maybe_create_example(path: &Path) -> anyhow::Result<()> { + if fs::symlink_metadata(path).is_ok() { + return Ok(()); + } + match super::write_file_atomic( + path, + CONFIG_TEMPLATE.as_bytes(), + PRIVATE_FILE_MODE, + "config.toml.example", + ) { + Ok(()) => { + terminal_output::print_config_template_created(path); + Ok(()) + } + Err(error) => { + terminal_output::print_config_template_warning(path, error); + Ok(()) + } + } +} + +fn parse_config_document(content: &str) -> anyhow::Result { + let mut config = BatConfigFile::default(); + let mut section = SectionPath::Runtime; + for (line_number, raw_line) in content.lines().enumerate() { + let line = strip_comment(raw_line).trim().to_string(); + if line.is_empty() { + continue; + } + if line.starts_with('[') { + section = parse_section_header(&line, line_number + 1)?; + continue; + } + let (key, value) = split_key_value(&line, line_number + 1)?; + config.set_value(section, key, value, line_number + 1)?; + } + Ok(config) +} + +impl BatConfigFile { + pub(crate) fn apply_to_options(&self, options: &mut CliOptions) -> anyhow::Result<()> { + if let Some(value) = self.runtime.state_dir.as_ref() { + options.state_dir = value.clone(); + } + if let Some(value) = self.runtime.interval_seconds { + options.interval = Duration::from_secs(value); + } + if let Some(value) = self.runtime.error_retry_seconds { + options.error_retry_interval = Duration::from_secs(value); + } + if let Some(value) = self.runtime.quiet_up_to_date { + options.quiet_up_to_date = value; + } + if let Some(value) = self.runtime.output_format { + options.output_format = value; + } + if let Some(value) = self.runtime.banner { + options.banner = value; + } + if let Some(value) = self.runtime.progress { + options.progress = value; + } + if let Some(value) = self.runtime.tail_lines { + options.tail_lines = value; + } + + if let Some(value) = self.resource.output_root.as_ref() { + options.config.output_root = value.clone(); + } + if let Some(value) = self.resource.auto_discover { + options.config.auto_discover = value; + } + if let Some(value) = self.resource.app_version.as_ref() { + options.config.app_version = Some(value.clone()); + } + if let Some(value) = self.resource.connection_group.as_ref() { + options.config.connection_group = Some(value.clone()); + } + if let Some(value) = self.resource.launcher_version.as_ref() { + options.config.launcher_version = value.clone(); + } + if let Some(value) = self.resource.platforms.as_ref() { + options.config.platforms = Some(value.clone()); + } + if let Some(value) = self.resource.snapshot_path.as_ref() { + options.config.snapshot_path = Some(value.clone()); + } + if let Some(value) = self.resource.dry_run { + options.config.dry_run = value; + } + if let Some(value) = self.resource.plan { + options.config.plan = value; + } + if let Some(value) = self.resource.force { + options.config.force = value; + } + if let Some(value) = self.resource.audit_local { + options.config.audit_local = value; + } + if let Some(value) = self.resource.repair { + options.config.repair = value; + } + if let Some(value) = self.resource.server_info.as_ref() { + options.config.server_info_source = value.to_source()?; + } + + if let Some(value) = self.localized.output_root.as_ref() { + options.config.localized_output_root = value.clone(); + } + + if let Some(value) = self.repository.import_repository { + options.config.import_repository = value; + } + if let Some(value) = self.repository.import_cas_root.as_ref() { + options.config.import_cas_root = Some(value.clone()); + } + if let Some(value) = self.repository.import_resource_repository_path.as_ref() { + options.config.import_resource_repository_path = Some(value.clone()); + } + + if let Some(value) = self.network.curl_command.as_ref() { + options.config.curl_command = value.clone(); + } + if let Some(value) = self.network.proxy.as_ref() { + options.config.curl_proxy = value.clone(); + } + if let Some(value) = self.network.unzip_command.as_ref() { + options.config.unzip_command = value.clone(); + } + if let Some(value) = self.network.download_concurrency { + options.config.download_concurrency = value; + } + + if let Some(value) = self.translation.worker.provider.as_ref() { + options.translation_provider = Some(value.clone()); + } + if let Some(value) = self.translation.worker.fixture.as_ref() { + options.translation_fixture = Some(value.clone()); + } + if let Some(value) = self.translation.worker.translation_memory_path.as_ref() { + options.translation_memory_path = Some(value.clone()); + } + if let Some(value) = self.translation.worker.concurrency { + options.worker_concurrency = value; + } + if let Some(value) = self.translation.worker.max_attempts { + options.worker_max_attempts = value; + } + if let Some(value) = self.translation.worker.lease_seconds { + options.worker_lease_seconds = value; + } + if let Some(value) = self.translation.worker.retry_backoff_seconds { + options.worker_retry_backoff = Duration::from_secs(value); + } + if let Some(value) = self.translation.worker.max_tasks { + options.worker_max_tasks = Some(value); + } + if let Some(value) = self.translation.worker.worker_id.as_ref() { + options.worker_id = Some(value.clone()); + } + Ok(()) + } + + fn set_value( + &mut self, + section: SectionPath, + key: &str, + value: &str, + line_number: usize, + ) -> anyhow::Result<()> { + match (section, key) { + (SectionPath::Runtime, "state_dir") => { + self.runtime.state_dir = Some(parse_required_path( + value, + "runtime.state_dir", + line_number, + )?); + } + (SectionPath::Runtime, "interval_seconds") => { + self.runtime.interval_seconds = Some(parse_u64_value( + value, + "runtime.interval_seconds", + line_number, + )?); + } + (SectionPath::Runtime, "error_retry_seconds") => { + self.runtime.error_retry_seconds = Some(parse_u64_value( + value, + "runtime.error_retry_seconds", + line_number, + )?); + } + (SectionPath::Runtime, "quiet_up_to_date") => { + self.runtime.quiet_up_to_date = Some(parse_bool_value( + value, + "runtime.quiet_up_to_date", + line_number, + )?); + } + (SectionPath::Runtime, "output_format") => { + self.runtime.output_format = Some(parse_output_format( + value, + "runtime.output_format", + line_number, + )?); + } + (SectionPath::Runtime, "banner") => { + self.runtime.banner = Some(parse_bool_value(value, "runtime.banner", line_number)?); + } + (SectionPath::Runtime, "progress") => { + self.runtime.progress = + Some(parse_bool_value(value, "runtime.progress", line_number)?); + } + (SectionPath::Runtime, "tail_lines") => { + self.runtime.tail_lines = + Some(parse_tail_lines(value, "runtime.tail_lines", line_number)?); + } + + (SectionPath::Resource, "output_root") => { + self.resource.output_root = Some(parse_required_path( + value, + "resource.output_root", + line_number, + )?); + } + (SectionPath::Resource, "auto_discover") => { + self.resource.auto_discover = Some(parse_bool_value( + value, + "resource.auto_discover", + line_number, + )?); + } + (SectionPath::Resource, "app_version") => { + self.resource.app_version = + parse_optional_string(value, "resource.app_version", line_number)?; + } + (SectionPath::Resource, "connection_group") => { + self.resource.connection_group = + parse_optional_string(value, "resource.connection_group", line_number)?; + } + (SectionPath::Resource, "launcher_version") => { + self.resource.launcher_version = Some(parse_required_string( + value, + "resource.launcher_version", + line_number, + )?); + } + (SectionPath::Resource, "platforms") => { + self.resource.platforms = Some(parse_platform_list( + value, + "resource.platforms", + line_number, + )?); + } + (SectionPath::Resource, "snapshot_path") => { + self.resource.snapshot_path = + parse_optional_path(value, "resource.snapshot_path", line_number)?; + } + (SectionPath::Resource, "dry_run") => { + self.resource.dry_run = + Some(parse_bool_value(value, "resource.dry_run", line_number)?); + } + (SectionPath::Resource, "plan") => { + self.resource.plan = Some(parse_bool_value(value, "resource.plan", line_number)?); + } + (SectionPath::Resource, "force") => { + self.resource.force = Some(parse_bool_value(value, "resource.force", line_number)?); + } + (SectionPath::Resource, "audit_local") => { + self.resource.audit_local = Some(parse_bool_value( + value, + "resource.audit_local", + line_number, + )?); + } + (SectionPath::Resource, "repair") => { + self.resource.repair = + Some(parse_bool_value(value, "resource.repair", line_number)?); + } + + (SectionPath::ResourceServerInfo, "kind") => { + self.resource + .server_info + .get_or_insert_with(ServerInfoSection::default) + .kind = Some(parse_required_string( + value, + "resource.server_info.kind", + line_number, + )?); + } + (SectionPath::ResourceServerInfo, "value") => { + self.resource + .server_info + .get_or_insert_with(ServerInfoSection::default) + .value = + parse_optional_string(value, "resource.server_info.value", line_number)?; + } + + (SectionPath::Localized, "output_root") => { + self.localized.output_root = Some(parse_required_path( + value, + "localized.output_root", + line_number, + )?); + } + + (SectionPath::Repository, "import_repository") => { + self.repository.import_repository = Some(parse_bool_value( + value, + "repository.import_repository", + line_number, + )?); + } + (SectionPath::Repository, "import_cas_root") => { + self.repository.import_cas_root = + parse_optional_path(value, "repository.import_cas_root", line_number)?; + } + (SectionPath::Repository, "import_resource_repository_path") => { + self.repository.import_resource_repository_path = parse_optional_path( + value, + "repository.import_resource_repository_path", + line_number, + )?; + } + + (SectionPath::Network, "curl_command") => { + self.network.curl_command = Some(parse_required_path( + value, + "network.curl_command", + line_number, + )?); + } + (SectionPath::Network, "proxy") => { + let proxy = parse_required_string(value, "network.proxy", line_number)?; + self.network.proxy = Some(parse_proxy_config(&proxy).map_err(|error| { + anyhow::anyhow!("network.proxy 无效 第 {} 行：{error}", line_number) + })?); + } + (SectionPath::Network, "unzip_command") => { + self.network.unzip_command = Some(parse_required_path( + value, + "network.unzip_command", + line_number, + )?); + } + (SectionPath::Network, "download_concurrency") => { + self.network.download_concurrency = Some(parse_download_concurrency( + &parse_scalar_text(value, "network.download_concurrency", line_number)?, + "config.toml 中的 network.download_concurrency", + )?); + } + + (SectionPath::TranslationWorker, "provider") => { + self.translation.worker.provider = parse_translation_provider(value, line_number)?; + } + (SectionPath::TranslationWorker, "fixture") => { + self.translation.worker.fixture = + parse_optional_path(value, "translation.worker.fixture", line_number)?; + } + (SectionPath::TranslationWorker, "translation_memory_path") => { + self.translation.worker.translation_memory_path = parse_optional_path( + value, + "translation.worker.translation_memory_path", + line_number, + )?; + } + (SectionPath::TranslationWorker, "concurrency") => { + self.translation.worker.concurrency = Some(parse_translation_worker_concurrency( + &parse_scalar_text(value, "translation.worker.concurrency", line_number)?, + "config.toml 中的 translation.worker.concurrency", + )?); + } + (SectionPath::TranslationWorker, "max_attempts") => { + self.translation.worker.max_attempts = Some(parse_positive_u32( + &parse_scalar_text(value, "translation.worker.max_attempts", line_number)?, + "config.toml 中的 translation.worker.max_attempts", + )?); + } + (SectionPath::TranslationWorker, "lease_seconds") => { + self.translation.worker.lease_seconds = Some(parse_positive_u64( + &parse_scalar_text(value, "translation.worker.lease_seconds", line_number)?, + "config.toml 中的 translation.worker.lease_seconds", + )?); + } + (SectionPath::TranslationWorker, "retry_backoff_seconds") => { + self.translation.worker.retry_backoff_seconds = Some(parse_u64_value( + value, + "translation.worker.retry_backoff_seconds", + line_number, + )?); + } + (SectionPath::TranslationWorker, "max_tasks") => { + self.translation.worker.max_tasks = + parse_optional_usize(value, "translation.worker.max_tasks", line_number)?; + } + (SectionPath::TranslationWorker, "worker_id") => { + self.translation.worker.worker_id = + parse_optional_string(value, "translation.worker.worker_id", line_number)?; + } + + _ => { + return Err(anyhow::anyhow!( + "不支持的 config.toml 项：[{section:?}] {key}（第 {line_number} 行）" + )); + } + } + Ok(()) + } +} + +impl ServerInfoSection { + fn to_source(&self) -> anyhow::Result> { + let Some(kind) = self.kind.as_ref() else { + return Ok(None); + }; + match kind.to_ascii_lowercase().as_str() { + "none" => Ok(None), + "local_path" => { + let value = self + .value + .as_ref() + .ok_or_else(|| anyhow::anyhow!("resource.server_info.value 不能为空"))?; + Ok(Some(OfficialServerInfoSource::LocalPath(PathBuf::from( + value, + )))) + } + "official_file" => { + let value = self + .value + .as_ref() + .ok_or_else(|| anyhow::anyhow!("resource.server_info.value 不能为空"))?; + Ok(Some(OfficialServerInfoSource::OfficialFile(value.clone()))) + } + "official_url" => { + let value = self + .value + .as_ref() + .ok_or_else(|| anyhow::anyhow!("resource.server_info.value 不能为空"))?; + Ok(Some(OfficialServerInfoSource::OfficialUrl(value.clone()))) + } + other => Err(anyhow::anyhow!( + "不支持的 resource.server_info.kind：{other}" + )), + } + } +} + +fn parse_section_header(line: &str, line_number: usize) -> anyhow::Result { + if !line.starts_with('[') || !line.ends_with(']') { + return Err(anyhow::anyhow!("第 {line_number} 行不是有效的表头：{line}")); + } + let inner = line[1..line.len() - 1].trim(); + if inner.is_empty() { + return Err(anyhow::anyhow!("第 {line_number} 行的表头不能为空")); + } + let parts: Vec<&str> = inner.split('.').map(str::trim).collect(); + for part in &parts { + if !is_valid_ident(part) { + return Err(anyhow::anyhow!("第 {line_number} 行的表头无效：{line}")); + } + } + match parts.as_slice() { + ["runtime"] => Ok(SectionPath::Runtime), + ["resource"] => Ok(SectionPath::Resource), + ["resource", "server_info"] => Ok(SectionPath::ResourceServerInfo), + ["localized"] => Ok(SectionPath::Localized), + ["repository"] => Ok(SectionPath::Repository), + ["network"] => Ok(SectionPath::Network), + ["translation", "worker"] => Ok(SectionPath::TranslationWorker), + _ => Err(anyhow::anyhow!("第 {line_number} 行不支持的表头：{line}")), + } +} + +fn split_key_value(line: &str, line_number: usize) -> anyhow::Result<(&str, &str)> { + let Some((key, value)) = line.split_once('=') else { + return Err(anyhow::anyhow!("第 {line_number} 行缺少 =：{line}")); + }; + let key = key.trim(); + let value = value.trim(); + if !is_valid_ident(key) { + return Err(anyhow::anyhow!("第 {line_number} 行的键无效：{key}")); + } + if value.is_empty() { + return Err(anyhow::anyhow!("第 {line_number} 行的值不能为空：{line}")); + } + Ok((key, value)) +} + +fn strip_comment(line: &str) -> String { + let mut output = String::with_capacity(line.len()); + let mut in_single = false; + let mut in_double = false; + let mut escaped = false; + for ch in line.chars() { + if escaped { + output.push(ch); + escaped = false; + continue; + } + match ch { + '\\' if in_double => { + output.push(ch); + escaped = true; + } + '\'' if !in_double => { + in_single = !in_single; + output.push(ch); + } + '"' if !in_single => { + in_double = !in_double; + output.push(ch); + } + '#' if !in_single && !in_double => break, + _ => output.push(ch), + } + } + output +} + +fn parse_required_string(value: &str, field: &str, line_number: usize) -> anyhow::Result { + let parsed = parse_string_value(value, field, line_number)?; + if parsed.trim().is_empty() { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 不能为空")); + } + Ok(parsed) +} + +fn parse_optional_string( + value: &str, + field: &str, + line_number: usize, +) -> anyhow::Result> { + let parsed = parse_string_value(value, field, line_number)?; + if parsed.trim().is_empty() { + Ok(None) + } else { + Ok(Some(parsed)) + } +} + +fn parse_required_path(value: &str, field: &str, line_number: usize) -> anyhow::Result { + Ok(PathBuf::from(parse_required_string( + value, + field, + line_number, + )?)) +} + +fn parse_optional_path( + value: &str, + field: &str, + line_number: usize, +) -> anyhow::Result> { + Ok(parse_optional_string(value, field, line_number)?.map(PathBuf::from)) +} + +fn parse_optional_usize( + value: &str, + field: &str, + line_number: usize, +) -> anyhow::Result> { + let parsed = parse_scalar_text(value, field, line_number)?; + if parsed.trim().is_empty() { + return Ok(None); + } + Ok(Some(parsed.parse::().map_err(|error| { + anyhow::anyhow!("第 {line_number} 行 {field} 无效：{error}") + })?)) +} + +fn parse_u64_value(value: &str, field: &str, line_number: usize) -> anyhow::Result { + parse_scalar_text(value, field, line_number)? + .parse::() + .map_err(|error| anyhow::anyhow!("第 {line_number} 行 {field} 无效：{error}")) +} + +fn parse_bool_value(value: &str, field: &str, line_number: usize) -> anyhow::Result { + match parse_scalar_text(value, field, line_number)? + .to_ascii_lowercase() + .as_str() + { + "true" => Ok(true), + "false" => Ok(false), + other => Err(anyhow::anyhow!( + "第 {line_number} 行 {field} 的布尔值无效：{other}" + )), + } +} + +fn parse_output_format( + value: &str, + field: &str, + line_number: usize, +) -> anyhow::Result { + match parse_string_value(value, field, line_number)? + .to_ascii_lowercase() + .as_str() + { + "human" => Ok(OutputFormat::Human), + "json" => Ok(OutputFormat::Json), + other => Err(anyhow::anyhow!( + "第 {line_number} 行 {field} 的输出格式无效：{other}" + )), + } +} + +fn parse_platform_list( + value: &str, + field: &str, + line_number: usize, +) -> anyhow::Result> { + let platforms = parse_string_list(value, field, line_number)?; + let joined = platforms.join(","); + parse_platforms(&joined) + .map_err(|error| anyhow::anyhow!("第 {line_number} 行 {field} 无效：{error}")) +} + +fn parse_translation_provider(value: &str, line_number: usize) -> anyhow::Result> { + let provider = parse_string_value(value, "translation.worker.provider", line_number)?; + if provider.trim().is_empty() { + return Ok(None); + } + if TranslationProviderKind::parse(provider.trim()).is_none() { + return Err(anyhow::anyhow!( + "第 {line_number} 行 translation.worker.provider 无效：{provider}" + )); + } + Ok(Some(provider.trim().to_string())) +} + +fn parse_string_list(value: &str, field: &str, line_number: usize) -> anyhow::Result> { + let trimmed = value.trim(); + if !trimmed.starts_with('[') || !trimmed.ends_with(']') { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 必须是数组")); + } + let inner = trimmed[1..trimmed.len() - 1].trim(); + if inner.is_empty() { + return Ok(Vec::new()); + } + let mut items = Vec::new(); + let mut current = String::new(); + let mut in_single = false; + let mut in_double = false; + let mut escaped = false; + for ch in inner.chars() { + if escaped { + current.push(ch); + escaped = false; + continue; + } + match ch { + '\\' if in_double => { + current.push(ch); + escaped = true; + } + '\'' if !in_double => { + in_single = !in_single; + current.push(ch); + } + '"' if !in_single => { + in_double = !in_double; + current.push(ch); + } + ',' if !in_single && !in_double => { + let item = current.trim(); + if !item.is_empty() { + items.push(parse_string_value(item, field, line_number)?); + } + current.clear(); + } + _ => current.push(ch), + } + } + let item = current.trim(); + if !item.is_empty() { + items.push(parse_string_value(item, field, line_number)?); + } + Ok(items) +} + +fn parse_string_value(value: &str, field: &str, line_number: usize) -> anyhow::Result { + let trimmed = value.trim(); + if trimmed.len() < 2 { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 必须是字符串")); + } + let quote = trimmed.as_bytes()[0]; + if quote != b'\'' && quote != b'"' { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 必须使用引号")); + } + if trimmed.as_bytes()[trimmed.len() - 1] != quote { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 引号不匹配")); + } + let body = &trimmed[1..trimmed.len() - 1]; + if quote == b'\'' { + return Ok(body.to_string()); + } + let mut output = String::with_capacity(body.len()); + let mut escaped = false; + for ch in body.chars() { + if escaped { + let mapped = match ch { + '\\' => '\\', + '"' => '"', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '0' => '\0', + other => { + return Err(anyhow::anyhow!( + "第 {line_number} 行 {field} 包含不支持的转义：\\{other}" + )) + } + }; + output.push(mapped); + escaped = false; + } else if ch == '\\' { + escaped = true; + } else { + output.push(ch); + } + } + if escaped { + return Err(anyhow::anyhow!( + "第 {line_number} 行 {field} 的字符串以未完成的转义结束" + )); + } + Ok(output) +} + +fn is_valid_ident(value: &str) -> bool { + let mut chars = value.chars(); + let Some(first) = chars.next() else { + return false; + }; + (first == '_' || first.is_ascii_alphabetic()) + && chars.all(|character| character == '_' || character.is_ascii_alphanumeric()) +} + +fn parse_tail_lines(value: &str, field: &str, line_number: usize) -> anyhow::Result { + let parsed = parse_scalar_text(value, field, line_number)? + .parse::() + .map_err(|error| anyhow::anyhow!("第 {line_number} 行 {field} 无效：{error}"))?; + if parsed == 0 { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 必须大于 0")); + } + Ok(parsed) +} + +fn parse_scalar_text(value: &str, field: &str, line_number: usize) -> anyhow::Result { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Err(anyhow::anyhow!("第 {line_number} 行 {field} 不能为空")); + } + if trimmed.starts_with('"') || trimmed.starts_with('\'') { + return parse_string_value(trimmed, field, line_number); + } + Ok(trimmed.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn write_private_config(path: &Path, content: &str) { + fs::write(path, content).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = fs::metadata(path).unwrap().permissions(); + permissions.set_mode(PRIVATE_FILE_MODE); + fs::set_permissions(path, permissions).unwrap(); + } + } + + #[test] + fn template_is_parseable() { + let config = parse_config_document(CONFIG_TEMPLATE).unwrap(); + let mut options = CliOptions::default(); + config.apply_to_options(&mut options).unwrap(); + assert_eq!(options.config.output_root, PathBuf::from("./bat-resources")); + assert_eq!( + options.config.localized_output_root, + PathBuf::from("./bat-localized") + ); + assert_eq!(options.output_format, OutputFormat::Human); + assert_eq!(options.state_dir, PathBuf::from("/tmp/bat-pid")); + assert_eq!(options.worker_concurrency, 8); + assert_eq!(options.config.download_concurrency, 8); + assert!(options.config.server_info_source.is_none()); + } + + #[test] + fn config_file_is_written_when_missing() { + let temp = TempDir::new().unwrap(); + let loaded = load_from_binary_dir(temp.path()).unwrap(); + assert!(loaded.is_none()); + let example = temp.path().join(CONFIG_EXAMPLE_FILE_NAME); + assert!(example.exists()); + let content = fs::read_to_string(example).unwrap(); + let parsed = parse_config_document(&content).unwrap(); + let mut options = CliOptions::default(); + parsed.apply_to_options(&mut options).unwrap(); + assert_eq!(options.config.output_root, PathBuf::from("./bat-resources")); + } + + #[test] + fn existing_config_is_read_without_creating_or_modifying_example() { + let temp = TempDir::new().unwrap(); + let config_path = temp.path().join(CONFIG_FILE_NAME); + let example_path = temp.path().join(CONFIG_EXAMPLE_FILE_NAME); + write_private_config(&config_path, "[resource]\noutput_root = '/srv/config'\n"); + std::fs::write(&example_path, "example-sentinel").unwrap(); + + let loaded = load_from_binary_dir(temp.path()).unwrap().unwrap(); + assert_eq!( + loaded.resource.output_root, + Some(PathBuf::from("/srv/config")) + ); + assert_eq!( + std::fs::read_to_string(example_path).unwrap(), + "example-sentinel" + ); + } + + #[test] + fn example_config_is_never_loaded_as_actual_config() { + let temp = TempDir::new().unwrap(); + std::fs::write( + temp.path().join(CONFIG_EXAMPLE_FILE_NAME), + "[resource]\noutput_root = '/example-only'\n", + ) + .unwrap(); + + assert!(load_from_binary_dir(temp.path()).unwrap().is_none()); + } + + #[test] + fn invalid_actual_config_is_rejected() { + let cases = [ + "[resource]\noutput_root = '/unterminated\n", + "[network]\ndownload_concurrency = true\n", + "[network]\ndownload_concurrency = 0\n", + "[unknown]\nvalue = 'not-supported'\n", + ]; + + for contents in cases { + let temp = TempDir::new().unwrap(); + write_private_config(&temp.path().join(CONFIG_FILE_NAME), contents); + let error = load_from_binary_dir(temp.path()).unwrap_err(); + assert!(error.to_string().contains("config.toml")); + } + } + + #[cfg(unix)] + #[test] + fn group_readable_config_is_rejected_before_parsing() { + use std::os::unix::fs::PermissionsExt; + + let temp = TempDir::new().unwrap(); + let path = temp.path().join(CONFIG_FILE_NAME); + fs::write( + &path, + "[network]\nproxy = 'http://user:secret@example.invalid'\n", + ) + .unwrap(); + let mut permissions = fs::metadata(&path).unwrap().permissions(); + permissions.set_mode(0o640); + fs::set_permissions(&path, permissions).unwrap(); + + let error = load_from_binary_dir(temp.path()).unwrap_err(); + assert!(error.to_string().contains("0600")); + } + + #[test] + fn config_file_overrides_are_typed() { + let content = r#" +[runtime] +state_dir = '/srv/state' +interval_seconds = 12 +error_retry_seconds = 3 +quiet_up_to_date = true +output_format = 'json' +banner = false +progress = false +tail_lines = 7 + +[resource] +output_root = '/srv/bat' +auto_discover = true +app_version = '1.2.3' +connection_group = 'Prod' +launcher_version = '1.8.0' +platforms = ['windows'] +snapshot_path = '/tmp/snapshot.json' +dry_run = true +plan = true +force = true +audit_local = false +repair = false + +[resource.server_info] +kind = 'official_url' +value = 'https://example.invalid/server-info.json' + +[localized] +output_root = '/srv/bat-localized' + +[repository] +import_repository = true +import_cas_root = '/srv/cas' +import_resource_repository_path = '/srv/resources.sqlite' + +[network] +curl_command = '/usr/bin/curl' +proxy = 'http://127.0.0.1:7890' +unzip_command = '/usr/bin/unzip' +download_concurrency = 16 + +[translation.worker] +provider = 'crowdin' +fixture = '/tmp/mock.json' +translation_memory_path = '/srv/tm.sqlite' +concurrency = 4 +max_attempts = 5 +lease_seconds = 60 +retry_backoff_seconds = 0 +max_tasks = 2 +worker_id = 'worker-a' +"#; + let config = parse_config_document(content).unwrap(); + let mut options = CliOptions::default(); + config.apply_to_options(&mut options).unwrap(); + assert_eq!(options.state_dir, PathBuf::from("/srv/state")); + assert_eq!(options.output_format, OutputFormat::Json); + assert_eq!(options.config.output_root, PathBuf::from("/srv/bat")); + assert_eq!( + options.config.localized_output_root, + PathBuf::from("/srv/bat-localized") + ); + assert_eq!(options.config.download_concurrency, 16); + assert_eq!(options.worker_concurrency, 4); + assert_eq!( + options.translation_memory_path, + Some(PathBuf::from("/srv/tm.sqlite")) + ); + assert_eq!(options.worker_max_tasks, Some(2)); + assert_eq!(options.translation_provider.as_deref(), Some("crowdin")); + assert!(matches!( + options.config.server_info_source, + Some(OfficialServerInfoSource::OfficialUrl(ref value)) + if value == "https://example.invalid/server-info.json" + )); + } +} diff --git a/infrastructure/src/bin/bat/report_output.rs b/infrastructure/src/bin/bat/report_output.rs index f77a3ed..6625b2f 100644 --- a/infrastructure/src/bin/bat/report_output.rs +++ b/infrastructure/src/bin/bat/report_output.rs @@ -111,6 +111,13 @@ impl HumanReport for bat_infrastructure::TranslationWorkerReport { print_field("失败任务", self.failed_count); print_field("已安排重试", self.retry_scheduled_count); print_field("剩余任务", self.remaining_count); + print_path_field("Translation Memory", &self.translation_memory_path); + print_field("TM 可用", format_bool(self.translation_memory_available)); + print_field("TM 命中 TextUnit", self.translation_memory_hit_count); + print_field("Provider TextUnit", self.provider_unit_count); + for failure in &self.translation_memory_failures { + println!(" - TM: {failure}"); + } for failure in &self.failures { println!( " - {} [{}] retryable={} {}", diff --git a/infrastructure/src/bin/bat/task_registry.rs b/infrastructure/src/bin/bat/task_registry.rs index 76f7e97..fd41e61 100644 --- a/infrastructure/src/bin/bat/task_registry.rs +++ b/infrastructure/src/bin/bat/task_registry.rs @@ -519,6 +519,8 @@ pub(super) struct DaemonTaskContext { pub(super) registry: TaskRegistry, pub(super) queue: mpsc::Sender, pub(super) base_config: OfficialUpdateConfig, + /// daemon 中未显式传入参数的 translation worker 默认配置。 + pub(super) translation_worker_config: TranslationWorkerConfig, /// 串行化会读取或修改已发布资源状态的 daemon 操作。 pub(super) sync_lock: Arc>, pub(super) restart_controller: DaemonRestartController, @@ -562,11 +564,15 @@ pub(super) fn run_task_worker( let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; + let cancel_check = Arc::clone(&cancel); runtime - .block_on(bat_infrastructure::run_translation_worker_at( - &resource_root, - worker_config, - )) + .block_on( + bat_infrastructure::run_translation_worker_at_with_cancellation( + &resource_root, + worker_config, + Arc::new(move || cancel_check.load(Ordering::Relaxed)), + ), + ) .and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from)) }) } else { @@ -608,7 +614,8 @@ pub(super) fn run_task_worker( record.result = Some(report); }), Err(error) => { - let cancelled = cancel.load(Ordering::Relaxed); + let cancelled = + cancel.load(Ordering::Relaxed) || daemon_control_stop_requested(Some(&control)); // 下载失败携带类型化 DownloadError（含准确网络域码）；其余归 internal。 let code = error .downcast_ref::() diff --git a/infrastructure/src/bin/bat/terminal_output.rs b/infrastructure/src/bin/bat/terminal_output.rs index 909277e..d15697d 100644 --- a/infrastructure/src/bin/bat/terminal_output.rs +++ b/infrastructure/src/bin/bat/terminal_output.rs @@ -71,28 +71,24 @@ pub(super) fn print_startup_banner() { eprintln!("{STARTUP_BANNER}"); } -pub(super) fn print_env_template_created(path: &Path) { +pub(super) fn print_config_template_created(path: &Path) { eprintln!( - "已生成配置模板 {}（编辑其中的 BAT_* 配置后，直接运行 `bat` 即可按 .env 启动）", + "已生成配置模板 {}（编辑 `config.toml`，`config.toml.example` 不会被程序自动读取）", path.display() ); } -pub(super) fn print_env_template_warning(path: &Path, error: impl std::fmt::Display) { - eprintln!("警告：生成 .env 配置模板失败 {}：{error}", path.display()); -} - -pub(super) fn print_env_read_warning(path: &Path, error: impl std::fmt::Display) { - eprintln!("警告：读取 .env 失败 {}：{error}", path.display()); -} - -pub(super) fn print_env_parse_warning(line_number: usize, raw_line: &str) { +pub(super) fn print_config_template_warning(path: &Path, error: impl std::fmt::Display) { eprintln!( - "警告：.env 第 {} 行无法解析，已忽略：{raw_line}", - line_number + "警告：生成 config.toml.example 模板失败 {}：{error}", + path.display() ); } +pub(super) fn print_deprecated_env_file_warning() { + eprintln!("警告：BAT_SKIP_ENV_FILE 已废弃且不再影响启动，已忽略"); +} + #[derive(Debug, Clone)] pub(super) struct ProgressLogger { enabled: bool, diff --git a/infrastructure/src/bin/bat/translation_query.rs b/infrastructure/src/bin/bat/translation_query.rs index ecdfafb..1e17064 100644 --- a/infrastructure/src/bin/bat/translation_query.rs +++ b/infrastructure/src/bin/bat/translation_query.rs @@ -1,4 +1,8 @@ +use super::report_output::print_json_value; use super::*; +use bat_core::domain::TranslationMemoryContext; +use bat_core::repositories::TranslationMemoryRepository; +use std::collections::BTreeMap; #[derive(Debug, Deserialize)] struct TranslationTaskResultUpdateParam { @@ -332,6 +336,8 @@ fn build_manual_translation_results( unit_id: unit_id.to_string(), source_text: param.source_text.clone(), translated_text: param.translated_text.clone(), + source_kind: bat_infrastructure::TranslationTaskResultSourceKind::Manual, + translation_memory_record_id: None, provider: provider.to_string(), provider_run_id: provider_run_id.to_string(), translated_unix_seconds, @@ -355,3 +361,241 @@ pub(super) fn translation_task_query_json(query: &OfficialTextUnitTaskQuery) -> "has_failure_reason": query.has_failure_reason, }) } + +pub(super) fn run_translation_memory_command(options: &CliOptions) -> anyhow::Result<()> { + let method = match options.command { + CliCommand::TranslationMemorySummary => RPC_METHOD_TRANSLATION_MEMORY_SUMMARY, + CliCommand::TranslationMemoryQuery => RPC_METHOD_TRANSLATION_MEMORY_QUERY, + CliCommand::TranslationMemoryConfirm => RPC_METHOD_TRANSLATION_MEMORY_CONFIRM, + _ => return Err(anyhow::anyhow!("不是 Translation Memory 命令")), + }; + if daemon_rpc_available(&options.state_dir) + && options.resource_root.is_none() + && !options.output_explicit + { + let _control_lock = DaemonControlLock::acquire(&options.state_dir)?; + let report = daemon_rpc_call( + &options.state_dir, + method, + translation_memory_cli_params(options)?, + )?; + print_json_value(options.output_format, &report)?; + return Ok(()); + } + let path = translation_memory_cli_path(options)?; + let report = match options.command { + CliCommand::TranslationMemorySummary => build_translation_memory_summary_report(&path)?, + CliCommand::TranslationMemoryQuery => { + let source_text = options + .translation_memory_source_text + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?; + let context = parse_translation_memory_context( + options.translation_memory_context_json.as_deref(), + )?; + build_translation_memory_query_report( + &path, + source_text, + &context, + options.query_limit, + )? + } + CliCommand::TranslationMemoryConfirm => { + let record_id = options + .translation_memory_record_id + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?; + let reviewer = options + .translation_memory_reviewer + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?; + build_translation_memory_confirm_report( + &path, + record_id, + reviewer, + options.translation_memory_reason.clone(), + )? + } + _ => unreachable!(), + }; + print_json_value(options.output_format, &report) +} + +fn translation_memory_cli_path(options: &CliOptions) -> anyhow::Result { + if let Some(path) = options.translation_memory_path.as_ref() { + return lexical_absolute(path).map_err(anyhow::Error::msg); + } + let resource_root = options + .resource_root + .as_deref() + .map(lexical_absolute) + .transpose() + .map_err(anyhow::Error::msg)? + .unwrap_or(active_official_resource_root(&options.config.output_root)?); + Ok(bat_infrastructure::translation_memory_repository_path( + &resource_root, + )) +} + +fn translation_memory_cli_params( + options: &CliOptions, +) -> anyhow::Result> { + let mut params = serde_json::Map::new(); + if let Some(path) = options.translation_memory_path.as_ref() { + params.insert( + "translation_memory_path".to_string(), + serde_json::json!(path), + ); + } + match options.command { + CliCommand::TranslationMemorySummary => {} + CliCommand::TranslationMemoryQuery => { + let source_text = options + .translation_memory_source_text + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM query 必须指定 --tm-source-text"))?; + let context = parse_translation_memory_context( + options.translation_memory_context_json.as_deref(), + )?; + params.insert("source_text".to_string(), serde_json::json!(source_text)); + params.insert("source_context".to_string(), serde_json::json!(context)); + params.insert("limit".to_string(), serde_json::json!(options.query_limit)); + } + CliCommand::TranslationMemoryConfirm => { + let record_id = options + .translation_memory_record_id + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-record-id"))?; + let reviewer = options + .translation_memory_reviewer + .as_deref() + .ok_or_else(|| anyhow::anyhow!("TM confirm 必须指定 --tm-reviewer"))?; + params.insert("record_id".to_string(), serde_json::json!(record_id)); + params.insert("reviewer".to_string(), serde_json::json!(reviewer)); + if let Some(reason) = options.translation_memory_reason.as_deref() { + params.insert("reason".to_string(), serde_json::json!(reason)); + } + } + _ => unreachable!(), + } + Ok(Some(serde_json::Value::Object(params))) +} + +pub(super) fn build_translation_memory_summary_report( + path: &std::path::Path, +) -> anyhow::Result { + if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? { + return Ok(serde_json::json!({ + "available": false, + "path": path, + "reason": "database_missing", + })); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let summary = runtime.block_on(async { + let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .summary() + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })?; + Ok(serde_json::json!({ + "available": true, + "path": path, + "schema_version": summary.schema_version, + "summary": summary, + })) +} + +pub(super) fn build_translation_memory_query_report( + path: &std::path::Path, + source_text: &str, + source_context: &TranslationMemoryContext, + limit: usize, +) -> anyhow::Result { + if source_text.trim().is_empty() { + return Err(anyhow::anyhow!("TM query 的 source_text 不能为空")); + } + if !(1..=1000).contains(&limit) { + return Err(anyhow::anyhow!("TM query 的 limit 必须在 1..=1000 范围内")); + } + if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? { + return Ok(serde_json::json!({ + "available": false, + "path": path, + "source_text": source_text, + "source_context": source_context, + "matches": [], + "reason": "database_missing", + })); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let matches = runtime.block_on(async { + let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .find_matches(source_text, source_context, limit) + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })?; + Ok(serde_json::json!({ + "available": true, + "path": path, + "source_text": source_text, + "source_context": source_context, + "matches": matches, + })) +} + +pub(super) fn build_translation_memory_confirm_report( + path: &std::path::Path, + record_id: &str, + reviewer: &str, + reason: Option, +) -> anyhow::Result { + if record_id.trim().is_empty() || reviewer.trim().is_empty() { + return Err(anyhow::anyhow!( + "TM confirm 必须指定非空 record_id 和 reviewer" + )); + } + if !sqlite_file_exists_no_symlink(path, "Translation Memory 数据库")? { + return Err(anyhow::anyhow!( + "Translation Memory 数据库不存在：{}", + path.display() + )); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let entry = runtime.block_on(async { + let repository = bat_infrastructure::SqliteTranslationMemoryRepository::open(path) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + repository + .confirm(record_id, reviewer, reason) + .await + .map_err(|error| anyhow::anyhow!("{error}")) + })?; + Ok(serde_json::json!({ + "available": true, + "path": path, + "entry": entry, + })) +} + +fn parse_translation_memory_context( + value: Option<&str>, +) -> anyhow::Result { + let Some(value) = value else { + return Ok(BTreeMap::new()); + }; + serde_json::from_str::(value) + .map_err(|error| anyhow::anyhow!("--tm-context-json 必须是 JSON object：{error}")) +} diff --git a/infrastructure/src/bin/bat/workflow_commands.rs b/infrastructure/src/bin/bat/workflow_commands.rs index 6d072aa..1263dd5 100644 --- a/infrastructure/src/bin/bat/workflow_commands.rs +++ b/infrastructure/src/bin/bat/workflow_commands.rs @@ -261,26 +261,10 @@ pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()> .map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg)) .transpose()? .unwrap_or(active_official_resource_root(&options.config.output_root)?); - let provider = options - .translation_provider - .as_deref() - .unwrap_or(TranslationProviderKind::Mock.as_str()); - let provider = TranslationProviderKind::parse(provider) - .ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效：{provider}"))?; - let config = TranslationWorkerConfig { - provider, - fixture_path: options.translation_fixture.clone(), - concurrency: options.worker_concurrency, - max_attempts: options.worker_max_attempts, - lease_seconds: options.worker_lease_seconds, - retry_backoff: options.worker_retry_backoff, - max_tasks: options.worker_max_tasks, - worker_id: options - .worker_id - .clone() - .unwrap_or_else(|| format!("bat-worker-{}", std::process::id())), - }; - config.validate()?; + let config = super::translation_worker_config_from_options( + options, + &format!("bat-worker-{}", std::process::id()), + )?; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; diff --git a/infrastructure/src/lib.rs b/infrastructure/src/lib.rs index fbd61f4..5fd044b 100644 --- a/infrastructure/src/lib.rs +++ b/infrastructure/src/lib.rs @@ -29,6 +29,7 @@ pub mod patch_ops; pub mod path_security; pub mod release_flow; pub mod resources; +pub mod translation_memory; pub mod translation_tasks; pub mod translation_worker; pub mod translation_workflow; @@ -140,24 +141,29 @@ pub use path_security::{ }; pub use release_flow::ReleaseFlowStatusCode; pub use resources::{InMemoryResourceRepository, SqliteResourceRepository}; +pub use translation_memory::{ + translation_memory_context, translation_memory_repository_path, + SqliteTranslationMemoryRepository, TRANSLATION_MEMORY_REPOSITORY_FILE, + TRANSLATION_MEMORY_SCHEMA_COMPONENT, TRANSLATION_MEMORY_SCHEMA_VERSION, +}; pub use translation_tasks::{ build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at, write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState, ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff, - TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus, - TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus, - TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE, - TRANSLATION_TASK_SCHEMA_VERSION, + TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskResultSourceKind, + TranslationTaskStatus, TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, + TranslationUnitStatus, TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, + TRANSLATION_TASK_REPOSITORY_FILE, TRANSLATION_TASK_SCHEMA_VERSION, }; pub use translation_worker::{ - run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider, - MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass, - TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse, - TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig, - TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, - DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS, - DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, - MOCK_TRANSLATION_FIXTURE_VERSION, + run_translation_worker_at, run_translation_worker_at_with_cancellation, + run_translation_worker_with_provider, CrowdinProvider, MockTranslationProvider, + TranslationProvider, TranslationProviderFailureClass, TranslationProviderKind, + TranslationProviderRequest, TranslationProviderResponse, TranslationProviderUnit, + TranslationProviderUnitResult, TranslationWorkerConfig, TranslationWorkerFailure, + TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS, + DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF, + MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, MOCK_TRANSLATION_FIXTURE_VERSION, }; pub use translation_workflow::{ completed_worker_translation_workbench, export_completed_worker_translation_workbench, diff --git a/infrastructure/src/localized_patch.rs b/infrastructure/src/localized_patch.rs index 89afe3c..043f2fd 100644 --- a/infrastructure/src/localized_patch.rs +++ b/infrastructure/src/localized_patch.rs @@ -91,6 +91,12 @@ pub struct LocalizedPatchOperationMetadata { /// Provider run ID that produced the text, if applicable. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_run_id: Option, + /// Source kind of the translation result. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_source_kind: Option, + /// Trusted Translation Memory record used for the text, if applicable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_memory_record_id: Option, /// Review state used by the publication input. pub review_status: String, } @@ -331,6 +337,12 @@ pub struct LocalizedPatchOperation { /// Provider run ID that produced the text, if applicable. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_run_id: Option, + /// Source kind of the translation result. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_source_kind: Option, + /// Trusted Translation Memory record used for the text, if applicable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_memory_record_id: Option, /// Review state used by the publication input. #[serde(default, skip_serializing_if = "Option::is_none")] pub review_status: Option, @@ -941,6 +953,8 @@ impl LocalizedPatchOperation { source_text_blake3: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, review_status: None, }, metadata, @@ -966,6 +980,8 @@ impl LocalizedPatchOperation { source_text_blake3: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, review_status: None, }, metadata, @@ -990,6 +1006,8 @@ impl LocalizedPatchOperation { source_text_blake3: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, review_status: None, }, metadata, @@ -1005,6 +1023,8 @@ impl LocalizedPatchOperation { operation.source_text_blake3 = Some(metadata.source_text_blake3.clone()); operation.translation_provider = metadata.translation_provider.clone(); operation.provider_run_id = metadata.provider_run_id.clone(); + operation.translation_source_kind = metadata.translation_source_kind.clone(); + operation.translation_memory_record_id = metadata.translation_memory_record_id.clone(); operation.review_status = Some(metadata.review_status.clone()); } operation @@ -1029,6 +1049,8 @@ impl Default for LocalizedPatchOperation { source_text_blake3: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, review_status: None, } } @@ -1401,6 +1423,8 @@ mod tests { source_text_blake3: Some(blake3::hash(source).to_hex().to_string()), translation_provider: Some("mock".to_string()), provider_run_id: Some("mock:unit-1:attempt-1".to_string()), + translation_source_kind: Some("provider".to_string()), + translation_memory_record_id: None, review_status: Some("provider_completed".to_string()), }], }], diff --git a/infrastructure/src/translation_memory.rs b/infrastructure/src/translation_memory.rs new file mode 100644 index 0000000..56101fe --- /dev/null +++ b/infrastructure/src/translation_memory.rs @@ -0,0 +1,859 @@ +//! 跨 official release 的 Translation Memory SQLite 仓储。 + +use crate::path_security::{set_file_mode, STATE_FILE_MODE}; +use async_trait::async_trait; +use bat_core::domain::{ + TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry, + TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind, + TranslationMemorySummary, TranslationMemoryTrustStatus, +}; +use bat_core::repositories::TranslationMemoryRepository; +use bat_core::{Error, Result}; +use serde::de::DeserializeOwned; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; +use sqlx::{Row, SqlitePool}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// TM SQLite schema 版本。 +pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1; +/// TM schema migration component。 +pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory"; +/// 默认 TM 数据库文件名。 +pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite"; + +/// SQLite-backed Translation Memory 仓储。 +#[derive(Debug, Clone)] +pub struct SqliteTranslationMemoryRepository { + pub(crate) pool: SqlitePool, +} + +impl SqliteTranslationMemoryRepository { + /// 创建或打开 TM 数据库并执行迁移。 + pub async fn new(path: impl AsRef) -> Result { + Self::open_with(path.as_ref(), true).await + } + + /// 只打开已有 TM 数据库，不创建新文件。 + pub async fn open(path: impl AsRef) -> Result { + Self::open_with(path.as_ref(), false).await + } + + /// 根据 active release 根目录计算默认的跨 release TM 路径。 + /// + /// 正式 release 根目录形如 `/versions/`，因此默认结果为 + /// `/translation-memory.sqlite`，不会写入已发布版本目录。 + pub fn repository_path(resource_root: &Path) -> PathBuf { + if resource_root + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "versions") + { + if let Some(output_root) = resource_root.parent().and_then(Path::parent) { + return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE); + } + } + resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE) + } + + async fn open_with(path: &Path, create_if_missing: bool) -> Result { + let absolute = bat_infrastructure_absolute(path)?; + let parent = absolute.parent().ok_or_else(|| { + Error::InvalidArgument(format!("TM 数据库缺少父目录：{}", absolute.display())) + })?; + ensure_safe_tm_parent(parent)?; + if create_if_missing { + tokio::fs::create_dir_all(parent).await?; + ensure_safe_tm_parent(parent)?; + } + if let Ok(metadata) = fs::symlink_metadata(&absolute) { + if metadata.file_type().is_symlink() { + return Err(Error::InvalidArgument(format!( + "TM 数据库不能是 symlink：{}", + absolute.display() + ))); + } + if !metadata.is_file() { + return Err(Error::InvalidArgument(format!( + "TM 数据库不是普通文件：{}", + absolute.display() + ))); + } + } else if !create_if_missing { + return Err(Error::NotFound(absolute.display().to_string())); + } + + let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display())) + .map_err(|error| Error::Other(error.into()))? + .create_if_missing(create_if_missing) + .journal_mode(SqliteJournalMode::Wal) + .busy_timeout(Duration::from_secs(30)); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .map_err(db_error)?; + set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库") + .map_err(Error::InvalidArgument)?; + let repository = Self { pool }; + repository.init_schema().await?; + Ok(repository) + } + + async fn init_schema(&self) -> Result<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS schema_migrations ( + component TEXT PRIMARY KEY NOT NULL, + version INTEGER NOT NULL CHECK(version >= 1) + ) + "#, + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS translation_memory ( + record_id TEXT PRIMARY KEY NOT NULL, + source_text TEXT NOT NULL, + source_hash TEXT NOT NULL, + normalized_source_text TEXT NOT NULL, + source_context_json TEXT NOT NULL, + source_context_hash TEXT NOT NULL, + translated_text TEXT NOT NULL, + translation_source_kind TEXT NOT NULL, + trust_status TEXT NOT NULL, + official_release_id TEXT NOT NULL, + source_trace_json TEXT NOT NULL, + provider TEXT, + provider_run_id TEXT, + created_unix_seconds INTEGER NOT NULL, + updated_unix_seconds INTEGER NOT NULL, + trusted_unix_seconds INTEGER, + trusted_by TEXT, + trusted_reason TEXT, + supersedes_record_id TEXT, + superseded_by_record_id TEXT, + CHECK (length(source_text) > 0), + CHECK (length(source_hash) > 0), + CHECK (length(source_context_hash) > 0), + CHECK (length(official_release_id) > 0), + CHECK (translation_source_kind IN ('provider', 'manual', 'imported')), + CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected')) + ) + "#, + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \ + ON translation_memory(source_hash)", + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \ + ON translation_memory(normalized_source_text)", + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_translation_memory_context \ + ON translation_memory(source_hash, source_context_hash)", + ) + .execute(&self.pool) + .await + .map_err(db_error)?; + + let current: Option = + sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1") + .bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT) + .fetch_optional(&self.pool) + .await + .map_err(db_error)?; + if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) { + return Err(Error::InvalidArgument(format!( + "不支持的 Translation Memory schema 版本：{}", + current.unwrap_or_default() + ))); + } + sqlx::query( + r#" + INSERT INTO schema_migrations(component, version) + VALUES (?1, ?2) + ON CONFLICT(component) DO UPDATE SET version = excluded.version + "#, + ) + .bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT) + .bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) + .execute(&self.pool) + .await + .map_err(db_error)?; + Ok(()) + } + + async fn find_optional(&self, record_id: &str) -> Result> { + let row = sqlx::query( + r#" + SELECT record_id, source_text, source_hash, normalized_source_text, + source_context_json, source_context_hash, translated_text, + translation_source_kind, trust_status, official_release_id, + source_trace_json, provider, provider_run_id, + created_unix_seconds, updated_unix_seconds, trusted_unix_seconds, + trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id + FROM translation_memory + WHERE record_id = ?1 + "#, + ) + .bind(record_id) + .fetch_optional(&self.pool) + .await + .map_err(db_error)?; + row.map(row_to_entry).transpose() + } +} + +#[async_trait] +impl TranslationMemoryRepository for SqliteTranslationMemoryRepository { + async fn upsert_candidate( + &self, + draft: TranslationMemoryDraft, + ) -> Result { + validate_draft(&draft)?; + let entry = entry_from_draft(draft)?; + if let Some(existing) = self.find_optional(&entry.record_id).await? { + if existing.trust_status != TranslationMemoryTrustStatus::Candidate { + return Ok(existing); + } + let source_trace_json = serde_json::to_string(&entry.source_trace) + .map_err(|error| Error::Serialization(error.to_string()))?; + sqlx::query( + r#" + UPDATE translation_memory + SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4, + updated_unix_seconds = ?5 + WHERE record_id = ?1 AND trust_status = 'candidate' + "#, + ) + .bind(&entry.record_id) + .bind(source_trace_json) + .bind(&entry.provider) + .bind(&entry.provider_run_id) + .bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX)) + .execute(&self.pool) + .await + .map_err(db_error)?; + return self.find(&entry.record_id).await; + } + + let source_context_json = serde_json::to_string(&entry.source_context) + .map_err(|error| Error::Serialization(error.to_string()))?; + let source_trace_json = serde_json::to_string(&entry.source_trace) + .map_err(|error| Error::Serialization(error.to_string()))?; + let result = sqlx::query( + r#" + INSERT INTO translation_memory ( + record_id, source_text, source_hash, normalized_source_text, + source_context_json, source_context_hash, translated_text, + translation_source_kind, trust_status, official_release_id, + source_trace_json, provider, provider_run_id, + created_unix_seconds, updated_unix_seconds, + trusted_unix_seconds, trusted_by, trusted_reason, + supersedes_record_id, superseded_by_record_id + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, + ?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19) + ON CONFLICT(record_id) DO UPDATE SET + source_trace_json = excluded.source_trace_json, + provider = excluded.provider, + provider_run_id = excluded.provider_run_id, + updated_unix_seconds = excluded.updated_unix_seconds + WHERE translation_memory.trust_status = 'candidate' + "#, + ) + .bind(&entry.record_id) + .bind(&entry.source_text) + .bind(&entry.source_hash) + .bind(&entry.normalized_source_text) + .bind(source_context_json) + .bind(&entry.source_context_hash) + .bind(&entry.translated_text) + .bind(entry.translation_source_kind.as_str()) + .bind(entry.trust_status.as_str()) + .bind(&entry.official_release_id) + .bind(source_trace_json) + .bind(&entry.provider) + .bind(&entry.provider_run_id) + .bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX)) + .bind( + entry + .trusted_unix_seconds + .map(|value| i64::try_from(value).unwrap_or(i64::MAX)), + ) + .bind(&entry.trusted_by) + .bind(&entry.trusted_reason) + .bind(&entry.supersedes_record_id) + .bind(&entry.superseded_by_record_id) + .execute(&self.pool) + .await + .map_err(db_error)?; + if result.rows_affected() == 0 { + return self.find(&entry.record_id).await; + } + Ok(entry) + } + + async fn find_matches( + &self, + source_text: &str, + source_context: &TranslationMemoryContext, + limit: usize, + ) -> Result> { + if source_text.is_empty() { + return Err(Error::InvalidArgument( + "Translation Memory 查询 source_text 不能为空".to_string(), + )); + } + if limit == 0 { + return Err(Error::InvalidArgument( + "Translation Memory 查询 limit 必须大于 0".to_string(), + )); + } + let source_hash = hash_text(source_text); + let normalized_source_text = normalize_source_text(source_text); + let rows = sqlx::query( + r#" + SELECT record_id, source_text, source_hash, normalized_source_text, + source_context_json, source_context_hash, translated_text, + translation_source_kind, trust_status, official_release_id, + source_trace_json, provider, provider_run_id, + created_unix_seconds, updated_unix_seconds, trusted_unix_seconds, + trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id + FROM translation_memory + WHERE source_hash = ?1 OR normalized_source_text = ?2 + ORDER BY updated_unix_seconds DESC, record_id ASC + "#, + ) + .bind(source_hash) + .bind(&normalized_source_text) + .fetch_all(&self.pool) + .await + .map_err(db_error)?; + let mut matches = rows + .into_iter() + .map(row_to_entry) + .collect::>>()? + .into_iter() + .filter_map(|entry| { + let raw_exact = entry.source_text == source_text; + let normalized_exact = entry.normalized_source_text == normalized_source_text; + if !raw_exact && !normalized_exact { + return None; + } + let same_context = !source_context.is_empty() + && !entry.source_context.is_empty() + && entry.source_context == *source_context; + let strong = raw_exact + && same_context + && entry.trust_status == TranslationMemoryTrustStatus::Trusted; + let match_kind = if strong { + TranslationMemoryMatchKind::StrongExact + } else if raw_exact { + TranslationMemoryMatchKind::CandidateExact + } else { + TranslationMemoryMatchKind::SourceOnly + }; + Some(TranslationMemoryMatch { + can_auto_reuse: strong, + entry, + match_kind, + }) + }) + .collect::>(); + matches.sort_by(|left, right| { + match_rank(left) + .cmp(&match_rank(right)) + .then_with(|| { + right + .entry + .updated_unix_seconds + .cmp(&left.entry.updated_unix_seconds) + }) + .then_with(|| left.entry.record_id.cmp(&right.entry.record_id)) + }); + matches.truncate(limit); + Ok(matches) + } + + async fn confirm( + &self, + record_id: &str, + reviewer: &str, + reason: Option, + ) -> Result { + if reviewer.trim().is_empty() { + return Err(Error::InvalidArgument( + "Translation Memory reviewer 不能为空".to_string(), + )); + } + let current = self.find(record_id).await?; + if current.trust_status == TranslationMemoryTrustStatus::Trusted { + return Ok(current); + } + if current.trust_status != TranslationMemoryTrustStatus::Candidate { + return Err(Error::InvalidArgument(format!( + "Translation Memory 记录 {} 当前状态为 {}，不能确认", + record_id, + current.trust_status.as_str() + ))); + } + let now = unix_seconds_now(); + sqlx::query( + r#" + UPDATE translation_memory + SET trust_status = 'trusted', trusted_unix_seconds = ?2, + trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2 + WHERE record_id = ?1 AND trust_status = 'candidate' + "#, + ) + .bind(record_id) + .bind(i64::try_from(now).unwrap_or(i64::MAX)) + .bind(reviewer.trim()) + .bind(reason.filter(|value| !value.trim().is_empty())) + .execute(&self.pool) + .await + .map_err(db_error)?; + self.find(record_id).await + } + + async fn find(&self, record_id: &str) -> Result { + self.find_optional(record_id) + .await? + .ok_or_else(|| Error::NotFound(record_id.to_string())) + } + + async fn summary(&self) -> Result { + let row = sqlx::query( + r#" + SELECT COUNT(*) AS record_count, + SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count, + SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count, + SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count, + SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count + FROM translation_memory + "#, + ) + .fetch_one(&self.pool) + .await + .map_err(db_error)?; + Ok(TranslationMemorySummary { + schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION, + record_count: row.try_get::("record_count").map_err(db_error)? as u64, + trusted_count: row.try_get::("trusted_count").map_err(db_error)? as u64, + candidate_count: row.try_get::("candidate_count").map_err(db_error)? as u64, + superseded_count: row + .try_get::("superseded_count") + .map_err(db_error)? as u64, + rejected_count: row.try_get::("rejected_count").map_err(db_error)? as u64, + }) + } +} + +/// 由 active official release 根目录计算默认 TM 数据库路径。 +pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf { + SqliteTranslationMemoryRepository::repository_path(resource_root) +} + +/// 从 TextUnit 定位字段构建 TM 上下文。 +#[allow(clippy::too_many_arguments)] +pub fn translation_memory_context( + destination: &str, + archive_entry: Option<&str>, + serialized_file: Option<&str>, + path_id: Option, + class_id: Option, + field_path: Option<&str>, + format: Option<&str>, + asset_name: Option<&str>, + text_source_kind: Option<&str>, + parser_context: &TranslationMemoryContext, +) -> TranslationMemoryContext { + let mut context = TranslationMemoryContext::new(); + context.insert("destination".to_string(), destination.to_string()); + insert_optional(&mut context, "archive_entry", archive_entry); + insert_optional(&mut context, "serialized_file", serialized_file); + if let Some(value) = path_id { + context.insert("path_id".to_string(), value.to_string()); + } + if let Some(value) = class_id { + context.insert("class_id".to_string(), value.to_string()); + } + insert_optional(&mut context, "field_path", field_path); + insert_optional(&mut context, "format", format); + insert_optional(&mut context, "asset_name", asset_name); + insert_optional(&mut context, "text_source_kind", text_source_kind); + for (key, value) in parser_context { + context.insert(format!("context.{key}"), value.clone()); + } + context +} + +fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) { + if let Some(value) = value.filter(|value| !value.is_empty()) { + context.insert(key.to_string(), value.to_string()); + } +} + +fn entry_from_draft(draft: TranslationMemoryDraft) -> Result { + let source_hash = hash_text(&draft.source_text); + let normalized_source_text = normalize_source_text(&draft.source_text); + let source_context_hash = hash_context(&draft.source_context)?; + let source_kind = draft.translation_source_kind; + let record_id = record_id( + &source_hash, + &source_context_hash, + &draft.translated_text, + &draft.official_release_id, + &source_kind, + ); + let observed = draft.observed_unix_seconds; + Ok(TranslationMemoryEntry { + record_id, + source_text: draft.source_text, + source_hash, + normalized_source_text, + source_context: draft.source_context, + source_context_hash, + translated_text: draft.translated_text, + translation_source_kind: source_kind, + trust_status: TranslationMemoryTrustStatus::Candidate, + official_release_id: draft.official_release_id, + source_trace: draft.source_trace, + provider: draft.provider, + provider_run_id: draft.provider_run_id, + created_unix_seconds: observed, + updated_unix_seconds: observed, + trusted_unix_seconds: None, + trusted_by: None, + trusted_reason: None, + supersedes_record_id: None, + superseded_by_record_id: None, + }) +} + +fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> { + if draft.source_text.is_empty() { + return Err(Error::InvalidArgument( + "Translation Memory source_text 不能为空".to_string(), + )); + } + if draft.translated_text.trim().is_empty() { + return Err(Error::InvalidArgument( + "Translation Memory translated_text 不能为空".to_string(), + )); + } + if draft.official_release_id.trim().is_empty() + || draft.source_trace.official_release_id.trim().is_empty() + { + return Err(Error::InvalidArgument( + "Translation Memory official_release_id 不能为空".to_string(), + )); + } + if draft.official_release_id != draft.source_trace.official_release_id { + return Err(Error::InvalidArgument( + "Translation Memory draft 的 release provenance 不一致".to_string(), + )); + } + Ok(()) +} + +fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result { + let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?; + let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?; + Ok(TranslationMemoryEntry { + record_id: row.try_get("record_id").map_err(db_error)?, + source_text: row.try_get("source_text").map_err(db_error)?, + source_hash: row.try_get("source_hash").map_err(db_error)?, + normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?, + source_context, + source_context_hash: row.try_get("source_context_hash").map_err(db_error)?, + translated_text: row.try_get("translated_text").map_err(db_error)?, + translation_source_kind: parse_source_kind( + row.try_get::("translation_source_kind") + .map_err(db_error)? + .as_str(), + )?, + trust_status: parse_trust_status( + row.try_get::("trust_status") + .map_err(db_error)? + .as_str(), + )?, + official_release_id: row.try_get("official_release_id").map_err(db_error)?, + source_trace, + provider: row.try_get("provider").map_err(db_error)?, + provider_run_id: row.try_get("provider_run_id").map_err(db_error)?, + created_unix_seconds: i64_to_u64( + row.try_get("created_unix_seconds").map_err(db_error)?, + "created", + )?, + updated_unix_seconds: i64_to_u64( + row.try_get("updated_unix_seconds").map_err(db_error)?, + "updated", + )?, + trusted_unix_seconds: optional_i64_to_u64( + row.try_get("trusted_unix_seconds").map_err(db_error)?, + "trusted", + )?, + trusted_by: row.try_get("trusted_by").map_err(db_error)?, + trusted_reason: row.try_get("trusted_reason").map_err(db_error)?, + supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?, + superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?, + }) +} + +fn parse_json(value: String) -> Result { + serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string())) +} + +fn parse_source_kind(value: &str) -> Result { + match value { + "provider" => Ok(TranslationMemorySourceKind::Provider), + "manual" => Ok(TranslationMemorySourceKind::Manual), + "imported" => Ok(TranslationMemorySourceKind::Imported), + _ => Err(Error::Serialization(format!( + "未知 Translation Memory source kind：{value}" + ))), + } +} + +fn parse_trust_status(value: &str) -> Result { + match value { + "candidate" => Ok(TranslationMemoryTrustStatus::Candidate), + "trusted" => Ok(TranslationMemoryTrustStatus::Trusted), + "superseded" => Ok(TranslationMemoryTrustStatus::Superseded), + "rejected" => Ok(TranslationMemoryTrustStatus::Rejected), + _ => Err(Error::Serialization(format!( + "未知 Translation Memory trust status：{value}" + ))), + } +} + +fn match_rank(value: &TranslationMemoryMatch) -> u8 { + match value.match_kind { + TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0, + TranslationMemoryMatchKind::CandidateExact => 1, + TranslationMemoryMatchKind::SourceOnly => 2, + TranslationMemoryMatchKind::StrongExact => 1, + } +} + +fn hash_text(value: &str) -> String { + blake3::hash(value.as_bytes()).to_hex().to_string() +} + +fn normalize_source_text(value: &str) -> String { + value.replace("\r\n", "\n").replace('\r', "\n") +} + +fn hash_context(context: &TranslationMemoryContext) -> Result { + let bytes = + serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?; + Ok(blake3::hash(&bytes).to_hex().to_string()) +} + +fn record_id( + source_hash: &str, + context_hash: &str, + translated_text: &str, + official_release_id: &str, + source_kind: &TranslationMemorySourceKind, +) -> String { + let mut key = Vec::new(); + for value in [ + source_hash, + context_hash, + translated_text, + official_release_id, + source_kind.as_str(), + ] { + key.extend_from_slice(value.as_bytes()); + key.push(0); + } + format!("tm-{}", blake3::hash(&key).to_hex()) +} + +fn unix_seconds_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn i64_to_u64(value: i64, label: &str) -> Result { + u64::try_from(value) + .map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效"))) +} + +fn optional_i64_to_u64(value: Option, label: &str) -> Result> { + value.map(|value| i64_to_u64(value, label)).transpose() +} + +fn db_error(error: sqlx::Error) -> Error { + Error::Other(error.into()) +} + +fn bat_infrastructure_absolute(path: &Path) -> Result { + crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument) +} + +fn ensure_safe_tm_parent(parent: &Path) -> Result<()> { + crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库") + .map_err(Error::InvalidArgument) +} + +#[cfg(test)] +mod tests { + use super::*; + use bat_core::domain::TranslationMemorySourceTrace; + + fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft { + let source_trace = TranslationMemorySourceTrace { + official_release_id: release.to_string(), + unit_id: Some(format!("{release}-unit")), + task_id: Some(format!("{release}-task")), + destination: Some("Bundles/story.bundle".to_string()), + archive_entry: None, + serialized_file: Some("CAB-story".to_string()), + path_id: Some(1), + class_id: Some(49), + field_path: Some("m_Text".to_string()), + format: Some("plain".to_string()), + asset_name: Some("Story".to_string()), + text_source_kind: Some("text_asset".to_string()), + source_url: Some("https://example.invalid/story".to_string()), + }; + TranslationMemoryDraft { + source_text: source.to_string(), + source_context: translation_memory_context( + "Bundles/story.bundle", + None, + Some("CAB-story"), + Some(1), + Some(49), + Some("m_Text"), + Some("plain"), + Some("Story"), + Some("text_asset"), + &TranslationMemoryContext::new(), + ), + translated_text: translated.to_string(), + translation_source_kind: TranslationMemorySourceKind::Provider, + official_release_id: release.to_string(), + source_trace, + provider: Some("mock".to_string()), + provider_run_id: Some(format!("mock:{release}")), + observed_unix_seconds: 1, + } + } + + #[tokio::test] + async fn initializes_schema_and_reuses_trusted_entry_across_releases() { + let temp = tempfile::TempDir::new().unwrap(); + let repository = SqliteTranslationMemoryRepository::new( + temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE), + ) + .await + .unwrap(); + let entry = repository + .upsert_candidate(draft("release-1", "Hello", "你好")) + .await + .unwrap(); + assert_eq!(repository.summary().await.unwrap().candidate_count, 1); + let trusted = repository + .confirm(&entry.record_id, "reviewer", Some("accepted".to_string())) + .await + .unwrap(); + assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted); + + let query = draft("release-2", "Hello", "ignored"); + let matches = repository + .find_matches("Hello", &query.source_context, 10) + .await + .unwrap(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].match_kind, + TranslationMemoryMatchKind::StrongExact + ); + assert!(matches[0].can_auto_reuse); + assert_eq!(matches[0].entry.translated_text, "你好"); + assert_eq!(matches[0].entry.official_release_id, "release-1"); + } + + #[tokio::test] + async fn rejects_future_schema_version() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("tm.sqlite"); + let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap(); + sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1") + .bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT) + .bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1) + .execute(&repository.pool) + .await + .unwrap(); + repository.pool.close().await; + + let error = SqliteTranslationMemoryRepository::new(&path) + .await + .unwrap_err(); + assert!(error + .to_string() + .contains("不支持的 Translation Memory schema")); + } + + #[tokio::test] + async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() { + let temp = tempfile::TempDir::new().unwrap(); + let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite")) + .await + .unwrap(); + let first = draft("release-1", "Hello", "你好"); + repository.upsert_candidate(first.clone()).await.unwrap(); + repository.upsert_candidate(first).await.unwrap(); + assert_eq!(repository.summary().await.unwrap().record_count, 1); + + let mut different = draft("release-2", "Hello", "你好"); + different + .source_context + .insert("field_path".to_string(), "m_Other".to_string()); + let matches = repository + .find_matches("Hello", &different.source_context, 10) + .await + .unwrap(); + assert_eq!( + matches[0].match_kind, + TranslationMemoryMatchKind::CandidateExact + ); + assert!(!matches[0].can_auto_reuse); + } + + #[tokio::test] + async fn rejects_empty_translation_candidates() { + let temp = tempfile::TempDir::new().unwrap(); + let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite")) + .await + .unwrap(); + let error = repository + .upsert_candidate(draft("release-1", "Hello", " \n")) + .await + .unwrap_err(); + assert!(error.to_string().contains("translated_text")); + assert_eq!(repository.summary().await.unwrap().record_count, 0); + } +} diff --git a/infrastructure/src/translation_tasks.rs b/infrastructure/src/translation_tasks.rs index 444c42c..8a05d6a 100644 --- a/infrastructure/src/translation_tasks.rs +++ b/infrastructure/src/translation_tasks.rs @@ -161,6 +161,12 @@ pub struct TranslationTaskUnitResult { pub source_text: String, /// Provider-produced or human-supplied translation. pub translated_text: String, + /// Result source kind. + #[serde(default)] + pub source_kind: TranslationTaskResultSourceKind, + /// Trusted Translation Memory record used for this result, when applicable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_memory_record_id: Option, /// Provider identifier. pub provider: String, /// Provider run that produced this result. @@ -169,6 +175,30 @@ pub struct TranslationTaskUnitResult { pub translated_unix_seconds: u64, } +/// Source of one persisted TextUnit translation result. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TranslationTaskResultSourceKind { + /// Result returned by the configured provider. + #[default] + Provider, + /// Result submitted through the manual task update interface. + Manual, + /// Result reused from a trusted Translation Memory entry. + TranslationMemory, +} + +impl TranslationTaskResultSourceKind { + /// Returns the stable JSON label. + pub const fn as_str(self) -> &'static str { + match self { + Self::Provider => "provider", + Self::Manual => "manual", + Self::TranslationMemory => "translation_memory", + } + } +} + /// One provider execution associated with one or more translation units. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ProviderRun { @@ -247,12 +277,18 @@ pub fn build_translation_handoff( Vec::new(), ) }); - let unit_status = match state.0 { - TranslationTaskStatus::Queued => TranslationUnitStatus::Queued, - TranslationTaskStatus::Running => TranslationUnitStatus::Translating, - TranslationTaskStatus::Failed => TranslationUnitStatus::Failed, - TranslationTaskStatus::Completed => TranslationUnitStatus::Translated, - TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped, + let unit_status = if state.5.is_empty() { + match state.0 { + TranslationTaskStatus::Queued => TranslationUnitStatus::Queued, + TranslationTaskStatus::Running => TranslationUnitStatus::Translating, + TranslationTaskStatus::Failed => TranslationUnitStatus::Failed, + TranslationTaskStatus::Completed => TranslationUnitStatus::Translated, + TranslationTaskStatus::Skipped => TranslationUnitStatus::Skipped, + } + } else { + // A task can retain successful TM hits while the remaining provider + // units are failed or waiting for retry. + TranslationUnitStatus::Translated }; let unit = TranslationUnit { unit_id: task.task_id.clone(), @@ -1168,11 +1204,29 @@ impl SqliteTranslationTaskRepository { pub async fn fail_claim( &self, failure: TranslationTaskFailure, + ) -> Result { + self.fail_claim_with_results(failure, &[]).await + } + + /// Records a provider failure while retaining any already-resolved TextUnit + /// results, such as trusted Translation Memory hits. + pub async fn fail_claim_with_results( + &self, + failure: TranslationTaskFailure, + translation_results: &[TranslationTaskUnitResult], ) -> Result { let now = unix_seconds_now_i64(); let next_attempt = failure .next_attempt_unix_seconds .map(|value| i64::try_from(value).unwrap_or(i64::MAX)); + let translation_results_json = if translation_results.is_empty() { + None + } else { + Some( + serde_json::to_string(translation_results) + .map_err(|error| bat_core::Error::Serialization(error.to_string()))?, + ) + }; let result = sqlx::query( r#" UPDATE translation_tasks @@ -1183,11 +1237,12 @@ impl SqliteTranslationTaskRepository { lease_expires_unix_seconds = NULL, failure_class = ?4, failure_retryable = ?5, - next_attempt_unix_seconds = ?6 + next_attempt_unix_seconds = ?6, + translation_results_json = COALESCE(?7, translation_results_json) WHERE task_id = ?1 AND worker_status = 'running' - AND lease_owner = ?7 - AND provider_run_id = ?8 + AND lease_owner = ?8 + AND provider_run_id = ?9 "#, ) .bind(&failure.task_id) @@ -1196,6 +1251,7 @@ impl SqliteTranslationTaskRepository { .bind(&failure.failure_class) .bind(if failure.retryable { 1_i64 } else { 0_i64 }) .bind(next_attempt) + .bind(translation_results_json) .bind(&failure.worker_id) .bind(&failure.provider_run_id) .execute(&self.pool) @@ -1747,6 +1803,8 @@ mod tests { unit_id: "unit-a".to_string(), source_text: "source".to_string(), translated_text: "manual translation".to_string(), + source_kind: TranslationTaskResultSourceKind::Manual, + translation_memory_record_id: None, provider: "manual".to_string(), provider_run_id: "manual-run-1".to_string(), translated_unix_seconds: 321, @@ -1864,6 +1922,8 @@ mod tests { unit_id: "unit-a".to_string(), source_text: "source".to_string(), translated_text: "translated".to_string(), + source_kind: TranslationTaskResultSourceKind::Provider, + translation_memory_record_id: None, provider: "mock".to_string(), provider_run_id: second_run.clone(), translated_unix_seconds: 1, diff --git a/infrastructure/src/translation_worker.rs b/infrastructure/src/translation_worker.rs index 4aad1e7..07d1584 100644 --- a/infrastructure/src/translation_worker.rs +++ b/infrastructure/src/translation_worker.rs @@ -2,15 +2,21 @@ //! //! worker 只消费已发布 release 中的 TextUnit 索引和 SQLite 任务状态，不 //! 修改官方资源。provider 的输入、输出和错误分类是稳定 contract；状态、 -//! 租约和译文结果始终写入 `translation-tasks.sqlite`。 +//! 租约和任务结果写入 release 级 `translation-tasks.sqlite`，跨 release 的 +//! Translation Memory 写入项目级独立 SQLite 数据库。 use crate::official_parse::{read_textunit_index_at, OfficialTextUnitIndexUnit}; use crate::official_textunit_queue::read_textunit_task_queue_at; +use crate::translation_memory::{translation_memory_context, SqliteTranslationMemoryRepository}; use crate::translation_tasks::{ PersistedTranslationTask, SqliteTranslationTaskRepository, TranslationTaskFailure, - TranslationTaskUnitResult, + TranslationTaskResultSourceKind, TranslationTaskUnitResult, }; use async_trait::async_trait; +use bat_core::domain::{ + TranslationMemoryDraft, TranslationMemorySourceKind, TranslationMemorySourceTrace, +}; +use bat_core::repositories::TranslationMemoryRepository; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::env; @@ -89,6 +95,9 @@ pub struct TranslationWorkerConfig { pub max_tasks: Option, /// worker 实例前缀，用于 lease 诊断。 pub worker_id: String, + /// Translation Memory SQLite path. `None` uses the output-root default. + #[serde(skip_serializing_if = "Option::is_none")] + pub translation_memory_path: Option, } impl Default for TranslationWorkerConfig { @@ -102,6 +111,7 @@ impl Default for TranslationWorkerConfig { retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF, max_tasks: None, worker_id: format!("bat-worker-{}", std::process::id()), + translation_memory_path: None, } } } @@ -576,6 +586,16 @@ pub struct TranslationWorkerReport { pub remaining_count: usize, /// 失败诊断。 pub failures: Vec, + /// Translation Memory database path used by this run. + pub translation_memory_path: PathBuf, + /// Whether the Translation Memory repository was available. + pub translation_memory_available: bool, + /// Number of TextUnits reused from trusted Translation Memory. + pub translation_memory_hit_count: usize, + /// Number of TextUnits sent to the provider after Translation Memory lookup. + pub provider_unit_count: usize, + /// Translation Memory diagnostics that did not invalidate provider work. + pub translation_memory_failures: Vec, } /// worker 失败诊断。 @@ -599,7 +619,10 @@ struct WorkerStats { completed_count: AtomicUsize, failed_count: AtomicUsize, retry_scheduled_count: AtomicUsize, + translation_memory_hit_count: AtomicUsize, + provider_unit_count: AtomicUsize, failures: Mutex>, + translation_memory_failures: Mutex>, } struct WorkerTaskContext<'a> { @@ -611,12 +634,25 @@ struct WorkerTaskContext<'a> { max_attempts: u32, retry_backoff: Duration, stats: &'a WorkerStats, + translation_memory: Option<&'a dyn TranslationMemoryRepository>, } /// 运行一个 provider worker 轮次。 pub async fn run_translation_worker_at( resource_root: &Path, config: &TranslationWorkerConfig, +) -> anyhow::Result { + run_translation_worker_at_with_cancellation(resource_root, config, Arc::new(|| false)).await +} + +/// 运行一个可协作取消的 provider worker 轮次。 +/// +/// 取消只在 claim 循环边界检查；已经开始的 provider 请求会先完成， +/// 避免丢失 lease 结果或遗留未清理的外部子进程。 +pub async fn run_translation_worker_at_with_cancellation( + resource_root: &Path, + config: &TranslationWorkerConfig, + should_cancel: Arc bool + Send + Sync>, ) -> anyhow::Result { config.validate()?; let provider: Arc = match config.provider { @@ -625,7 +661,13 @@ pub async fn run_translation_worker_at( )?), TranslationProviderKind::Crowdin => Arc::new(CrowdinProvider::from_env()?), }; - run_translation_worker_with_provider(resource_root, config, provider).await + run_translation_worker_with_provider_and_cancellation( + resource_root, + config, + provider, + should_cancel, + ) + .await } /// 使用指定 provider 运行 worker，供测试和插件宿主使用。 @@ -633,6 +675,21 @@ pub async fn run_translation_worker_with_provider( resource_root: &Path, config: &TranslationWorkerConfig, provider: Arc, +) -> anyhow::Result { + run_translation_worker_with_provider_and_cancellation( + resource_root, + config, + provider, + Arc::new(|| false), + ) + .await +} + +async fn run_translation_worker_with_provider_and_cancellation( + resource_root: &Path, + config: &TranslationWorkerConfig, + provider: Arc, + should_cancel: Arc bool + Send + Sync>, ) -> anyhow::Result { config.validate()?; let queue = read_textunit_task_queue_at(resource_root) @@ -643,6 +700,21 @@ pub async fn run_translation_worker_with_provider( .map_err(anyhow::Error::msg)? .ok_or_else(|| anyhow::anyhow!("缺少官方 TextUnit 明细索引"))?, ); + let translation_memory_path = config + .translation_memory_path + .clone() + .unwrap_or_else(|| SqliteTranslationMemoryRepository::repository_path(resource_root)); + let (translation_memory, translation_memory_startup_failure) = + match SqliteTranslationMemoryRepository::new(&translation_memory_path).await { + Ok(repository) => (Some(Arc::new(repository)), None), + Err(error) => ( + None, + Some(format!( + "打开 Translation Memory 数据库失败 {}：{error}", + translation_memory_path.display() + )), + ), + }; let repository = Arc::new( SqliteTranslationTaskRepository::new(SqliteTranslationTaskRepository::repository_path( resource_root, @@ -659,6 +731,13 @@ pub async fn run_translation_worker_with_provider( .await .map_err(|error| anyhow::anyhow!("回收翻译 worker lease 失败：{error}"))?; let stats = Arc::new(WorkerStats::default()); + if let Some(failure) = translation_memory_startup_failure { + stats + .translation_memory_failures + .lock() + .map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))? + .push(failure); + } let claimed_limit = Arc::new(AtomicUsize::new(0)); let mut handles = Vec::with_capacity(config.concurrency); @@ -674,8 +753,13 @@ pub async fn run_translation_worker_with_provider( let max_attempts = config.max_attempts; let lease_seconds = config.lease_seconds; let retry_backoff = config.retry_backoff; + let translation_memory = translation_memory.clone(); + let should_cancel = Arc::clone(&should_cancel); handles.push(tokio::spawn(async move { loop { + if should_cancel() { + return Err(anyhow::anyhow!("翻译 worker 已取消")); + } if let Some(max_tasks) = max_tasks { let reservation = claimed_limit.fetch_add(1, Ordering::AcqRel); if reservation >= max_tasks { @@ -704,6 +788,9 @@ pub async fn run_translation_worker_with_provider( max_attempts, retry_backoff, stats: &stats, + translation_memory: translation_memory + .as_deref() + .map(|repository| repository as &dyn TranslationMemoryRepository), }, &task, ) @@ -712,10 +799,24 @@ pub async fn run_translation_worker_with_provider( Ok::<(), anyhow::Error>(()) })); } + let mut first_worker_error = None; for handle in handles { - handle - .await - .map_err(|error| anyhow::anyhow!("等待翻译 worker 失败：{error}"))??; + match handle.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + if first_worker_error.is_none() { + first_worker_error = Some(error); + } + } + Err(error) => { + if first_worker_error.is_none() { + first_worker_error = Some(anyhow::anyhow!("等待翻译 worker 失败：{error}")); + } + } + } + } + if let Some(error) = first_worker_error { + return Err(error); } let remaining_count = repository @@ -736,6 +837,11 @@ pub async fn run_translation_worker_with_provider( .map_err(|_| anyhow::anyhow!("读取翻译 worker 失败列表时 mutex poisoned"))? .clone(); let failed_count = stats.failed_count.load(Ordering::Relaxed); + let translation_memory_failures = stats + .translation_memory_failures + .lock() + .map_err(|_| anyhow::anyhow!("读取 Translation Memory 诊断时 mutex poisoned"))? + .clone(); Ok(TranslationWorkerReport { command: "translation-worker", status: if failed_count == 0 { @@ -752,6 +858,11 @@ pub async fn run_translation_worker_with_provider( retry_scheduled_count: stats.retry_scheduled_count.load(Ordering::Relaxed), remaining_count, failures, + translation_memory_path, + translation_memory_available: translation_memory.is_some(), + translation_memory_hit_count: stats.translation_memory_hit_count.load(Ordering::Relaxed), + provider_unit_count: stats.provider_unit_count.load(Ordering::Relaxed), + translation_memory_failures, }) } @@ -759,66 +870,176 @@ async fn process_claimed_task( context: &WorkerTaskContext<'_>, task: &PersistedTranslationTask, ) -> anyhow::Result<()> { - let request = match provider_request(task, context.index) { - Ok(request) => request, - Err(error) => { - record_provider_failure( - context, - task, - TranslationProviderError::new( - TranslationProviderFailureClass::InvalidRequest, - error.to_string(), - ), - ) - .await?; - return Ok(()); - } - }; - match context.provider.translate(request.clone()).await { - Ok(response) => { - let results = - match validate_provider_response(&request, response, context.provider_name) { - Ok(results) => results, - Err(error) => { - record_provider_failure( - context, - task, - TranslationProviderError::new( - TranslationProviderFailureClass::InvalidRequest, - error.to_string(), - ), - ) - .await?; - return Ok(()); + let task_units = task_index_units(task, context.index)?; + let mut results = BTreeMap::new(); + let mut provider_units = Vec::new(); + for unit in &task_units { + if let Some(translation_memory) = context.translation_memory { + let source_context = translation_memory_context( + &unit.destination, + unit.archive_entry.as_deref(), + unit.serialized_file.as_deref(), + unit.path_id, + unit.class_id, + unit.field_path.as_deref(), + unit.format.as_deref(), + unit.asset_name.as_deref(), + unit.text_source_kind.as_deref(), + &unit.context, + ); + match translation_memory + .find_matches(&unit.source_text, &source_context, 1) + .await + { + Ok(matches) => { + if let Some(found) = matches.into_iter().find(|item| item.can_auto_reuse) { + context + .stats + .translation_memory_hit_count + .fetch_add(1, Ordering::Relaxed); + results.insert( + unit.id.clone(), + translation_memory_result(task, unit, &found.entry), + ); + continue; } - }; - context - .repository - .complete_claim( - &task.task.task_id, - context.worker_id, - &task.provider_run_id.clone().unwrap_or_default(), - context.provider_name, + } + Err(error) => { + record_translation_memory_failure( + context, + format!( + "任务 {} TextUnit {} 查询失败：{}", + task.task.task_id, unit.id, error + ), + )?; + } + } + } + provider_units.push(*unit); + } + + if !provider_units.is_empty() { + context + .stats + .provider_unit_count + .fetch_add(provider_units.len(), Ordering::Relaxed); + let request = match provider_request(task, &provider_units) { + Ok(request) => request, + Err(error) => { + record_provider_failure( + context, + task, + TranslationProviderError::new( + TranslationProviderFailureClass::InvalidRequest, + error.to_string(), + ), &results, ) - .await - .map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败：{error}"))?; - context - .stats - .completed_count - .fetch_add(1, Ordering::Relaxed); - } - Err(error) => { - record_provider_failure(context, task, error).await?; + .await?; + return Ok(()); + } + }; + match context.provider.translate(request.clone()).await { + Ok(response) => { + let provider_results = + match validate_provider_response(&request, response, context.provider_name) { + Ok(results) => results, + Err(error) => { + record_provider_failure( + context, + task, + TranslationProviderError::new( + TranslationProviderFailureClass::InvalidRequest, + error.to_string(), + ), + &results, + ) + .await?; + return Ok(()); + } + }; + for result in &provider_results { + results.insert(result.unit_id.clone(), result.clone()); + if let Some(unit) = provider_units.iter().find(|unit| unit.id == result.unit_id) + { + let source_context = translation_memory_context( + &unit.destination, + unit.archive_entry.as_deref(), + unit.serialized_file.as_deref(), + unit.path_id, + unit.class_id, + unit.field_path.as_deref(), + unit.format.as_deref(), + unit.asset_name.as_deref(), + unit.text_source_kind.as_deref(), + &unit.context, + ); + let draft = TranslationMemoryDraft { + source_text: result.source_text.clone(), + source_context, + translated_text: result.translated_text.clone(), + translation_source_kind: TranslationMemorySourceKind::Provider, + official_release_id: task.task.official_release_id.clone(), + source_trace: translation_memory_trace(task, unit), + provider: Some(context.provider_name.to_string()), + provider_run_id: Some(request.provider_run_id.clone()), + observed_unix_seconds: unix_seconds_now(), + }; + if let Some(translation_memory) = context.translation_memory { + if let Err(error) = translation_memory.upsert_candidate(draft).await { + record_translation_memory_failure( + context, + format!( + "任务 {} TextUnit {} 写入候选失败：{}", + task.task.task_id, unit.id, error + ), + )?; + } + } + } + } + } + Err(error) => { + record_provider_failure(context, task, error, &results).await?; + return Ok(()); + } } } + + let ordered_results = task_units + .iter() + .filter_map(|unit| results.get(&unit.id).cloned()) + .collect::>(); + if ordered_results.len() != task_units.len() { + return Err(anyhow::anyhow!( + "任务 {} 的译文结果不完整：expected={} actual={}", + task.task.task_id, + task_units.len(), + ordered_results.len() + )); + } + context + .repository + .complete_claim( + &task.task.task_id, + context.worker_id, + &task.provider_run_id.clone().unwrap_or_default(), + context.provider_name, + &ordered_results, + ) + .await + .map_err(|error| anyhow::anyhow!("写入翻译任务完成结果失败：{error}"))?; + context + .stats + .completed_count + .fetch_add(1, Ordering::Relaxed); Ok(()) } -fn provider_request( +fn task_index_units<'a>( task: &PersistedTranslationTask, - index: &crate::official_parse::OfficialTextUnitIndex, -) -> anyhow::Result { + index: &'a crate::official_parse::OfficialTextUnitIndex, +) -> anyhow::Result> { let parse_entry_key = task .task .parse_entry_key @@ -832,7 +1053,6 @@ fn provider_request( && unit.destination == task.task.destination && unit.archive_entry == task.task.archive_entry }) - .map(|unit| provider_unit(task, unit)) .collect::>(); if units.is_empty() { return Err(anyhow::anyhow!( @@ -840,6 +1060,13 @@ fn provider_request( task.task.task_id )); } + Ok(units) +} + +fn provider_request( + task: &PersistedTranslationTask, + index_units: &[&OfficialTextUnitIndexUnit], +) -> anyhow::Result { let provider_run_id = task .provider_run_id .clone() @@ -850,7 +1077,10 @@ fn provider_request( task_id: task.task.task_id.clone(), destination: task.task.destination.clone(), archive_entry: task.task.archive_entry.clone(), - units, + units: index_units + .iter() + .map(|unit| provider_unit(task, unit)) + .collect(), }) } @@ -914,10 +1144,18 @@ fn validate_provider_response( result.unit_id )); } + if result.translated_text.trim().is_empty() { + return Err(anyhow::anyhow!( + "provider 返回空 translated_text：{}", + result.unit_id + )); + } results.push(TranslationTaskUnitResult { unit_id: result.unit_id, source_text: result.source_text, translated_text: result.translated_text, + source_kind: TranslationTaskResultSourceKind::Provider, + translation_memory_record_id: None, provider: provider_name.to_string(), provider_run_id: request.provider_run_id.clone(), translated_unix_seconds: unix_seconds_now(), @@ -933,25 +1171,81 @@ fn validate_provider_response( Ok(results) } +fn translation_memory_result( + task: &PersistedTranslationTask, + unit: &OfficialTextUnitIndexUnit, + entry: &bat_core::domain::TranslationMemoryEntry, +) -> TranslationTaskUnitResult { + TranslationTaskUnitResult { + unit_id: unit.id.clone(), + source_text: unit.source_text.clone(), + translated_text: entry.translated_text.clone(), + source_kind: TranslationTaskResultSourceKind::TranslationMemory, + translation_memory_record_id: Some(entry.record_id.clone()), + provider: "translation_memory".to_string(), + provider_run_id: task.provider_run_id.clone().unwrap_or_default(), + translated_unix_seconds: unix_seconds_now(), + } +} + +fn translation_memory_trace( + task: &PersistedTranslationTask, + unit: &OfficialTextUnitIndexUnit, +) -> TranslationMemorySourceTrace { + TranslationMemorySourceTrace { + official_release_id: task.task.official_release_id.clone(), + unit_id: Some(unit.id.clone()), + task_id: Some(task.task.task_id.clone()), + destination: Some(unit.destination.clone()), + archive_entry: unit.archive_entry.clone(), + serialized_file: unit.serialized_file.clone(), + path_id: unit.path_id, + class_id: unit.class_id, + field_path: unit.field_path.clone(), + format: unit.format.clone(), + asset_name: unit.asset_name.clone(), + text_source_kind: unit.text_source_kind.clone(), + source_url: Some(unit.source_url.clone()), + } +} + +fn record_translation_memory_failure( + context: &WorkerTaskContext<'_>, + message: String, +) -> anyhow::Result<()> { + context + .stats + .translation_memory_failures + .lock() + .map_err(|_| anyhow::anyhow!("写入 Translation Memory 诊断时 mutex poisoned"))? + .push(message); + Ok(()) +} + async fn record_provider_failure( context: &WorkerTaskContext<'_>, task: &PersistedTranslationTask, error: TranslationProviderError, + partial_results: &BTreeMap, ) -> anyhow::Result<()> { let retryable = error.retryable && task.attempt_count < context.max_attempts; let next_attempt = retryable.then(|| unix_seconds_now().saturating_add(context.retry_backoff.as_secs())); + let partial_results = partial_results.values().cloned().collect::>(); context .repository - .fail_claim(TranslationTaskFailure { - task_id: task.task.task_id.clone(), - worker_id: context.worker_id.to_string(), - provider_run_id: task.provider_run_id.clone().unwrap_or_default(), - failure_class: error.class.as_str().to_string(), - failure_reason: error.message.clone(), - retryable, - next_attempt_unix_seconds: next_attempt, - }) + .fail_claim_with_results( + TranslationTaskFailure { + task_id: task.task.task_id.clone(), + worker_id: context.worker_id.to_string(), + provider_run_id: task.provider_run_id.clone().unwrap_or_default(), + failure_class: error.class.as_str().to_string(), + failure_reason: error.message.clone(), + retryable, + next_attempt_unix_seconds: next_attempt, + }, + &partial_results, + ) .await .map_err(|failure| anyhow::anyhow!("写入翻译任务失败状态失败：{failure}"))?; context.stats.failed_count.fetch_add(1, Ordering::Relaxed); @@ -1161,6 +1455,277 @@ mod tests { assert_eq!(task.translation_results[0].translated_text, "translated-0"); } + #[tokio::test] + async fn worker_honors_cancellation_before_claiming_tasks() { + let (temp, queue) = fixture_root(); + crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap(); + crate::official_parse::write_textunit_index_at(temp.path(), &index(temp.path())).unwrap(); + let config = TranslationWorkerConfig { + concurrency: 2, + ..TranslationWorkerConfig::default() + }; + + let result = + run_translation_worker_at_with_cancellation(temp.path(), &config, Arc::new(|| true)) + .await; + + assert!(result.unwrap_err().to_string().contains("已取消")); + let repository = SqliteTranslationTaskRepository::open( + SqliteTranslationTaskRepository::repository_path(temp.path()), + ) + .await + .unwrap(); + let task = repository.find(&queue.tasks[0].task_id).await.unwrap(); + assert_eq!( + task.task_status, + crate::translation_tasks::TranslationTaskStatus::Queued + ); + assert_eq!(task.attempt_count, 0); + } + + #[tokio::test] + async fn worker_reuses_trusted_tm_for_part_of_a_task_and_calls_provider_for_the_rest() { + let (temp, queue) = fixture_root(); + let textunit_index = index(temp.path()); + crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap(); + crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap(); + + let translation_memory_path = temp.path().join("translation-memory.sqlite"); + let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path) + .await + .unwrap(); + let unit = &textunit_index.units[0]; + let source_context = translation_memory_context( + &unit.destination, + unit.archive_entry.as_deref(), + unit.serialized_file.as_deref(), + unit.path_id, + unit.class_id, + unit.field_path.as_deref(), + unit.format.as_deref(), + unit.asset_name.as_deref(), + unit.text_source_kind.as_deref(), + &unit.context, + ); + let candidate = translation_memory + .upsert_candidate(TranslationMemoryDraft { + source_text: unit.source_text.clone(), + source_context, + translated_text: "trusted-from-tm".to_string(), + translation_source_kind: TranslationMemorySourceKind::Manual, + official_release_id: queue.official_release_id.clone(), + source_trace: TranslationMemorySourceTrace { + official_release_id: queue.official_release_id.clone(), + unit_id: Some(unit.id.clone()), + task_id: Some(queue.tasks[0].task_id.clone()), + destination: Some(unit.destination.clone()), + archive_entry: unit.archive_entry.clone(), + serialized_file: unit.serialized_file.clone(), + path_id: unit.path_id, + class_id: unit.class_id, + field_path: unit.field_path.clone(), + format: unit.format.clone(), + asset_name: unit.asset_name.clone(), + text_source_kind: unit.text_source_kind.clone(), + source_url: Some(unit.source_url.clone()), + }, + provider: None, + provider_run_id: None, + observed_unix_seconds: 1, + }) + .await + .unwrap(); + let trusted = translation_memory + .confirm( + &candidate.record_id, + "test-reviewer", + Some("accepted".to_string()), + ) + .await + .unwrap(); + assert_eq!( + trusted.trust_status, + bat_core::domain::TranslationMemoryTrustStatus::Trusted + ); + + let fixture = temp.path().join("partial-mock.json"); + std::fs::write( + &fixture, + serde_json::to_vec(&serde_json::json!({ + "schema_version": 1, + "translations": { + "direct:bundle#unit:1": "translated-by-provider" + } + })) + .unwrap(), + ) + .unwrap(); + let config = TranslationWorkerConfig { + fixture_path: Some(fixture), + translation_memory_path: Some(translation_memory_path), + concurrency: 1, + retry_backoff: Duration::ZERO, + ..TranslationWorkerConfig::default() + }; + let report = run_translation_worker_at(temp.path(), &config) + .await + .unwrap(); + assert_eq!(report.translation_memory_hit_count, 1); + assert_eq!(report.provider_unit_count, 1); + assert_eq!(report.completed_count, 1); + + let repository = SqliteTranslationTaskRepository::open( + SqliteTranslationTaskRepository::repository_path(temp.path()), + ) + .await + .unwrap(); + let task = repository.find(&queue.tasks[0].task_id).await.unwrap(); + assert_eq!(task.translation_results.len(), 2); + assert_eq!( + task.translation_results[0].translated_text, + "trusted-from-tm" + ); + assert_eq!( + task.translation_results[0].source_kind, + TranslationTaskResultSourceKind::TranslationMemory + ); + assert_eq!( + task.translation_results[0] + .translation_memory_record_id + .as_deref(), + Some(trusted.record_id.as_str()) + ); + assert_eq!( + task.translation_results[1].translated_text, + "translated-by-provider" + ); + assert_eq!( + task.translation_results[1].source_kind, + TranslationTaskResultSourceKind::Provider + ); + } + + #[tokio::test] + async fn worker_retains_tm_hits_when_provider_fails_for_remaining_units() { + let (temp, queue) = fixture_root(); + let textunit_index = index(temp.path()); + crate::official_textunit_queue::write_textunit_task_queue_at(temp.path(), &queue).unwrap(); + crate::official_parse::write_textunit_index_at(temp.path(), &textunit_index).unwrap(); + + let translation_memory_path = temp.path().join("translation-memory.sqlite"); + let translation_memory = SqliteTranslationMemoryRepository::new(&translation_memory_path) + .await + .unwrap(); + let unit = &textunit_index.units[0]; + let source_context = translation_memory_context( + &unit.destination, + unit.archive_entry.as_deref(), + unit.serialized_file.as_deref(), + unit.path_id, + unit.class_id, + unit.field_path.as_deref(), + unit.format.as_deref(), + unit.asset_name.as_deref(), + unit.text_source_kind.as_deref(), + &unit.context, + ); + let candidate = translation_memory + .upsert_candidate(TranslationMemoryDraft { + source_text: unit.source_text.clone(), + source_context, + translated_text: "trusted-from-tm".to_string(), + translation_source_kind: TranslationMemorySourceKind::Manual, + official_release_id: queue.official_release_id.clone(), + source_trace: TranslationMemorySourceTrace { + official_release_id: queue.official_release_id.clone(), + unit_id: Some(unit.id.clone()), + task_id: Some(queue.tasks[0].task_id.clone()), + destination: Some(unit.destination.clone()), + archive_entry: unit.archive_entry.clone(), + serialized_file: unit.serialized_file.clone(), + path_id: unit.path_id, + class_id: unit.class_id, + field_path: unit.field_path.clone(), + format: unit.format.clone(), + asset_name: unit.asset_name.clone(), + text_source_kind: unit.text_source_kind.clone(), + source_url: Some(unit.source_url.clone()), + }, + provider: None, + provider_run_id: None, + observed_unix_seconds: 1, + }) + .await + .unwrap(); + let trusted = translation_memory + .confirm( + &candidate.record_id, + "test-reviewer", + Some("accepted".to_string()), + ) + .await + .unwrap(); + + let fixture = temp.path().join("provider-failure.json"); + std::fs::write( + &fixture, + serde_json::to_vec(&serde_json::json!({ + "schema_version": 1, + "failures": { + "textunit/release-1/bundle": { + "class": "rate_limited", + "message": "fixture throttled", + "retryable": false + } + } + })) + .unwrap(), + ) + .unwrap(); + let config = TranslationWorkerConfig { + fixture_path: Some(fixture), + translation_memory_path: Some(translation_memory_path), + concurrency: 1, + max_attempts: 1, + retry_backoff: Duration::ZERO, + ..TranslationWorkerConfig::default() + }; + let report = run_translation_worker_at(temp.path(), &config) + .await + .unwrap(); + assert_eq!(report.translation_memory_hit_count, 1); + assert_eq!(report.provider_unit_count, 1); + assert_eq!(report.failed_count, 1); + + let repository = SqliteTranslationTaskRepository::open( + SqliteTranslationTaskRepository::repository_path(temp.path()), + ) + .await + .unwrap(); + let task = repository.find(&queue.tasks[0].task_id).await.unwrap(); + assert_eq!( + task.task_status, + crate::translation_tasks::TranslationTaskStatus::Failed + ); + assert_eq!(task.translation_results.len(), 1); + assert_eq!( + task.translation_results[0] + .translation_memory_record_id + .as_deref(), + Some(trusted.record_id.as_str()) + ); + + let handoff = crate::translation_tasks::build_translation_handoff(&queue, &[task]); + assert_eq!( + handoff.units[0].status, + crate::translation_tasks::TranslationUnitStatus::Translated + ); + assert_eq!( + handoff.job.status, + crate::translation_tasks::TranslationJobStatus::Failed + ); + } + #[tokio::test] async fn mock_worker_retries_retryable_failures_and_keeps_diagnostic() { let (temp, queue) = fixture_root(); diff --git a/infrastructure/src/translation_workflow.rs b/infrastructure/src/translation_workflow.rs index 1b4fff5..8eb6811 100644 --- a/infrastructure/src/translation_workflow.rs +++ b/infrastructure/src/translation_workflow.rs @@ -9,7 +9,7 @@ use crate::path_security::{ use crate::{ LocalizedPatchInput, LocalizedPatchOperationMetadata, LocalizedStringFieldPatch, LocalizedTextAssetPatch, PersistedTranslationTask, SqliteTranslationTaskRepository, - TranslationTaskStatus, TranslationTaskUnitResult, + TranslationTaskResultSourceKind, TranslationTaskStatus, TranslationTaskUnitResult, }; use bat_assetbundle::{ patch_unityfs_field, patch_unityfs_string_field, patch_unityfs_text_asset, FieldPatch, @@ -72,6 +72,12 @@ pub struct TranslationWorkbenchEntry { /// Provider run that produced this translation, when imported from worker output. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_run_id: Option, + /// Source of the worker result (`provider`, `manual`, or `translation_memory`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_source_kind: Option, + /// Trusted Translation Memory record used for this translation, when applicable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub translation_memory_record_id: Option, /// Worker completion time for provider-produced text. #[serde(default, skip_serializing_if = "Option::is_none")] pub translated_unix_seconds: Option, @@ -616,6 +622,8 @@ fn localized_patch_metadata(entry: &TranslationWorkbenchEntry) -> LocalizedPatch .to_string(), translation_provider: entry.translation_provider.clone(), provider_run_id: entry.provider_run_id.clone(), + translation_source_kind: entry.translation_source_kind.clone(), + translation_memory_record_id: entry.translation_memory_record_id.clone(), review_status: entry .review_status .clone() @@ -887,18 +895,30 @@ fn workbench_entry_from_worker_result( ) -> TranslationWorkbenchEntry { let mut entry = TranslationWorkbenchEntry::from_index(unit); entry.translated_text = Some(result.translated_text.clone()); - entry.translation_provider = task - .provider - .clone() - .or_else(|| Some(result.provider.clone())) - .filter(|provider| !provider.trim().is_empty()); + entry.translation_provider = match result.source_kind { + TranslationTaskResultSourceKind::TranslationMemory => None, + TranslationTaskResultSourceKind::Provider | TranslationTaskResultSourceKind::Manual => task + .provider + .clone() + .or_else(|| Some(result.provider.clone())) + .filter(|provider| !provider.trim().is_empty()), + }; entry.provider_run_id = task .provider_run_id .clone() .or_else(|| Some(result.provider_run_id.clone())) .filter(|provider_run_id| !provider_run_id.trim().is_empty()); + entry.translation_source_kind = Some(result.source_kind.as_str().to_string()); + entry.translation_memory_record_id = result.translation_memory_record_id.clone(); entry.translated_unix_seconds = Some(result.translated_unix_seconds); - entry.review_status = Some("provider_completed".to_string()); + entry.review_status = Some( + match result.source_kind { + TranslationTaskResultSourceKind::Provider => "provider_completed", + TranslationTaskResultSourceKind::Manual => "manual_submitted", + TranslationTaskResultSourceKind::TranslationMemory => "translation_memory_reused", + } + .to_string(), + ); entry } @@ -938,6 +958,8 @@ impl TranslationWorkbenchEntry { translated_text: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, translated_unix_seconds: None, review_status: None, format: unit.format.clone(), @@ -975,6 +997,8 @@ mod tests { translated_text: None, translation_provider: None, provider_run_id: None, + translation_source_kind: None, + translation_memory_record_id: None, translated_unix_seconds: None, review_status: None, format: Some("plain".to_string()),