mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 11:56:23 +08:00
feat(translation): add Rust Translation Memory and config migration
This commit is contained in:
@@ -0,0 +1,859 @@
|
||||
//! 跨 official release 的 Translation Memory SQLite 仓储。
|
||||
|
||||
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
|
||||
use async_trait::async_trait;
|
||||
use bat_core::domain::{
|
||||
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||||
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||||
TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||||
};
|
||||
use bat_core::repositories::TranslationMemoryRepository;
|
||||
use bat_core::{Error, Result};
|
||||
use serde::de::DeserializeOwned;
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
|
||||
use sqlx::{Row, SqlitePool};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// TM SQLite schema 版本。
|
||||
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1;
|
||||
/// TM schema migration component。
|
||||
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
|
||||
/// 默认 TM 数据库文件名。
|
||||
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
|
||||
|
||||
/// SQLite-backed Translation Memory 仓储。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SqliteTranslationMemoryRepository {
|
||||
pub(crate) pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteTranslationMemoryRepository {
|
||||
/// 创建或打开 TM 数据库并执行迁移。
|
||||
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
|
||||
Self::open_with(path.as_ref(), true).await
|
||||
}
|
||||
|
||||
/// 只打开已有 TM 数据库，不创建新文件。
|
||||
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||
Self::open_with(path.as_ref(), false).await
|
||||
}
|
||||
|
||||
/// 根据 active release 根目录计算默认的跨 release TM 路径。
|
||||
///
|
||||
/// 正式 release 根目录形如 `<output>/versions/<id>`，因此默认结果为
|
||||
/// `<output>/translation-memory.sqlite`，不会写入已发布版本目录。
|
||||
pub fn repository_path(resource_root: &Path) -> PathBuf {
|
||||
if resource_root
|
||||
.parent()
|
||||
.and_then(Path::file_name)
|
||||
.is_some_and(|name| name == "versions")
|
||||
{
|
||||
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
|
||||
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
|
||||
}
|
||||
}
|
||||
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
|
||||
}
|
||||
|
||||
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
|
||||
let absolute = bat_infrastructure_absolute(path)?;
|
||||
let parent = absolute.parent().ok_or_else(|| {
|
||||
Error::InvalidArgument(format!("TM 数据库缺少父目录：{}", absolute.display()))
|
||||
})?;
|
||||
ensure_safe_tm_parent(parent)?;
|
||||
if create_if_missing {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
ensure_safe_tm_parent(parent)?;
|
||||
}
|
||||
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"TM 数据库不能是 symlink：{}",
|
||||
absolute.display()
|
||||
)));
|
||||
}
|
||||
if !metadata.is_file() {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"TM 数据库不是普通文件：{}",
|
||||
absolute.display()
|
||||
)));
|
||||
}
|
||||
} else if !create_if_missing {
|
||||
return Err(Error::NotFound(absolute.display().to_string()));
|
||||
}
|
||||
|
||||
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
|
||||
.map_err(|error| Error::Other(error.into()))?
|
||||
.create_if_missing(create_if_missing)
|
||||
.journal_mode(SqliteJournalMode::Wal)
|
||||
.busy_timeout(Duration::from_secs(30));
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
|
||||
.map_err(Error::InvalidArgument)?;
|
||||
let repository = Self { pool };
|
||||
repository.init_schema().await?;
|
||||
Ok(repository)
|
||||
}
|
||||
|
||||
async fn init_schema(&self) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
component TEXT PRIMARY KEY NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version >= 1)
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS translation_memory (
|
||||
record_id TEXT PRIMARY KEY NOT NULL,
|
||||
source_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
normalized_source_text TEXT NOT NULL,
|
||||
source_context_json TEXT NOT NULL,
|
||||
source_context_hash TEXT NOT NULL,
|
||||
translated_text TEXT NOT NULL,
|
||||
translation_source_kind TEXT NOT NULL,
|
||||
trust_status TEXT NOT NULL,
|
||||
official_release_id TEXT NOT NULL,
|
||||
source_trace_json TEXT NOT NULL,
|
||||
provider TEXT,
|
||||
provider_run_id TEXT,
|
||||
created_unix_seconds INTEGER NOT NULL,
|
||||
updated_unix_seconds INTEGER NOT NULL,
|
||||
trusted_unix_seconds INTEGER,
|
||||
trusted_by TEXT,
|
||||
trusted_reason TEXT,
|
||||
supersedes_record_id TEXT,
|
||||
superseded_by_record_id TEXT,
|
||||
CHECK (length(source_text) > 0),
|
||||
CHECK (length(source_hash) > 0),
|
||||
CHECK (length(source_context_hash) > 0),
|
||||
CHECK (length(official_release_id) > 0),
|
||||
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
|
||||
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
|
||||
ON translation_memory(source_hash)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
|
||||
ON translation_memory(normalized_source_text)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
|
||||
ON translation_memory(source_hash, source_context_hash)",
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
|
||||
let current: Option<i64> =
|
||||
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"不支持的 Translation Memory schema 版本：{}",
|
||||
current.unwrap_or_default()
|
||||
)));
|
||||
}
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO schema_migrations(component, version)
|
||||
VALUES (?1, ?2)
|
||||
ON CONFLICT(component) DO UPDATE SET version = excluded.version
|
||||
"#,
|
||||
)
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
|
||||
let row = sqlx::query(
|
||||
r#"
|
||||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||
FROM translation_memory
|
||||
WHERE record_id = ?1
|
||||
"#,
|
||||
)
|
||||
.bind(record_id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
row.map(row_to_entry).transpose()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
|
||||
async fn upsert_candidate(
|
||||
&self,
|
||||
draft: TranslationMemoryDraft,
|
||||
) -> Result<TranslationMemoryEntry> {
|
||||
validate_draft(&draft)?;
|
||||
let entry = entry_from_draft(draft)?;
|
||||
if let Some(existing) = self.find_optional(&entry.record_id).await? {
|
||||
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||
return Ok(existing);
|
||||
}
|
||||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_memory
|
||||
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
|
||||
updated_unix_seconds = ?5
|
||||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(&entry.record_id)
|
||||
.bind(source_trace_json)
|
||||
.bind(&entry.provider)
|
||||
.bind(&entry.provider_run_id)
|
||||
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
return self.find(&entry.record_id).await;
|
||||
}
|
||||
|
||||
let source_context_json = serde_json::to_string(&entry.source_context)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO translation_memory (
|
||||
record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds,
|
||||
trusted_unix_seconds, trusted_by, trusted_reason,
|
||||
supersedes_record_id, superseded_by_record_id
|
||||
)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
|
||||
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
|
||||
ON CONFLICT(record_id) DO UPDATE SET
|
||||
source_trace_json = excluded.source_trace_json,
|
||||
provider = excluded.provider,
|
||||
provider_run_id = excluded.provider_run_id,
|
||||
updated_unix_seconds = excluded.updated_unix_seconds
|
||||
WHERE translation_memory.trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(&entry.record_id)
|
||||
.bind(&entry.source_text)
|
||||
.bind(&entry.source_hash)
|
||||
.bind(&entry.normalized_source_text)
|
||||
.bind(source_context_json)
|
||||
.bind(&entry.source_context_hash)
|
||||
.bind(&entry.translated_text)
|
||||
.bind(entry.translation_source_kind.as_str())
|
||||
.bind(entry.trust_status.as_str())
|
||||
.bind(&entry.official_release_id)
|
||||
.bind(source_trace_json)
|
||||
.bind(&entry.provider)
|
||||
.bind(&entry.provider_run_id)
|
||||
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
|
||||
.bind(
|
||||
entry
|
||||
.trusted_unix_seconds
|
||||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
|
||||
)
|
||||
.bind(&entry.trusted_by)
|
||||
.bind(&entry.trusted_reason)
|
||||
.bind(&entry.supersedes_record_id)
|
||||
.bind(&entry.superseded_by_record_id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if result.rows_affected() == 0 {
|
||||
return self.find(&entry.record_id).await;
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
async fn find_matches(
|
||||
&self,
|
||||
source_text: &str,
|
||||
source_context: &TranslationMemoryContext,
|
||||
limit: usize,
|
||||
) -> Result<Vec<TranslationMemoryMatch>> {
|
||||
if source_text.is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory 查询 source_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if limit == 0 {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory 查询 limit 必须大于 0".to_string(),
|
||||
));
|
||||
}
|
||||
let source_hash = hash_text(source_text);
|
||||
let normalized_source_text = normalize_source_text(source_text);
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||||
source_context_json, source_context_hash, translated_text,
|
||||
translation_source_kind, trust_status, official_release_id,
|
||||
source_trace_json, provider, provider_run_id,
|
||||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||||
FROM translation_memory
|
||||
WHERE source_hash = ?1 OR normalized_source_text = ?2
|
||||
ORDER BY updated_unix_seconds DESC, record_id ASC
|
||||
"#,
|
||||
)
|
||||
.bind(source_hash)
|
||||
.bind(&normalized_source_text)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
let mut matches = rows
|
||||
.into_iter()
|
||||
.map(row_to_entry)
|
||||
.collect::<Result<Vec<_>>>()?
|
||||
.into_iter()
|
||||
.filter_map(|entry| {
|
||||
let raw_exact = entry.source_text == source_text;
|
||||
let normalized_exact = entry.normalized_source_text == normalized_source_text;
|
||||
if !raw_exact && !normalized_exact {
|
||||
return None;
|
||||
}
|
||||
let same_context = !source_context.is_empty()
|
||||
&& !entry.source_context.is_empty()
|
||||
&& entry.source_context == *source_context;
|
||||
let strong = raw_exact
|
||||
&& same_context
|
||||
&& entry.trust_status == TranslationMemoryTrustStatus::Trusted;
|
||||
let match_kind = if strong {
|
||||
TranslationMemoryMatchKind::StrongExact
|
||||
} else if raw_exact {
|
||||
TranslationMemoryMatchKind::CandidateExact
|
||||
} else {
|
||||
TranslationMemoryMatchKind::SourceOnly
|
||||
};
|
||||
Some(TranslationMemoryMatch {
|
||||
can_auto_reuse: strong,
|
||||
entry,
|
||||
match_kind,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
matches.sort_by(|left, right| {
|
||||
match_rank(left)
|
||||
.cmp(&match_rank(right))
|
||||
.then_with(|| {
|
||||
right
|
||||
.entry
|
||||
.updated_unix_seconds
|
||||
.cmp(&left.entry.updated_unix_seconds)
|
||||
})
|
||||
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
|
||||
});
|
||||
matches.truncate(limit);
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
async fn confirm(
|
||||
&self,
|
||||
record_id: &str,
|
||||
reviewer: &str,
|
||||
reason: Option<String>,
|
||||
) -> Result<TranslationMemoryEntry> {
|
||||
if reviewer.trim().is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory reviewer 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let current = self.find(record_id).await?;
|
||||
if current.trust_status == TranslationMemoryTrustStatus::Trusted {
|
||||
return Ok(current);
|
||||
}
|
||||
if current.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||||
return Err(Error::InvalidArgument(format!(
|
||||
"Translation Memory 记录 {} 当前状态为 {}，不能确认",
|
||||
record_id,
|
||||
current.trust_status.as_str()
|
||||
)));
|
||||
}
|
||||
let now = unix_seconds_now();
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_memory
|
||||
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
|
||||
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
|
||||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||||
"#,
|
||||
)
|
||||
.bind(record_id)
|
||||
.bind(i64::try_from(now).unwrap_or(i64::MAX))
|
||||
.bind(reviewer.trim())
|
||||
.bind(reason.filter(|value| !value.trim().is_empty()))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
self.find(record_id).await
|
||||
}
|
||||
|
||||
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
|
||||
self.find_optional(record_id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::NotFound(record_id.to_string()))
|
||||
}
|
||||
|
||||
async fn summary(&self) -> Result<TranslationMemorySummary> {
|
||||
let row = sqlx::query(
|
||||
r#"
|
||||
SELECT COUNT(*) AS record_count,
|
||||
SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count,
|
||||
SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count,
|
||||
SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count,
|
||||
SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count
|
||||
FROM translation_memory
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
Ok(TranslationMemorySummary {
|
||||
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||||
record_count: row.try_get::<i64, _>("record_count").map_err(db_error)? as u64,
|
||||
trusted_count: row.try_get::<i64, _>("trusted_count").map_err(db_error)? as u64,
|
||||
candidate_count: row.try_get::<i64, _>("candidate_count").map_err(db_error)? as u64,
|
||||
superseded_count: row
|
||||
.try_get::<i64, _>("superseded_count")
|
||||
.map_err(db_error)? as u64,
|
||||
rejected_count: row.try_get::<i64, _>("rejected_count").map_err(db_error)? as u64,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 由 active official release 根目录计算默认 TM 数据库路径。
|
||||
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
|
||||
SqliteTranslationMemoryRepository::repository_path(resource_root)
|
||||
}
|
||||
|
||||
/// 从 TextUnit 定位字段构建 TM 上下文。
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn translation_memory_context(
|
||||
destination: &str,
|
||||
archive_entry: Option<&str>,
|
||||
serialized_file: Option<&str>,
|
||||
path_id: Option<i64>,
|
||||
class_id: Option<i32>,
|
||||
field_path: Option<&str>,
|
||||
format: Option<&str>,
|
||||
asset_name: Option<&str>,
|
||||
text_source_kind: Option<&str>,
|
||||
parser_context: &TranslationMemoryContext,
|
||||
) -> TranslationMemoryContext {
|
||||
let mut context = TranslationMemoryContext::new();
|
||||
context.insert("destination".to_string(), destination.to_string());
|
||||
insert_optional(&mut context, "archive_entry", archive_entry);
|
||||
insert_optional(&mut context, "serialized_file", serialized_file);
|
||||
if let Some(value) = path_id {
|
||||
context.insert("path_id".to_string(), value.to_string());
|
||||
}
|
||||
if let Some(value) = class_id {
|
||||
context.insert("class_id".to_string(), value.to_string());
|
||||
}
|
||||
insert_optional(&mut context, "field_path", field_path);
|
||||
insert_optional(&mut context, "format", format);
|
||||
insert_optional(&mut context, "asset_name", asset_name);
|
||||
insert_optional(&mut context, "text_source_kind", text_source_kind);
|
||||
for (key, value) in parser_context {
|
||||
context.insert(format!("context.{key}"), value.clone());
|
||||
}
|
||||
context
|
||||
}
|
||||
|
||||
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
|
||||
if let Some(value) = value.filter(|value| !value.is_empty()) {
|
||||
context.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
|
||||
let source_hash = hash_text(&draft.source_text);
|
||||
let normalized_source_text = normalize_source_text(&draft.source_text);
|
||||
let source_context_hash = hash_context(&draft.source_context)?;
|
||||
let source_kind = draft.translation_source_kind;
|
||||
let record_id = record_id(
|
||||
&source_hash,
|
||||
&source_context_hash,
|
||||
&draft.translated_text,
|
||||
&draft.official_release_id,
|
||||
&source_kind,
|
||||
);
|
||||
let observed = draft.observed_unix_seconds;
|
||||
Ok(TranslationMemoryEntry {
|
||||
record_id,
|
||||
source_text: draft.source_text,
|
||||
source_hash,
|
||||
normalized_source_text,
|
||||
source_context: draft.source_context,
|
||||
source_context_hash,
|
||||
translated_text: draft.translated_text,
|
||||
translation_source_kind: source_kind,
|
||||
trust_status: TranslationMemoryTrustStatus::Candidate,
|
||||
official_release_id: draft.official_release_id,
|
||||
source_trace: draft.source_trace,
|
||||
provider: draft.provider,
|
||||
provider_run_id: draft.provider_run_id,
|
||||
created_unix_seconds: observed,
|
||||
updated_unix_seconds: observed,
|
||||
trusted_unix_seconds: None,
|
||||
trusted_by: None,
|
||||
trusted_reason: None,
|
||||
supersedes_record_id: None,
|
||||
superseded_by_record_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
|
||||
if draft.source_text.is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory source_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.translated_text.trim().is_empty() {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory translated_text 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.official_release_id.trim().is_empty()
|
||||
|| draft.source_trace.official_release_id.trim().is_empty()
|
||||
{
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory official_release_id 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if draft.official_release_id != draft.source_trace.official_release_id {
|
||||
return Err(Error::InvalidArgument(
|
||||
"Translation Memory draft 的 release provenance 不一致".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
|
||||
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
|
||||
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
|
||||
Ok(TranslationMemoryEntry {
|
||||
record_id: row.try_get("record_id").map_err(db_error)?,
|
||||
source_text: row.try_get("source_text").map_err(db_error)?,
|
||||
source_hash: row.try_get("source_hash").map_err(db_error)?,
|
||||
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
|
||||
source_context,
|
||||
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
|
||||
translated_text: row.try_get("translated_text").map_err(db_error)?,
|
||||
translation_source_kind: parse_source_kind(
|
||||
row.try_get::<String, _>("translation_source_kind")
|
||||
.map_err(db_error)?
|
||||
.as_str(),
|
||||
)?,
|
||||
trust_status: parse_trust_status(
|
||||
row.try_get::<String, _>("trust_status")
|
||||
.map_err(db_error)?
|
||||
.as_str(),
|
||||
)?,
|
||||
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
|
||||
source_trace,
|
||||
provider: row.try_get("provider").map_err(db_error)?,
|
||||
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
|
||||
created_unix_seconds: i64_to_u64(
|
||||
row.try_get("created_unix_seconds").map_err(db_error)?,
|
||||
"created",
|
||||
)?,
|
||||
updated_unix_seconds: i64_to_u64(
|
||||
row.try_get("updated_unix_seconds").map_err(db_error)?,
|
||||
"updated",
|
||||
)?,
|
||||
trusted_unix_seconds: optional_i64_to_u64(
|
||||
row.try_get("trusted_unix_seconds").map_err(db_error)?,
|
||||
"trusted",
|
||||
)?,
|
||||
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
|
||||
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
|
||||
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
|
||||
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
|
||||
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
|
||||
}
|
||||
|
||||
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
|
||||
match value {
|
||||
"provider" => Ok(TranslationMemorySourceKind::Provider),
|
||||
"manual" => Ok(TranslationMemorySourceKind::Manual),
|
||||
"imported" => Ok(TranslationMemorySourceKind::Imported),
|
||||
_ => Err(Error::Serialization(format!(
|
||||
"未知 Translation Memory source kind：{value}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
|
||||
match value {
|
||||
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
|
||||
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
|
||||
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
|
||||
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
|
||||
_ => Err(Error::Serialization(format!(
|
||||
"未知 Translation Memory trust status：{value}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
|
||||
match value.match_kind {
|
||||
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
|
||||
TranslationMemoryMatchKind::CandidateExact => 1,
|
||||
TranslationMemoryMatchKind::SourceOnly => 2,
|
||||
TranslationMemoryMatchKind::StrongExact => 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_text(value: &str) -> String {
|
||||
blake3::hash(value.as_bytes()).to_hex().to_string()
|
||||
}
|
||||
|
||||
fn normalize_source_text(value: &str) -> String {
|
||||
value.replace("\r\n", "\n").replace('\r', "\n")
|
||||
}
|
||||
|
||||
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
|
||||
let bytes =
|
||||
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
Ok(blake3::hash(&bytes).to_hex().to_string())
|
||||
}
|
||||
|
||||
fn record_id(
|
||||
source_hash: &str,
|
||||
context_hash: &str,
|
||||
translated_text: &str,
|
||||
official_release_id: &str,
|
||||
source_kind: &TranslationMemorySourceKind,
|
||||
) -> String {
|
||||
let mut key = Vec::new();
|
||||
for value in [
|
||||
source_hash,
|
||||
context_hash,
|
||||
translated_text,
|
||||
official_release_id,
|
||||
source_kind.as_str(),
|
||||
] {
|
||||
key.extend_from_slice(value.as_bytes());
|
||||
key.push(0);
|
||||
}
|
||||
format!("tm-{}", blake3::hash(&key).to_hex())
|
||||
}
|
||||
|
||||
fn unix_seconds_now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
|
||||
u64::try_from(value)
|
||||
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
|
||||
}
|
||||
|
||||
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
|
||||
value.map(|value| i64_to_u64(value, label)).transpose()
|
||||
}
|
||||
|
||||
fn db_error(error: sqlx::Error) -> Error {
|
||||
Error::Other(error.into())
|
||||
}
|
||||
|
||||
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
|
||||
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
|
||||
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
|
||||
.map_err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bat_core::domain::TranslationMemorySourceTrace;
|
||||
|
||||
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
|
||||
let source_trace = TranslationMemorySourceTrace {
|
||||
official_release_id: release.to_string(),
|
||||
unit_id: Some(format!("{release}-unit")),
|
||||
task_id: Some(format!("{release}-task")),
|
||||
destination: Some("Bundles/story.bundle".to_string()),
|
||||
archive_entry: None,
|
||||
serialized_file: Some("CAB-story".to_string()),
|
||||
path_id: Some(1),
|
||||
class_id: Some(49),
|
||||
field_path: Some("m_Text".to_string()),
|
||||
format: Some("plain".to_string()),
|
||||
asset_name: Some("Story".to_string()),
|
||||
text_source_kind: Some("text_asset".to_string()),
|
||||
source_url: Some("https://example.invalid/story".to_string()),
|
||||
};
|
||||
TranslationMemoryDraft {
|
||||
source_text: source.to_string(),
|
||||
source_context: translation_memory_context(
|
||||
"Bundles/story.bundle",
|
||||
None,
|
||||
Some("CAB-story"),
|
||||
Some(1),
|
||||
Some(49),
|
||||
Some("m_Text"),
|
||||
Some("plain"),
|
||||
Some("Story"),
|
||||
Some("text_asset"),
|
||||
&TranslationMemoryContext::new(),
|
||||
),
|
||||
translated_text: translated.to_string(),
|
||||
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||||
official_release_id: release.to_string(),
|
||||
source_trace,
|
||||
provider: Some("mock".to_string()),
|
||||
provider_run_id: Some(format!("mock:{release}")),
|
||||
observed_unix_seconds: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(
|
||||
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let entry = repository
|
||||
.upsert_candidate(draft("release-1", "Hello", "你好"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
|
||||
let trusted = repository
|
||||
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
|
||||
|
||||
let query = draft("release-2", "Hello", "ignored");
|
||||
let matches = repository
|
||||
.find_matches("Hello", &query.source_context, 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(matches.len(), 1);
|
||||
assert_eq!(
|
||||
matches[0].match_kind,
|
||||
TranslationMemoryMatchKind::StrongExact
|
||||
);
|
||||
assert!(matches[0].can_auto_reuse);
|
||||
assert_eq!(matches[0].entry.translated_text, "你好");
|
||||
assert_eq!(matches[0].entry.official_release_id, "release-1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_future_schema_version() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let path = temp.path().join("tm.sqlite");
|
||||
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
|
||||
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
|
||||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
|
||||
.execute(&repository.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
repository.pool.close().await;
|
||||
|
||||
let error = SqliteTranslationMemoryRepository::new(&path)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error
|
||||
.to_string()
|
||||
.contains("不支持的 Translation Memory schema"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = draft("release-1", "Hello", "你好");
|
||||
repository.upsert_candidate(first.clone()).await.unwrap();
|
||||
repository.upsert_candidate(first).await.unwrap();
|
||||
assert_eq!(repository.summary().await.unwrap().record_count, 1);
|
||||
|
||||
let mut different = draft("release-2", "Hello", "你好");
|
||||
different
|
||||
.source_context
|
||||
.insert("field_path".to_string(), "m_Other".to_string());
|
||||
let matches = repository
|
||||
.find_matches("Hello", &different.source_context, 10)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
matches[0].match_kind,
|
||||
TranslationMemoryMatchKind::CandidateExact
|
||||
);
|
||||
assert!(!matches[0].can_auto_reuse);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_empty_translation_candidates() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let error = repository
|
||||
.upsert_candidate(draft("release-1", "Hello", " \n"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("translated_text"));
|
||||
assert_eq!(repository.summary().await.unwrap().record_count, 0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user