mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 08:14:55 +08:00
860 lines
33 KiB
Rust
860 lines
33 KiB
Rust
//! 跨 official release 的 Translation Memory SQLite 仓储。
|
||
|
||
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
|
||
use async_trait::async_trait;
|
||
use bat_core::domain::{
|
||
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
|
||
TranslationMemoryMatch, TranslationMemoryMatchKind, TranslationMemorySourceKind,
|
||
TranslationMemorySummary, TranslationMemoryTrustStatus,
|
||
};
|
||
use bat_core::repositories::TranslationMemoryRepository;
|
||
use bat_core::{Error, Result};
|
||
use serde::de::DeserializeOwned;
|
||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
|
||
use sqlx::{Row, SqlitePool};
|
||
use std::fs;
|
||
use std::path::{Path, PathBuf};
|
||
use std::str::FromStr;
|
||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||
|
||
/// TM SQLite schema 版本。
|
||
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 1;
|
||
/// TM schema migration component。
|
||
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
|
||
/// 默认 TM 数据库文件名。
|
||
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
|
||
|
||
/// SQLite-backed Translation Memory 仓储。
|
||
#[derive(Debug, Clone)]
|
||
pub struct SqliteTranslationMemoryRepository {
|
||
pub(crate) pool: SqlitePool,
|
||
}
|
||
|
||
impl SqliteTranslationMemoryRepository {
|
||
/// 创建或打开 TM 数据库并执行迁移。
|
||
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
|
||
Self::open_with(path.as_ref(), true).await
|
||
}
|
||
|
||
/// 只打开已有 TM 数据库,不创建新文件。
|
||
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||
Self::open_with(path.as_ref(), false).await
|
||
}
|
||
|
||
/// 根据 active release 根目录计算默认的跨 release TM 路径。
|
||
///
|
||
/// 正式 release 根目录形如 `<output>/versions/<id>`,因此默认结果为
|
||
/// `<output>/translation-memory.sqlite`,不会写入已发布版本目录。
|
||
pub fn repository_path(resource_root: &Path) -> PathBuf {
|
||
if resource_root
|
||
.parent()
|
||
.and_then(Path::file_name)
|
||
.is_some_and(|name| name == "versions")
|
||
{
|
||
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
|
||
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
|
||
}
|
||
}
|
||
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
|
||
}
|
||
|
||
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
|
||
let absolute = bat_infrastructure_absolute(path)?;
|
||
let parent = absolute.parent().ok_or_else(|| {
|
||
Error::InvalidArgument(format!("TM 数据库缺少父目录:{}", absolute.display()))
|
||
})?;
|
||
ensure_safe_tm_parent(parent)?;
|
||
if create_if_missing {
|
||
tokio::fs::create_dir_all(parent).await?;
|
||
ensure_safe_tm_parent(parent)?;
|
||
}
|
||
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
|
||
if metadata.file_type().is_symlink() {
|
||
return Err(Error::InvalidArgument(format!(
|
||
"TM 数据库不能是 symlink:{}",
|
||
absolute.display()
|
||
)));
|
||
}
|
||
if !metadata.is_file() {
|
||
return Err(Error::InvalidArgument(format!(
|
||
"TM 数据库不是普通文件:{}",
|
||
absolute.display()
|
||
)));
|
||
}
|
||
} else if !create_if_missing {
|
||
return Err(Error::NotFound(absolute.display().to_string()));
|
||
}
|
||
|
||
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
|
||
.map_err(|error| Error::Other(error.into()))?
|
||
.create_if_missing(create_if_missing)
|
||
.journal_mode(SqliteJournalMode::Wal)
|
||
.busy_timeout(Duration::from_secs(30));
|
||
let pool = SqlitePoolOptions::new()
|
||
.max_connections(1)
|
||
.connect_with(options)
|
||
.await
|
||
.map_err(db_error)?;
|
||
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
|
||
.map_err(Error::InvalidArgument)?;
|
||
let repository = Self { pool };
|
||
repository.init_schema().await?;
|
||
Ok(repository)
|
||
}
|
||
|
||
async fn init_schema(&self) -> Result<()> {
|
||
sqlx::query(
|
||
r#"
|
||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||
component TEXT PRIMARY KEY NOT NULL,
|
||
version INTEGER NOT NULL CHECK(version >= 1)
|
||
)
|
||
"#,
|
||
)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
sqlx::query(
|
||
r#"
|
||
CREATE TABLE IF NOT EXISTS translation_memory (
|
||
record_id TEXT PRIMARY KEY NOT NULL,
|
||
source_text TEXT NOT NULL,
|
||
source_hash TEXT NOT NULL,
|
||
normalized_source_text TEXT NOT NULL,
|
||
source_context_json TEXT NOT NULL,
|
||
source_context_hash TEXT NOT NULL,
|
||
translated_text TEXT NOT NULL,
|
||
translation_source_kind TEXT NOT NULL,
|
||
trust_status TEXT NOT NULL,
|
||
official_release_id TEXT NOT NULL,
|
||
source_trace_json TEXT NOT NULL,
|
||
provider TEXT,
|
||
provider_run_id TEXT,
|
||
created_unix_seconds INTEGER NOT NULL,
|
||
updated_unix_seconds INTEGER NOT NULL,
|
||
trusted_unix_seconds INTEGER,
|
||
trusted_by TEXT,
|
||
trusted_reason TEXT,
|
||
supersedes_record_id TEXT,
|
||
superseded_by_record_id TEXT,
|
||
CHECK (length(source_text) > 0),
|
||
CHECK (length(source_hash) > 0),
|
||
CHECK (length(source_context_hash) > 0),
|
||
CHECK (length(official_release_id) > 0),
|
||
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
|
||
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
|
||
)
|
||
"#,
|
||
)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
sqlx::query(
|
||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
|
||
ON translation_memory(source_hash)",
|
||
)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
sqlx::query(
|
||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
|
||
ON translation_memory(normalized_source_text)",
|
||
)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
sqlx::query(
|
||
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
|
||
ON translation_memory(source_hash, source_context_hash)",
|
||
)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
|
||
let current: Option<i64> =
|
||
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
|
||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||
.fetch_optional(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
|
||
return Err(Error::InvalidArgument(format!(
|
||
"不支持的 Translation Memory schema 版本:{}",
|
||
current.unwrap_or_default()
|
||
)));
|
||
}
|
||
sqlx::query(
|
||
r#"
|
||
INSERT INTO schema_migrations(component, version)
|
||
VALUES (?1, ?2)
|
||
ON CONFLICT(component) DO UPDATE SET version = excluded.version
|
||
"#,
|
||
)
|
||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
Ok(())
|
||
}
|
||
|
||
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
|
||
let row = sqlx::query(
|
||
r#"
|
||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||
source_context_json, source_context_hash, translated_text,
|
||
translation_source_kind, trust_status, official_release_id,
|
||
source_trace_json, provider, provider_run_id,
|
||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||
FROM translation_memory
|
||
WHERE record_id = ?1
|
||
"#,
|
||
)
|
||
.bind(record_id)
|
||
.fetch_optional(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
row.map(row_to_entry).transpose()
|
||
}
|
||
}
|
||
|
||
#[async_trait]
|
||
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
|
||
async fn upsert_candidate(
|
||
&self,
|
||
draft: TranslationMemoryDraft,
|
||
) -> Result<TranslationMemoryEntry> {
|
||
validate_draft(&draft)?;
|
||
let entry = entry_from_draft(draft)?;
|
||
if let Some(existing) = self.find_optional(&entry.record_id).await? {
|
||
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||
return Ok(existing);
|
||
}
|
||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE translation_memory
|
||
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
|
||
updated_unix_seconds = ?5
|
||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||
"#,
|
||
)
|
||
.bind(&entry.record_id)
|
||
.bind(source_trace_json)
|
||
.bind(&entry.provider)
|
||
.bind(&entry.provider_run_id)
|
||
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
return self.find(&entry.record_id).await;
|
||
}
|
||
|
||
let source_context_json = serde_json::to_string(&entry.source_context)
|
||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||
let source_trace_json = serde_json::to_string(&entry.source_trace)
|
||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||
let result = sqlx::query(
|
||
r#"
|
||
INSERT INTO translation_memory (
|
||
record_id, source_text, source_hash, normalized_source_text,
|
||
source_context_json, source_context_hash, translated_text,
|
||
translation_source_kind, trust_status, official_release_id,
|
||
source_trace_json, provider, provider_run_id,
|
||
created_unix_seconds, updated_unix_seconds,
|
||
trusted_unix_seconds, trusted_by, trusted_reason,
|
||
supersedes_record_id, superseded_by_record_id
|
||
)
|
||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
|
||
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
|
||
ON CONFLICT(record_id) DO UPDATE SET
|
||
source_trace_json = excluded.source_trace_json,
|
||
provider = excluded.provider,
|
||
provider_run_id = excluded.provider_run_id,
|
||
updated_unix_seconds = excluded.updated_unix_seconds
|
||
WHERE translation_memory.trust_status = 'candidate'
|
||
"#,
|
||
)
|
||
.bind(&entry.record_id)
|
||
.bind(&entry.source_text)
|
||
.bind(&entry.source_hash)
|
||
.bind(&entry.normalized_source_text)
|
||
.bind(source_context_json)
|
||
.bind(&entry.source_context_hash)
|
||
.bind(&entry.translated_text)
|
||
.bind(entry.translation_source_kind.as_str())
|
||
.bind(entry.trust_status.as_str())
|
||
.bind(&entry.official_release_id)
|
||
.bind(source_trace_json)
|
||
.bind(&entry.provider)
|
||
.bind(&entry.provider_run_id)
|
||
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
|
||
.bind(
|
||
entry
|
||
.trusted_unix_seconds
|
||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
|
||
)
|
||
.bind(&entry.trusted_by)
|
||
.bind(&entry.trusted_reason)
|
||
.bind(&entry.supersedes_record_id)
|
||
.bind(&entry.superseded_by_record_id)
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
if result.rows_affected() == 0 {
|
||
return self.find(&entry.record_id).await;
|
||
}
|
||
Ok(entry)
|
||
}
|
||
|
||
async fn find_matches(
|
||
&self,
|
||
source_text: &str,
|
||
source_context: &TranslationMemoryContext,
|
||
limit: usize,
|
||
) -> Result<Vec<TranslationMemoryMatch>> {
|
||
if source_text.is_empty() {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory 查询 source_text 不能为空".to_string(),
|
||
));
|
||
}
|
||
if limit == 0 {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory 查询 limit 必须大于 0".to_string(),
|
||
));
|
||
}
|
||
let source_hash = hash_text(source_text);
|
||
let normalized_source_text = normalize_source_text(source_text);
|
||
let rows = sqlx::query(
|
||
r#"
|
||
SELECT record_id, source_text, source_hash, normalized_source_text,
|
||
source_context_json, source_context_hash, translated_text,
|
||
translation_source_kind, trust_status, official_release_id,
|
||
source_trace_json, provider, provider_run_id,
|
||
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
|
||
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
|
||
FROM translation_memory
|
||
WHERE source_hash = ?1 OR normalized_source_text = ?2
|
||
ORDER BY updated_unix_seconds DESC, record_id ASC
|
||
"#,
|
||
)
|
||
.bind(source_hash)
|
||
.bind(&normalized_source_text)
|
||
.fetch_all(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
let mut matches = rows
|
||
.into_iter()
|
||
.map(row_to_entry)
|
||
.collect::<Result<Vec<_>>>()?
|
||
.into_iter()
|
||
.filter_map(|entry| {
|
||
let raw_exact = entry.source_text == source_text;
|
||
let normalized_exact = entry.normalized_source_text == normalized_source_text;
|
||
if !raw_exact && !normalized_exact {
|
||
return None;
|
||
}
|
||
let same_context = !source_context.is_empty()
|
||
&& !entry.source_context.is_empty()
|
||
&& entry.source_context == *source_context;
|
||
let strong = raw_exact
|
||
&& same_context
|
||
&& entry.trust_status == TranslationMemoryTrustStatus::Trusted;
|
||
let match_kind = if strong {
|
||
TranslationMemoryMatchKind::StrongExact
|
||
} else if raw_exact {
|
||
TranslationMemoryMatchKind::CandidateExact
|
||
} else {
|
||
TranslationMemoryMatchKind::SourceOnly
|
||
};
|
||
Some(TranslationMemoryMatch {
|
||
can_auto_reuse: strong,
|
||
entry,
|
||
match_kind,
|
||
})
|
||
})
|
||
.collect::<Vec<_>>();
|
||
matches.sort_by(|left, right| {
|
||
match_rank(left)
|
||
.cmp(&match_rank(right))
|
||
.then_with(|| {
|
||
right
|
||
.entry
|
||
.updated_unix_seconds
|
||
.cmp(&left.entry.updated_unix_seconds)
|
||
})
|
||
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
|
||
});
|
||
matches.truncate(limit);
|
||
Ok(matches)
|
||
}
|
||
|
||
async fn confirm(
|
||
&self,
|
||
record_id: &str,
|
||
reviewer: &str,
|
||
reason: Option<String>,
|
||
) -> Result<TranslationMemoryEntry> {
|
||
if reviewer.trim().is_empty() {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory reviewer 不能为空".to_string(),
|
||
));
|
||
}
|
||
let current = self.find(record_id).await?;
|
||
if current.trust_status == TranslationMemoryTrustStatus::Trusted {
|
||
return Ok(current);
|
||
}
|
||
if current.trust_status != TranslationMemoryTrustStatus::Candidate {
|
||
return Err(Error::InvalidArgument(format!(
|
||
"Translation Memory 记录 {} 当前状态为 {},不能确认",
|
||
record_id,
|
||
current.trust_status.as_str()
|
||
)));
|
||
}
|
||
let now = unix_seconds_now();
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE translation_memory
|
||
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
|
||
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
|
||
WHERE record_id = ?1 AND trust_status = 'candidate'
|
||
"#,
|
||
)
|
||
.bind(record_id)
|
||
.bind(i64::try_from(now).unwrap_or(i64::MAX))
|
||
.bind(reviewer.trim())
|
||
.bind(reason.filter(|value| !value.trim().is_empty()))
|
||
.execute(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
self.find(record_id).await
|
||
}
|
||
|
||
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
|
||
self.find_optional(record_id)
|
||
.await?
|
||
.ok_or_else(|| Error::NotFound(record_id.to_string()))
|
||
}
|
||
|
||
async fn summary(&self) -> Result<TranslationMemorySummary> {
|
||
let row = sqlx::query(
|
||
r#"
|
||
SELECT COUNT(*) AS record_count,
|
||
SUM(CASE WHEN trust_status = 'trusted' THEN 1 ELSE 0 END) AS trusted_count,
|
||
SUM(CASE WHEN trust_status = 'candidate' THEN 1 ELSE 0 END) AS candidate_count,
|
||
SUM(CASE WHEN trust_status = 'superseded' THEN 1 ELSE 0 END) AS superseded_count,
|
||
SUM(CASE WHEN trust_status = 'rejected' THEN 1 ELSE 0 END) AS rejected_count
|
||
FROM translation_memory
|
||
"#,
|
||
)
|
||
.fetch_one(&self.pool)
|
||
.await
|
||
.map_err(db_error)?;
|
||
Ok(TranslationMemorySummary {
|
||
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
|
||
record_count: row.try_get::<i64, _>("record_count").map_err(db_error)? as u64,
|
||
trusted_count: row.try_get::<i64, _>("trusted_count").map_err(db_error)? as u64,
|
||
candidate_count: row.try_get::<i64, _>("candidate_count").map_err(db_error)? as u64,
|
||
superseded_count: row
|
||
.try_get::<i64, _>("superseded_count")
|
||
.map_err(db_error)? as u64,
|
||
rejected_count: row.try_get::<i64, _>("rejected_count").map_err(db_error)? as u64,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// 由 active official release 根目录计算默认 TM 数据库路径。
|
||
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
|
||
SqliteTranslationMemoryRepository::repository_path(resource_root)
|
||
}
|
||
|
||
/// 从 TextUnit 定位字段构建 TM 上下文。
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn translation_memory_context(
|
||
destination: &str,
|
||
archive_entry: Option<&str>,
|
||
serialized_file: Option<&str>,
|
||
path_id: Option<i64>,
|
||
class_id: Option<i32>,
|
||
field_path: Option<&str>,
|
||
format: Option<&str>,
|
||
asset_name: Option<&str>,
|
||
text_source_kind: Option<&str>,
|
||
parser_context: &TranslationMemoryContext,
|
||
) -> TranslationMemoryContext {
|
||
let mut context = TranslationMemoryContext::new();
|
||
context.insert("destination".to_string(), destination.to_string());
|
||
insert_optional(&mut context, "archive_entry", archive_entry);
|
||
insert_optional(&mut context, "serialized_file", serialized_file);
|
||
if let Some(value) = path_id {
|
||
context.insert("path_id".to_string(), value.to_string());
|
||
}
|
||
if let Some(value) = class_id {
|
||
context.insert("class_id".to_string(), value.to_string());
|
||
}
|
||
insert_optional(&mut context, "field_path", field_path);
|
||
insert_optional(&mut context, "format", format);
|
||
insert_optional(&mut context, "asset_name", asset_name);
|
||
insert_optional(&mut context, "text_source_kind", text_source_kind);
|
||
for (key, value) in parser_context {
|
||
context.insert(format!("context.{key}"), value.clone());
|
||
}
|
||
context
|
||
}
|
||
|
||
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
|
||
if let Some(value) = value.filter(|value| !value.is_empty()) {
|
||
context.insert(key.to_string(), value.to_string());
|
||
}
|
||
}
|
||
|
||
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
|
||
let source_hash = hash_text(&draft.source_text);
|
||
let normalized_source_text = normalize_source_text(&draft.source_text);
|
||
let source_context_hash = hash_context(&draft.source_context)?;
|
||
let source_kind = draft.translation_source_kind;
|
||
let record_id = record_id(
|
||
&source_hash,
|
||
&source_context_hash,
|
||
&draft.translated_text,
|
||
&draft.official_release_id,
|
||
&source_kind,
|
||
);
|
||
let observed = draft.observed_unix_seconds;
|
||
Ok(TranslationMemoryEntry {
|
||
record_id,
|
||
source_text: draft.source_text,
|
||
source_hash,
|
||
normalized_source_text,
|
||
source_context: draft.source_context,
|
||
source_context_hash,
|
||
translated_text: draft.translated_text,
|
||
translation_source_kind: source_kind,
|
||
trust_status: TranslationMemoryTrustStatus::Candidate,
|
||
official_release_id: draft.official_release_id,
|
||
source_trace: draft.source_trace,
|
||
provider: draft.provider,
|
||
provider_run_id: draft.provider_run_id,
|
||
created_unix_seconds: observed,
|
||
updated_unix_seconds: observed,
|
||
trusted_unix_seconds: None,
|
||
trusted_by: None,
|
||
trusted_reason: None,
|
||
supersedes_record_id: None,
|
||
superseded_by_record_id: None,
|
||
})
|
||
}
|
||
|
||
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
|
||
if draft.source_text.is_empty() {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory source_text 不能为空".to_string(),
|
||
));
|
||
}
|
||
if draft.translated_text.trim().is_empty() {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory translated_text 不能为空".to_string(),
|
||
));
|
||
}
|
||
if draft.official_release_id.trim().is_empty()
|
||
|| draft.source_trace.official_release_id.trim().is_empty()
|
||
{
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory official_release_id 不能为空".to_string(),
|
||
));
|
||
}
|
||
if draft.official_release_id != draft.source_trace.official_release_id {
|
||
return Err(Error::InvalidArgument(
|
||
"Translation Memory draft 的 release provenance 不一致".to_string(),
|
||
));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
|
||
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
|
||
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
|
||
Ok(TranslationMemoryEntry {
|
||
record_id: row.try_get("record_id").map_err(db_error)?,
|
||
source_text: row.try_get("source_text").map_err(db_error)?,
|
||
source_hash: row.try_get("source_hash").map_err(db_error)?,
|
||
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
|
||
source_context,
|
||
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
|
||
translated_text: row.try_get("translated_text").map_err(db_error)?,
|
||
translation_source_kind: parse_source_kind(
|
||
row.try_get::<String, _>("translation_source_kind")
|
||
.map_err(db_error)?
|
||
.as_str(),
|
||
)?,
|
||
trust_status: parse_trust_status(
|
||
row.try_get::<String, _>("trust_status")
|
||
.map_err(db_error)?
|
||
.as_str(),
|
||
)?,
|
||
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
|
||
source_trace,
|
||
provider: row.try_get("provider").map_err(db_error)?,
|
||
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
|
||
created_unix_seconds: i64_to_u64(
|
||
row.try_get("created_unix_seconds").map_err(db_error)?,
|
||
"created",
|
||
)?,
|
||
updated_unix_seconds: i64_to_u64(
|
||
row.try_get("updated_unix_seconds").map_err(db_error)?,
|
||
"updated",
|
||
)?,
|
||
trusted_unix_seconds: optional_i64_to_u64(
|
||
row.try_get("trusted_unix_seconds").map_err(db_error)?,
|
||
"trusted",
|
||
)?,
|
||
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
|
||
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
|
||
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
|
||
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
|
||
})
|
||
}
|
||
|
||
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
|
||
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
|
||
}
|
||
|
||
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
|
||
match value {
|
||
"provider" => Ok(TranslationMemorySourceKind::Provider),
|
||
"manual" => Ok(TranslationMemorySourceKind::Manual),
|
||
"imported" => Ok(TranslationMemorySourceKind::Imported),
|
||
_ => Err(Error::Serialization(format!(
|
||
"未知 Translation Memory source kind:{value}"
|
||
))),
|
||
}
|
||
}
|
||
|
||
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
|
||
match value {
|
||
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
|
||
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
|
||
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
|
||
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
|
||
_ => Err(Error::Serialization(format!(
|
||
"未知 Translation Memory trust status:{value}"
|
||
))),
|
||
}
|
||
}
|
||
|
||
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
|
||
match value.match_kind {
|
||
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
|
||
TranslationMemoryMatchKind::CandidateExact => 1,
|
||
TranslationMemoryMatchKind::SourceOnly => 2,
|
||
TranslationMemoryMatchKind::StrongExact => 1,
|
||
}
|
||
}
|
||
|
||
fn hash_text(value: &str) -> String {
|
||
blake3::hash(value.as_bytes()).to_hex().to_string()
|
||
}
|
||
|
||
fn normalize_source_text(value: &str) -> String {
|
||
value.replace("\r\n", "\n").replace('\r', "\n")
|
||
}
|
||
|
||
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
|
||
let bytes =
|
||
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
|
||
Ok(blake3::hash(&bytes).to_hex().to_string())
|
||
}
|
||
|
||
fn record_id(
|
||
source_hash: &str,
|
||
context_hash: &str,
|
||
translated_text: &str,
|
||
official_release_id: &str,
|
||
source_kind: &TranslationMemorySourceKind,
|
||
) -> String {
|
||
let mut key = Vec::new();
|
||
for value in [
|
||
source_hash,
|
||
context_hash,
|
||
translated_text,
|
||
official_release_id,
|
||
source_kind.as_str(),
|
||
] {
|
||
key.extend_from_slice(value.as_bytes());
|
||
key.push(0);
|
||
}
|
||
format!("tm-{}", blake3::hash(&key).to_hex())
|
||
}
|
||
|
||
fn unix_seconds_now() -> u64 {
|
||
SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_secs()
|
||
}
|
||
|
||
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
|
||
u64::try_from(value)
|
||
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
|
||
}
|
||
|
||
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
|
||
value.map(|value| i64_to_u64(value, label)).transpose()
|
||
}
|
||
|
||
fn db_error(error: sqlx::Error) -> Error {
|
||
Error::Other(error.into())
|
||
}
|
||
|
||
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
|
||
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
|
||
}
|
||
|
||
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
|
||
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
|
||
.map_err(Error::InvalidArgument)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use bat_core::domain::TranslationMemorySourceTrace;
|
||
|
||
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
|
||
let source_trace = TranslationMemorySourceTrace {
|
||
official_release_id: release.to_string(),
|
||
unit_id: Some(format!("{release}-unit")),
|
||
task_id: Some(format!("{release}-task")),
|
||
destination: Some("Bundles/story.bundle".to_string()),
|
||
archive_entry: None,
|
||
serialized_file: Some("CAB-story".to_string()),
|
||
path_id: Some(1),
|
||
class_id: Some(49),
|
||
field_path: Some("m_Text".to_string()),
|
||
format: Some("plain".to_string()),
|
||
asset_name: Some("Story".to_string()),
|
||
text_source_kind: Some("text_asset".to_string()),
|
||
source_url: Some("https://example.invalid/story".to_string()),
|
||
};
|
||
TranslationMemoryDraft {
|
||
source_text: source.to_string(),
|
||
source_context: translation_memory_context(
|
||
"Bundles/story.bundle",
|
||
None,
|
||
Some("CAB-story"),
|
||
Some(1),
|
||
Some(49),
|
||
Some("m_Text"),
|
||
Some("plain"),
|
||
Some("Story"),
|
||
Some("text_asset"),
|
||
&TranslationMemoryContext::new(),
|
||
),
|
||
translated_text: translated.to_string(),
|
||
translation_source_kind: TranslationMemorySourceKind::Provider,
|
||
official_release_id: release.to_string(),
|
||
source_trace,
|
||
provider: Some("mock".to_string()),
|
||
provider_run_id: Some(format!("mock:{release}")),
|
||
observed_unix_seconds: 1,
|
||
}
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let repository = SqliteTranslationMemoryRepository::new(
|
||
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
let entry = repository
|
||
.upsert_candidate(draft("release-1", "Hello", "你好"))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
|
||
let trusted = repository
|
||
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
|
||
|
||
let query = draft("release-2", "Hello", "ignored");
|
||
let matches = repository
|
||
.find_matches("Hello", &query.source_context, 10)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(matches.len(), 1);
|
||
assert_eq!(
|
||
matches[0].match_kind,
|
||
TranslationMemoryMatchKind::StrongExact
|
||
);
|
||
assert!(matches[0].can_auto_reuse);
|
||
assert_eq!(matches[0].entry.translated_text, "你好");
|
||
assert_eq!(matches[0].entry.official_release_id, "release-1");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn rejects_future_schema_version() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let path = temp.path().join("tm.sqlite");
|
||
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
|
||
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
|
||
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
|
||
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
|
||
.execute(&repository.pool)
|
||
.await
|
||
.unwrap();
|
||
repository.pool.close().await;
|
||
|
||
let error = SqliteTranslationMemoryRepository::new(&path)
|
||
.await
|
||
.unwrap_err();
|
||
assert!(error
|
||
.to_string()
|
||
.contains("不支持的 Translation Memory schema"));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||
.await
|
||
.unwrap();
|
||
let first = draft("release-1", "Hello", "你好");
|
||
repository.upsert_candidate(first.clone()).await.unwrap();
|
||
repository.upsert_candidate(first).await.unwrap();
|
||
assert_eq!(repository.summary().await.unwrap().record_count, 1);
|
||
|
||
let mut different = draft("release-2", "Hello", "你好");
|
||
different
|
||
.source_context
|
||
.insert("field_path".to_string(), "m_Other".to_string());
|
||
let matches = repository
|
||
.find_matches("Hello", &different.source_context, 10)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
matches[0].match_kind,
|
||
TranslationMemoryMatchKind::CandidateExact
|
||
);
|
||
assert!(!matches[0].can_auto_reuse);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn rejects_empty_translation_candidates() {
|
||
let temp = tempfile::TempDir::new().unwrap();
|
||
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
|
||
.await
|
||
.unwrap();
|
||
let error = repository
|
||
.upsert_candidate(draft("release-1", "Hello", " \n"))
|
||
.await
|
||
.unwrap_err();
|
||
assert!(error.to_string().contains("translated_text"));
|
||
assert_eq!(repository.summary().await.unwrap().record_count, 0);
|
||
}
|
||
}
|