Files
BlueArchiveToolkit/infrastructure/src/translation_memory.rs
T

2703 lines
99 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! 跨 official release 的 Translation Memory SQLite 仓储。
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
use crate::sqlite_migration::{
self, ExpectedColumn, ExpectedIndex, ExpectedTable, SqliteSchemaSnapshot,
};
use async_trait::async_trait;
use bat_core::domain::{
TranslationMemoryConflict, TranslationMemoryContext, TranslationMemoryDraft,
TranslationMemoryEntry, TranslationMemoryMatch, TranslationMemoryMatchKind,
TranslationMemorySourceKind, TranslationMemorySummary, TranslationMemoryTrustStatus,
};
use bat_core::repositories::TranslationMemoryRepository;
use bat_core::{Error, Result};
use serde::de::DeserializeOwned;
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode};
use sqlx::{Row, SqlitePool};
use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// TM SQLite schema 版本。
pub const TRANSLATION_MEMORY_SCHEMA_VERSION: u32 = 2;
/// TM schema migration component。
pub const TRANSLATION_MEMORY_SCHEMA_COMPONENT: &str = "translation_memory";
/// 默认 TM 数据库文件名。
pub const TRANSLATION_MEMORY_REPOSITORY_FILE: &str = "translation-memory.sqlite";
/// SQLite-backed Translation Memory 仓储。
#[derive(Debug, Clone)]
pub struct SqliteTranslationMemoryRepository {
pub(crate) pool: SqlitePool,
}
impl SqliteTranslationMemoryRepository {
/// 创建或打开 TM 数据库并执行迁移。
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
Self::open_with(path.as_ref(), true).await
}
/// 只打开已有 TM 数据库,不创建新文件。
pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
Self::open_with(path.as_ref(), false).await
}
/// 根据 active release 根目录计算默认的跨 release TM 路径。
///
/// 正式 release 根目录形如 `<output>/versions/<id>`,因此默认结果为
/// `<output>/translation-memory.sqlite`,不会写入已发布版本目录。
pub fn repository_path(resource_root: &Path) -> PathBuf {
if resource_root
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "versions")
{
if let Some(output_root) = resource_root.parent().and_then(Path::parent) {
return output_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE);
}
}
resource_root.join(TRANSLATION_MEMORY_REPOSITORY_FILE)
}
async fn open_with(path: &Path, create_if_missing: bool) -> Result<Self> {
let absolute = bat_infrastructure_absolute(path)?;
let parent = absolute.parent().ok_or_else(|| {
Error::InvalidArgument(format!("TM 数据库缺少父目录:{}", absolute.display()))
})?;
ensure_safe_tm_parent(parent)?;
if create_if_missing {
tokio::fs::create_dir_all(parent).await?;
ensure_safe_tm_parent(parent)?;
}
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
if metadata.file_type().is_symlink() {
return Err(Error::InvalidArgument(format!(
"TM 数据库不能是 symlink{}",
absolute.display()
)));
}
if !metadata.is_file() {
return Err(Error::InvalidArgument(format!(
"TM 数据库不是普通文件:{}",
absolute.display()
)));
}
} else if !create_if_missing {
return Err(Error::NotFound(absolute.display().to_string()));
}
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
if metadata.len() > 0 {
let snapshot = sqlite_migration::read_only_preflight(
&absolute,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
classify_translation_memory_schema(&snapshot)?;
}
}
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
.map_err(|error| Error::Other(error.into()))?
.create_if_missing(create_if_missing)
.journal_mode(SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30));
let pool = sqlite_migration::connect_writable_pool(options)
.await
.map_err(db_error)?;
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
.map_err(Error::InvalidArgument)?;
let repository = Self { pool };
repository.init_schema().await?;
Ok(repository)
}
async fn init_schema(&self) -> Result<()> {
self.init_schema_with_failure(None).await
}
#[cfg(test)]
async fn init_schema_with_test_failure(&self, fail_after_step: usize) -> Result<()> {
self.init_schema_with_failure(Some(fail_after_step)).await
}
async fn init_schema_with_failure(&self, fail_after_step: Option<usize>) -> Result<()> {
let mut transaction = sqlite_migration::begin_immediate(&self.pool)
.await
.map_err(db_error)?;
let result = self
.migrate_in_transaction(&mut transaction, fail_after_step)
.await;
match result {
Ok(()) => transaction.commit().await.map_err(db_error),
Err(error) => {
let _ = transaction.rollback().await;
Err(error)
}
}
}
async fn migrate_in_transaction(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
fail_after_step: Option<usize>,
) -> Result<()> {
let snapshot = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
match classify_translation_memory_schema(&snapshot)? {
TranslationMemorySchemaState::Empty => {
create_translation_memory_schema(transaction, fail_after_step).await?;
sqlite_migration::write_component_version(
transaction,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
TRANSLATION_MEMORY_SCHEMA_VERSION,
)
.await
.map_err(db_error)?;
}
TranslationMemorySchemaState::V1 => {
let revalidated = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
if !matches_translation_memory_v1_fingerprint(&revalidated)
&& !matches_translation_memory_v1_component_fingerprint(&revalidated)
{
return Err(invalid_translation_memory_schema(
"V1 migration 起点在事务内发生变化".to_string(),
));
}
if !revalidated
.tables
.contains_key(sqlite_migration::SCHEMA_MIGRATIONS_TABLE)
{
sqlite_migration::create_schema_migrations_table(transaction)
.await
.map_err(db_error)?;
}
create_translation_memory_governance_table(transaction, fail_after_step).await?;
let migrated = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
if !matches_translation_memory_v2_fingerprint(&migrated)
&& !matches_translation_memory_v2_component_fingerprint(&migrated)
{
return Err(invalid_translation_memory_schema(
"V1 migration 结果与 V2 fingerprint 不一致".to_string(),
));
}
sqlite_migration::write_component_version(
transaction,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
TRANSLATION_MEMORY_SCHEMA_VERSION,
)
.await
.map_err(db_error)?;
}
TranslationMemorySchemaState::V2 => {
if snapshot.component_version.is_none() {
if !snapshot
.tables
.contains_key(sqlite_migration::SCHEMA_MIGRATIONS_TABLE)
{
sqlite_migration::create_schema_migrations_table(transaction)
.await
.map_err(db_error)?;
}
sqlite_migration::write_component_version(
transaction,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
TRANSLATION_MEMORY_SCHEMA_VERSION,
)
.await
.map_err(db_error)?;
}
}
}
let final_snapshot = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
if final_snapshot.component_version != Some(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|| !matches_translation_memory_v2_fingerprint(&final_snapshot)
{
return Err(invalid_translation_memory_schema(
"migration 结果与当前 schema fingerprint 不一致".to_string(),
));
}
Ok(())
}
async fn find_optional(&self, record_id: &str) -> Result<Option<TranslationMemoryEntry>> {
let row = sqlx::query(
r#"
SELECT record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
FROM translation_memory
WHERE record_id = ?1
"#,
)
.bind(record_id)
.fetch_optional(&self.pool)
.await
.map_err(db_error)?;
row.map(row_to_entry).transpose()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TranslationMemorySchemaState {
Empty,
V1,
V2,
}
fn classify_translation_memory_schema(
snapshot: &SqliteSchemaSnapshot,
) -> Result<TranslationMemorySchemaState> {
if snapshot.is_empty() {
return Ok(TranslationMemorySchemaState::Empty);
}
if let Some(observed) = snapshot.component_version {
if observed > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) {
return Err(invalid_translation_memory_schema(format!(
"不支持的 Translation Memory schema 版本:observed={observed}, supported={TRANSLATION_MEMORY_SCHEMA_VERSION}"
)));
}
if observed < 1 {
return Err(invalid_translation_memory_schema(format!(
"schema version 无效:observed={observed}, supported=1..={TRANSLATION_MEMORY_SCHEMA_VERSION}"
)));
}
}
let v1 = matches_translation_memory_v1_fingerprint(snapshot)
|| (snapshot.component_version.is_none()
&& matches_translation_memory_v1_component_fingerprint(snapshot));
let v2 = matches_translation_memory_v2_fingerprint(snapshot)
|| (snapshot.component_version.is_none()
&& matches_translation_memory_v2_component_fingerprint(snapshot));
match snapshot.component_version {
Some(1) if v1 => Ok(TranslationMemorySchemaState::V1),
None if v1 => Ok(TranslationMemorySchemaState::V1),
Some(2) if v2 => Ok(TranslationMemorySchemaState::V2),
None if v2 => Ok(TranslationMemorySchemaState::V2),
Some(observed) => Err(invalid_translation_memory_schema(format!(
"schema version 与实际结构不一致:observed={observed}, supported={TRANSLATION_MEMORY_SCHEMA_VERSION}"
))),
None => Err(invalid_translation_memory_schema(
"未识别的 legacy schema,拒绝静默修复".to_string(),
)),
}
}
fn invalid_translation_memory_schema(detail: String) -> Error {
Error::InvalidArgument(format!("Translation Memory schema 无效:{detail}"))
}
fn matches_translation_memory_v1_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, true, false)
}
fn matches_translation_memory_v1_component_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, false, false)
}
fn matches_translation_memory_v2_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, true, true)
}
fn matches_translation_memory_v2_component_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, false, true)
}
fn matches_translation_memory_fingerprint_with_migrations(
snapshot: &SqliteSchemaSnapshot,
include_migrations: bool,
include_governance: bool,
) -> bool {
let mut indexes = vec![
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_source_hash",
columns: &["source_hash"],
},
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_normalized_source",
columns: &["normalized_source_text"],
},
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_context",
columns: &["source_hash", "source_context_hash"],
},
];
if include_governance {
indexes.extend([
ExpectedIndex {
table: "translation_memory_trust_events",
name: "idx_translation_memory_trust_events_identity",
columns: &["source_hash", "source_context_hash"],
},
ExpectedIndex {
table: "translation_memory_trust_events",
name: "idx_translation_memory_trust_events_observed",
columns: &["observed_unix_seconds"],
},
]);
}
let mut tables = vec![ExpectedTable {
name: "translation_memory",
columns: &TRANSLATION_MEMORY_COLUMNS,
}];
if include_governance {
tables.push(ExpectedTable {
name: "translation_memory_trust_events",
columns: &TRANSLATION_MEMORY_TRUST_EVENT_COLUMNS,
});
}
if include_migrations {
tables.insert(0, sqlite_migration::schema_migrations_table());
}
sqlite_migration::matches_fingerprint(snapshot, &tables, &indexes)
}
const TRANSLATION_MEMORY_COLUMNS: [ExpectedColumn<'static>; 20] = [
ExpectedColumn {
name: "record_id",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: true,
},
ExpectedColumn {
name: "source_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "normalized_source_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "translated_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "translation_source_kind",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trust_status",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "official_release_id",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_trace_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "provider",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "provider_run_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "created_unix_seconds",
data_type: "INTEGER",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "updated_unix_seconds",
data_type: "INTEGER",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_unix_seconds",
data_type: "INTEGER",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_by",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_reason",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "supersedes_record_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "superseded_by_record_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
];
const TRANSLATION_MEMORY_TRUST_EVENT_COLUMNS: [ExpectedColumn<'static>; 11] = [
ExpectedColumn {
name: "event_id",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: true,
},
ExpectedColumn {
name: "action",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "winner_record_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "affected_record_ids_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "reviewer",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "reason",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "observed_unix_seconds",
data_type: "INTEGER",
not_null: true,
default_value: None,
primary_key: false,
},
];
async fn create_translation_memory_schema(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
fail_after_step: Option<usize>,
) -> Result<()> {
let steps = [
"CREATE TABLE schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)",
"CREATE TABLE translation_memory (
record_id TEXT PRIMARY KEY NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
normalized_source_text TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
translated_text TEXT NOT NULL,
translation_source_kind TEXT NOT NULL,
trust_status TEXT NOT NULL,
official_release_id TEXT NOT NULL,
source_trace_json TEXT NOT NULL,
provider TEXT,
provider_run_id TEXT,
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
trusted_unix_seconds INTEGER,
trusted_by TEXT,
trusted_reason TEXT,
supersedes_record_id TEXT,
superseded_by_record_id TEXT,
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(official_release_id) > 0),
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
)",
"CREATE INDEX idx_translation_memory_source_hash
ON translation_memory(source_hash)",
"CREATE INDEX idx_translation_memory_normalized_source
ON translation_memory(normalized_source_text)",
"CREATE INDEX idx_translation_memory_context
ON translation_memory(source_hash, source_context_hash)",
"CREATE TABLE translation_memory_trust_events (
event_id TEXT PRIMARY KEY NOT NULL,
action TEXT NOT NULL,
winner_record_id TEXT,
affected_record_ids_json TEXT NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
reviewer TEXT NOT NULL,
reason TEXT,
observed_unix_seconds INTEGER NOT NULL,
CHECK (action IN ('confirm', 'supersede', 'resolve_conflict')),
CHECK (length(affected_record_ids_json) > 0),
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(reviewer) > 0)
)",
"CREATE INDEX idx_translation_memory_trust_events_identity
ON translation_memory_trust_events(source_hash, source_context_hash)",
"CREATE INDEX idx_translation_memory_trust_events_observed
ON translation_memory_trust_events(observed_unix_seconds)",
];
for (index, statement) in steps.iter().enumerate() {
sqlx::query(statement)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
if fail_after_step == Some(index + 1) {
return Err(Error::Other(anyhow::anyhow!(
"Translation Memory migration failed after step {}",
index + 1
)));
}
}
Ok(())
}
async fn create_translation_memory_governance_table(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
fail_after_step: Option<usize>,
) -> Result<()> {
let steps = [
"CREATE TABLE translation_memory_trust_events (
event_id TEXT PRIMARY KEY NOT NULL,
action TEXT NOT NULL,
winner_record_id TEXT,
affected_record_ids_json TEXT NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
reviewer TEXT NOT NULL,
reason TEXT,
observed_unix_seconds INTEGER NOT NULL,
CHECK (action IN ('confirm', 'supersede', 'resolve_conflict')),
CHECK (length(affected_record_ids_json) > 0),
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(reviewer) > 0)
)",
"CREATE INDEX idx_translation_memory_trust_events_identity
ON translation_memory_trust_events(source_hash, source_context_hash)",
"CREATE INDEX idx_translation_memory_trust_events_observed
ON translation_memory_trust_events(observed_unix_seconds)",
];
for (index, statement) in steps.iter().enumerate() {
sqlx::query(statement)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
if fail_after_step == Some(index + 1) {
return Err(Error::Other(anyhow::anyhow!(
"Translation Memory V1 migration failed after step {}",
index + 1
)));
}
}
Ok(())
}
#[async_trait]
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
async fn upsert_candidate(
&self,
draft: TranslationMemoryDraft,
) -> Result<TranslationMemoryEntry> {
validate_draft(&draft)?;
let entry = entry_from_draft(draft)?;
if let Some(existing) = self.find_optional(&entry.record_id).await? {
if existing.trust_status != TranslationMemoryTrustStatus::Candidate {
return Ok(existing);
}
let source_trace_json = serde_json::to_string(&entry.source_trace)
.map_err(|error| Error::Serialization(error.to_string()))?;
sqlx::query(
r#"
UPDATE translation_memory
SET source_trace_json = ?2, provider = ?3, provider_run_id = ?4,
updated_unix_seconds = ?5
WHERE record_id = ?1 AND trust_status = 'candidate'
"#,
)
.bind(&entry.record_id)
.bind(source_trace_json)
.bind(&entry.provider)
.bind(&entry.provider_run_id)
.bind(i64::try_from(entry.updated_unix_seconds).unwrap_or(i64::MAX))
.execute(&self.pool)
.await
.map_err(db_error)?;
return self.find(&entry.record_id).await;
}
let source_context_json = serde_json::to_string(&entry.source_context)
.map_err(|error| Error::Serialization(error.to_string()))?;
let source_trace_json = serde_json::to_string(&entry.source_trace)
.map_err(|error| Error::Serialization(error.to_string()))?;
let result = sqlx::query(
r#"
INSERT INTO translation_memory (
record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds,
trusted_unix_seconds, trusted_by, trusted_reason,
supersedes_record_id, superseded_by_record_id
)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)
ON CONFLICT(record_id) DO UPDATE SET
source_trace_json = excluded.source_trace_json,
provider = excluded.provider,
provider_run_id = excluded.provider_run_id,
updated_unix_seconds = excluded.updated_unix_seconds
WHERE translation_memory.trust_status = 'candidate'
"#,
)
.bind(&entry.record_id)
.bind(&entry.source_text)
.bind(&entry.source_hash)
.bind(&entry.normalized_source_text)
.bind(source_context_json)
.bind(&entry.source_context_hash)
.bind(&entry.translated_text)
.bind(entry.translation_source_kind.as_str())
.bind(entry.trust_status.as_str())
.bind(&entry.official_release_id)
.bind(source_trace_json)
.bind(&entry.provider)
.bind(&entry.provider_run_id)
.bind(i64::try_from(entry.created_unix_seconds).unwrap_or(i64::MAX))
.bind(
entry
.trusted_unix_seconds
.map(|value| i64::try_from(value).unwrap_or(i64::MAX)),
)
.bind(&entry.trusted_by)
.bind(&entry.trusted_reason)
.bind(&entry.supersedes_record_id)
.bind(&entry.superseded_by_record_id)
.execute(&self.pool)
.await
.map_err(db_error)?;
if result.rows_affected() == 0 {
return self.find(&entry.record_id).await;
}
Ok(entry)
}
async fn find_matches(
&self,
source_text: &str,
source_context: &TranslationMemoryContext,
limit: usize,
) -> Result<Vec<TranslationMemoryMatch>> {
if source_text.is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory 查询 source_text 不能为空".to_string(),
));
}
if limit == 0 {
return Err(Error::InvalidArgument(
"Translation Memory 查询 limit 必须大于 0".to_string(),
));
}
let source_hash = hash_text(source_text);
let normalized_source_text = normalize_source_text(source_text);
let rows = sqlx::query(
r#"
SELECT record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
FROM translation_memory
WHERE source_hash = ?1 OR normalized_source_text = ?2 OR source_text = ?3
ORDER BY updated_unix_seconds DESC, record_id ASC
"#,
)
.bind(source_hash)
.bind(&normalized_source_text)
.bind(source_text)
.fetch_all(&self.pool)
.await
.map_err(db_error)?;
let matches = rows
.into_iter()
.map(row_to_entry)
.collect::<Result<Vec<_>>>()?
.into_iter();
let exact_trusted_count = matches
.clone()
.filter(|entry| {
entry.source_text == source_text
&& entry.source_context == *source_context
&& is_current_trusted(entry)
})
.count();
let mut matches = matches
.filter_map(|entry| {
let raw_exact = entry.source_text == source_text;
let normalized_exact = entry.normalized_source_text == normalized_source_text;
if !raw_exact && !normalized_exact {
return None;
}
let same_context = raw_exact && entry.source_context == *source_context;
let conflict = same_context && exact_trusted_count > 1;
let strong = raw_exact
&& same_context
&& exact_trusted_count == 1
&& is_current_trusted(&entry);
let match_kind = if strong {
TranslationMemoryMatchKind::StrongExact
} else if conflict {
TranslationMemoryMatchKind::TrustedConflict
} else if raw_exact {
TranslationMemoryMatchKind::CandidateExact
} else {
TranslationMemoryMatchKind::SourceOnly
};
Some(TranslationMemoryMatch {
can_auto_reuse: strong,
entry,
match_kind,
})
})
.collect::<Vec<_>>();
matches.sort_by(|left, right| {
match_rank(left)
.cmp(&match_rank(right))
.then_with(|| {
right
.entry
.updated_unix_seconds
.cmp(&left.entry.updated_unix_seconds)
})
.then_with(|| left.entry.record_id.cmp(&right.entry.record_id))
});
matches.truncate(limit);
Ok(matches)
}
async fn confirm(
&self,
record_id: &str,
reviewer: &str,
reason: Option<String>,
) -> Result<TranslationMemoryEntry> {
self.confirm_with_supersede(record_id, reviewer, reason, None)
.await
}
async fn confirm_with_supersede(
&self,
record_id: &str,
reviewer: &str,
reason: Option<String>,
supersede_record_id: Option<&str>,
) -> Result<TranslationMemoryEntry> {
if reviewer.trim().is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory reviewer 不能为空".to_string(),
));
}
let reviewer = reviewer.trim();
let reason = reason
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
let mut transaction = sqlite_migration::begin_immediate(&self.pool)
.await
.map_err(db_error)?;
let result = self
.confirm_in_transaction(
&mut transaction,
record_id,
reviewer,
reason.as_deref(),
supersede_record_id,
)
.await;
match result {
Ok(()) => {
transaction.commit().await.map_err(db_error)?;
self.find(record_id).await
}
Err(error) => {
let _ = transaction.rollback().await;
Err(error)
}
}
}
async fn list_conflicts(&self, limit: usize) -> Result<Vec<TranslationMemoryConflict>> {
if limit == 0 || limit > 1000 {
return Err(Error::InvalidArgument(
"Translation Memory conflict limit 必须在 1..=1000 范围内".to_string(),
));
}
let entries = self.load_all_entries().await?;
Ok(conflict_groups(entries).into_iter().take(limit).collect())
}
async fn resolve_conflict(
&self,
winner_record_id: &str,
expected_trusted_record_ids: &[String],
reviewer: &str,
reason: &str,
) -> Result<TranslationMemoryEntry> {
if reviewer.trim().is_empty() || reason.trim().is_empty() {
return Err(Error::InvalidArgument(
"conflict resolution requires non-empty reviewer and reason".to_string(),
));
}
if expected_trusted_record_ids.is_empty() {
return Err(Error::InvalidArgument(
"conflict_snapshot_stale: expected_trusted_record_ids 不能为空".to_string(),
));
}
let mut expected = expected_trusted_record_ids.to_vec();
expected.sort();
expected.dedup();
if expected.len() != expected_trusted_record_ids.len() {
return Err(Error::InvalidArgument(
"conflict_snapshot_stale: expected_trusted_record_ids 必须唯一".to_string(),
));
}
let mut transaction = sqlite_migration::begin_immediate(&self.pool)
.await
.map_err(db_error)?;
let result = self
.resolve_conflict_in_transaction(
&mut transaction,
winner_record_id,
&expected,
reviewer.trim(),
reason.trim(),
)
.await;
match result {
Ok(()) => {
transaction.commit().await.map_err(db_error)?;
self.find(winner_record_id).await
}
Err(error) => {
let _ = transaction.rollback().await;
Err(error)
}
}
}
async fn find(&self, record_id: &str) -> Result<TranslationMemoryEntry> {
self.find_optional(record_id)
.await?
.ok_or_else(|| Error::NotFound(record_id.to_string()))
}
async fn summary(&self) -> Result<TranslationMemorySummary> {
let entries = self.load_all_entries().await?;
let record_count = entries.len() as u64;
let trusted_count = entries
.iter()
.filter(|entry| entry.trust_status == TranslationMemoryTrustStatus::Trusted)
.count() as u64;
let candidate_count = entries
.iter()
.filter(|entry| entry.trust_status == TranslationMemoryTrustStatus::Candidate)
.count() as u64;
let superseded_count = entries
.iter()
.filter(|entry| entry.trust_status == TranslationMemoryTrustStatus::Superseded)
.count() as u64;
let rejected_count = entries
.iter()
.filter(|entry| entry.trust_status == TranslationMemoryTrustStatus::Rejected)
.count() as u64;
let groups = group_entries(entries);
let trusted_conflict_group_count = groups
.values()
.filter(|entries| current_trusted_ids(entries).len() > 1)
.count() as u64;
let current_trusted_count = groups
.values()
.filter(|entries| current_trusted_ids(entries).len() == 1)
.count() as u64;
Ok(TranslationMemorySummary {
schema_version: TRANSLATION_MEMORY_SCHEMA_VERSION,
record_count,
trusted_count,
candidate_count,
superseded_count,
rejected_count,
trusted_conflict_group_count,
current_trusted_count,
})
}
}
impl SqliteTranslationMemoryRepository {
async fn load_all_entries(&self) -> Result<Vec<TranslationMemoryEntry>> {
let rows = sqlx::query(TRANSLATION_MEMORY_SELECT)
.fetch_all(&self.pool)
.await
.map_err(db_error)?;
rows.into_iter().map(row_to_entry).collect()
}
async fn confirm_in_transaction(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
record_id: &str,
reviewer: &str,
reason: Option<&str>,
supersede_record_id: Option<&str>,
) -> Result<()> {
let target = find_optional_in_transaction(transaction, record_id)
.await?
.ok_or_else(|| Error::NotFound(record_id.to_string()))?;
let identity_entries = exact_identity_entries_in_transaction(
transaction,
&target.source_text,
&target.source_context,
)
.await?;
let current_ids = current_trusted_ids(&identity_entries);
if target.trust_status == TranslationMemoryTrustStatus::Trusted
&& target.superseded_by_record_id.is_none()
{
if current_ids.len() > 1 {
return Err(trusted_conflict_error(&current_ids));
}
if current_ids == [record_id.to_string()] {
return Ok(());
}
}
if target.trust_status != TranslationMemoryTrustStatus::Candidate {
return Err(Error::InvalidArgument(format!(
"Translation Memory 记录 {} 当前状态为 {},不能确认",
record_id,
target.trust_status.as_str()
)));
}
if current_ids.len() > 1 {
return Err(trusted_conflict_error(&current_ids));
}
let now = unix_seconds_now();
if let Some(current_id) = current_ids.first() {
if target.translated_text
== identity_entries
.iter()
.find(|entry| entry.record_id == *current_id)
.map(|entry| entry.translated_text.as_str())
.unwrap_or_default()
{
return Err(Error::InvalidArgument(format!(
"trusted_translation_already_exists: current_trusted_record_id={current_id}"
)));
}
let Some(expected) = supersede_record_id else {
return Err(Error::InvalidArgument(format!(
"explicit_supersede_required: current_trusted_record_id={current_id}"
)));
};
if expected != current_id {
return Err(Error::InvalidArgument(format!(
"stale_supersede_target: current_trusted_record_id={current_id}, expected={expected}"
)));
}
if reason.is_none_or(str::is_empty) {
return Err(Error::InvalidArgument(
"supersede requires non-empty reason".to_string(),
));
}
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'superseded', superseded_by_record_id = ?2,
updated_unix_seconds = ?3
WHERE record_id = ?1 AND trust_status = 'trusted'",
)
.bind(current_id)
.bind(record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(db_error)?;
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
trusted_by = ?3, trusted_reason = ?4,
supersedes_record_id = ?5, updated_unix_seconds = ?2
WHERE record_id = ?1 AND trust_status = 'candidate'",
)
.bind(record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.bind(reviewer)
.bind(reason)
.bind(current_id)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
insert_trust_event(
transaction,
"supersede",
Some(record_id),
&[current_id.clone(), record_id.to_string()],
&target,
reviewer,
reason,
now,
)
.await?;
} else {
if supersede_record_id.is_some() {
return Err(Error::InvalidArgument(
"stale_supersede_target: 当前 exact identity 没有 current Trusted".to_string(),
));
}
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
trusted_by = ?3, trusted_reason = ?4, updated_unix_seconds = ?2
WHERE record_id = ?1 AND trust_status = 'candidate'",
)
.bind(record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.bind(reviewer)
.bind(reason)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
insert_trust_event(
transaction,
"confirm",
Some(record_id),
&[record_id.to_string()],
&target,
reviewer,
reason,
now,
)
.await?;
}
let final_entries = exact_identity_entries_in_transaction(
transaction,
&target.source_text,
&target.source_context,
)
.await?;
if current_trusted_ids(&final_entries) != [record_id.to_string()] {
return Err(Error::InvalidArgument(
"Translation Memory confirm 未建立唯一 current Trusted".to_string(),
));
}
Ok(())
}
async fn resolve_conflict_in_transaction(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
winner_record_id: &str,
expected_trusted_record_ids: &[String],
reviewer: &str,
reason: &str,
) -> Result<()> {
let winner = find_optional_in_transaction(transaction, winner_record_id)
.await?
.ok_or_else(|| Error::NotFound(winner_record_id.to_string()))?;
let identity_entries = exact_identity_entries_in_transaction(
transaction,
&winner.source_text,
&winner.source_context,
)
.await?;
let current_ids = current_trusted_ids(&identity_entries);
if current_ids.len() <= 1 || current_ids != expected_trusted_record_ids {
return Err(Error::InvalidArgument(format!(
"conflict_snapshot_stale: current_trusted_record_ids={}",
current_ids.join(",")
)));
}
if winner.trust_status != TranslationMemoryTrustStatus::Candidate
&& !current_ids.contains(&winner.record_id)
{
return Err(Error::InvalidArgument(format!(
"invalid_conflict_winner: record_id={winner_record_id}"
)));
}
let winner_is_candidate = winner.trust_status == TranslationMemoryTrustStatus::Candidate;
let loser_ids = current_ids
.iter()
.filter(|record_id| record_id.as_str() != winner_record_id)
.cloned()
.collect::<Vec<_>>();
if loser_ids.is_empty() && !winner_is_candidate {
return Err(Error::InvalidArgument(
"invalid_conflict_winner: winner 不属于 conflict set".to_string(),
));
}
let now = unix_seconds_now();
for loser_id in &loser_ids {
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'superseded', superseded_by_record_id = ?2,
updated_unix_seconds = ?3
WHERE record_id = ?1 AND trust_status = 'trusted'",
)
.bind(loser_id)
.bind(winner_record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(db_error)?;
}
if winner_is_candidate {
let supersedes = (loser_ids.len() == 1).then(|| loser_ids[0].clone());
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'trusted', trusted_unix_seconds = ?2,
trusted_by = ?3, trusted_reason = ?4,
supersedes_record_id = ?5, updated_unix_seconds = ?2
WHERE record_id = ?1 AND trust_status = 'candidate'",
)
.bind(winner_record_id)
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.bind(reviewer)
.bind(reason)
.bind(supersedes)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
} else if loser_ids.len() == 1 {
sqlx::query(
"UPDATE translation_memory
SET supersedes_record_id = ?, updated_unix_seconds = ?
WHERE record_id = ?",
)
.bind(&loser_ids[0])
.bind(i64::try_from(now).unwrap_or(i64::MAX))
.bind(winner_record_id)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
}
let mut affected = expected_trusted_record_ids.to_vec();
if winner_is_candidate {
affected.push(winner_record_id.to_string());
affected.sort();
}
insert_trust_event(
transaction,
"resolve_conflict",
Some(winner_record_id),
&affected,
&winner,
reviewer,
Some(reason),
now,
)
.await?;
let final_entries = exact_identity_entries_in_transaction(
transaction,
&winner.source_text,
&winner.source_context,
)
.await?;
let final_ids = current_trusted_ids(&final_entries);
if final_ids != [winner_record_id.to_string()]
|| expected_trusted_record_ids.iter().any(|record_id| {
record_id != winner_record_id
&& final_entries.iter().any(|entry| {
entry.record_id == *record_id
&& entry.trust_status != TranslationMemoryTrustStatus::Superseded
})
})
{
return Err(Error::InvalidArgument(
"Translation Memory conflict resolution 未建立唯一 current Trusted".to_string(),
));
}
Ok(())
}
}
const TRANSLATION_MEMORY_SELECT: &str = r#"
SELECT record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds, trusted_unix_seconds,
trusted_by, trusted_reason, supersedes_record_id, superseded_by_record_id
FROM translation_memory
"#;
async fn find_optional_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
record_id: &str,
) -> Result<Option<TranslationMemoryEntry>> {
let row = sqlx::query(&format!(
"{TRANSLATION_MEMORY_SELECT}\nWHERE record_id = ?1"
))
.bind(record_id)
.fetch_optional(&mut **transaction)
.await
.map_err(db_error)?;
row.map(row_to_entry).transpose()
}
async fn exact_identity_entries_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
source_text: &str,
source_context: &TranslationMemoryContext,
) -> Result<Vec<TranslationMemoryEntry>> {
let rows = sqlx::query(&format!(
"{TRANSLATION_MEMORY_SELECT}\nWHERE source_hash = ?1 OR source_text = ?2"
))
.bind(hash_text(source_text))
.bind(source_text)
.fetch_all(&mut **transaction)
.await
.map_err(db_error)?;
rows.into_iter()
.map(row_to_entry)
.collect::<Result<Vec<_>>>()
.map(|entries| {
entries
.into_iter()
.filter(|entry| {
entry.source_text == source_text && entry.source_context == *source_context
})
.collect()
})
}
fn is_current_trusted(entry: &TranslationMemoryEntry) -> bool {
entry.trust_status == TranslationMemoryTrustStatus::Trusted
&& entry.superseded_by_record_id.is_none()
}
fn current_trusted_ids(entries: &[TranslationMemoryEntry]) -> Vec<String> {
let mut ids = entries
.iter()
.filter(|entry| is_current_trusted(entry))
.map(|entry| entry.record_id.clone())
.collect::<Vec<_>>();
ids.sort();
ids
}
fn group_key(entry: &TranslationMemoryEntry) -> Result<(String, String)> {
let context = serde_json::to_string(&entry.source_context)
.map_err(|error| Error::Serialization(error.to_string()))?;
Ok((entry.source_text.clone(), context))
}
fn group_entries(
entries: Vec<TranslationMemoryEntry>,
) -> std::collections::BTreeMap<(String, String), Vec<TranslationMemoryEntry>> {
let mut groups = std::collections::BTreeMap::new();
for entry in entries {
if let Ok(key) = group_key(&entry) {
groups.entry(key).or_insert_with(Vec::new).push(entry);
}
}
groups
}
fn conflict_groups(entries: Vec<TranslationMemoryEntry>) -> Vec<TranslationMemoryConflict> {
let mut conflicts = group_entries(entries)
.into_values()
.filter_map(|mut records| {
let trusted_record_ids = current_trusted_ids(&records);
if trusted_record_ids.len() <= 1 {
return None;
}
records.sort_by(|left, right| left.record_id.cmp(&right.record_id));
let first = records.first()?;
Some(TranslationMemoryConflict {
source_text: first.source_text.clone(),
source_hash: first.source_hash.clone(),
source_context: first.source_context.clone(),
source_context_hash: first.source_context_hash.clone(),
trusted_record_ids,
records,
})
})
.collect::<Vec<_>>();
conflicts.sort_by(|left, right| {
left.source_text
.cmp(&right.source_text)
.then_with(|| left.source_context_hash.cmp(&right.source_context_hash))
});
conflicts
}
#[allow(clippy::too_many_arguments)]
async fn insert_trust_event(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
action: &str,
winner_record_id: Option<&str>,
affected_record_ids: &[String],
identity: &TranslationMemoryEntry,
reviewer: &str,
reason: Option<&str>,
observed_unix_seconds: u64,
) -> Result<()> {
let affected_record_ids_json = serde_json::to_string(affected_record_ids)
.map_err(|error| Error::Serialization(error.to_string()))?;
let source_context_json = serde_json::to_string(&identity.source_context)
.map_err(|error| Error::Serialization(error.to_string()))?;
let event_id = trust_event_id(
action,
winner_record_id,
affected_record_ids,
observed_unix_seconds,
);
sqlx::query(
"INSERT INTO translation_memory_trust_events (
event_id, action, winner_record_id, affected_record_ids_json,
source_text, source_hash, source_context_json, source_context_hash,
reviewer, reason, observed_unix_seconds
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
)
.bind(event_id)
.bind(action)
.bind(winner_record_id)
.bind(affected_record_ids_json)
.bind(&identity.source_text)
.bind(&identity.source_hash)
.bind(source_context_json)
.bind(&identity.source_context_hash)
.bind(reviewer)
.bind(reason)
.bind(i64::try_from(observed_unix_seconds).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(db_error)?;
Ok(())
}
fn trust_event_id(
action: &str,
winner_record_id: Option<&str>,
affected_record_ids: &[String],
observed_unix_seconds: u64,
) -> String {
let mut value = format!(
"{action}|{}|{observed_unix_seconds}|{}",
winner_record_id.unwrap_or_default(),
affected_record_ids.join(",")
);
value.push('|');
value.push_str(
&SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.to_string(),
);
format!("tm-event-{}", blake3::hash(value.as_bytes()).to_hex())
}
fn trusted_conflict_error(record_ids: &[String]) -> Error {
Error::InvalidArgument(format!(
"trusted_conflict_requires_resolution: trusted_record_ids={}",
record_ids.join(",")
))
}
/// 由 active official release 根目录计算默认 TM 数据库路径。
pub fn translation_memory_repository_path(resource_root: &Path) -> PathBuf {
SqliteTranslationMemoryRepository::repository_path(resource_root)
}
/// 从 TextUnit 定位字段构建 TM 上下文。
#[allow(clippy::too_many_arguments)]
pub fn translation_memory_context(
destination: &str,
archive_entry: Option<&str>,
serialized_file: Option<&str>,
path_id: Option<i64>,
class_id: Option<i32>,
field_path: Option<&str>,
format: Option<&str>,
asset_name: Option<&str>,
text_source_kind: Option<&str>,
parser_context: &TranslationMemoryContext,
) -> TranslationMemoryContext {
let mut context = TranslationMemoryContext::new();
context.insert("destination".to_string(), destination.to_string());
insert_optional(&mut context, "archive_entry", archive_entry);
insert_optional(&mut context, "serialized_file", serialized_file);
if let Some(value) = path_id {
context.insert("path_id".to_string(), value.to_string());
}
if let Some(value) = class_id {
context.insert("class_id".to_string(), value.to_string());
}
insert_optional(&mut context, "field_path", field_path);
insert_optional(&mut context, "format", format);
insert_optional(&mut context, "asset_name", asset_name);
insert_optional(&mut context, "text_source_kind", text_source_kind);
for (key, value) in parser_context {
context.insert(format!("context.{key}"), value.clone());
}
context
}
fn insert_optional(context: &mut TranslationMemoryContext, key: &str, value: Option<&str>) {
if let Some(value) = value.filter(|value| !value.is_empty()) {
context.insert(key.to_string(), value.to_string());
}
}
fn entry_from_draft(draft: TranslationMemoryDraft) -> Result<TranslationMemoryEntry> {
let source_hash = hash_text(&draft.source_text);
let normalized_source_text = normalize_source_text(&draft.source_text);
let source_context_hash = hash_context(&draft.source_context)?;
let source_kind = draft.translation_source_kind;
let record_id = record_id(
&source_hash,
&source_context_hash,
&draft.translated_text,
&draft.official_release_id,
&source_kind,
);
let observed = draft.observed_unix_seconds;
Ok(TranslationMemoryEntry {
record_id,
source_text: draft.source_text,
source_hash,
normalized_source_text,
source_context: draft.source_context,
source_context_hash,
translated_text: draft.translated_text,
translation_source_kind: source_kind,
trust_status: TranslationMemoryTrustStatus::Candidate,
official_release_id: draft.official_release_id,
source_trace: draft.source_trace,
provider: draft.provider,
provider_run_id: draft.provider_run_id,
created_unix_seconds: observed,
updated_unix_seconds: observed,
trusted_unix_seconds: None,
trusted_by: None,
trusted_reason: None,
supersedes_record_id: None,
superseded_by_record_id: None,
})
}
fn validate_draft(draft: &TranslationMemoryDraft) -> Result<()> {
if draft.source_text.is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory source_text 不能为空".to_string(),
));
}
if draft.translated_text.trim().is_empty() {
return Err(Error::InvalidArgument(
"Translation Memory translated_text 不能为空".to_string(),
));
}
if draft.official_release_id.trim().is_empty()
|| draft.source_trace.official_release_id.trim().is_empty()
{
return Err(Error::InvalidArgument(
"Translation Memory official_release_id 不能为空".to_string(),
));
}
if draft.official_release_id != draft.source_trace.official_release_id {
return Err(Error::InvalidArgument(
"Translation Memory draft 的 release provenance 不一致".to_string(),
));
}
Ok(())
}
fn row_to_entry(row: sqlx::sqlite::SqliteRow) -> Result<TranslationMemoryEntry> {
let source_context = parse_json(row.try_get("source_context_json").map_err(db_error)?)?;
let source_trace = parse_json(row.try_get("source_trace_json").map_err(db_error)?)?;
Ok(TranslationMemoryEntry {
record_id: row.try_get("record_id").map_err(db_error)?,
source_text: row.try_get("source_text").map_err(db_error)?,
source_hash: row.try_get("source_hash").map_err(db_error)?,
normalized_source_text: row.try_get("normalized_source_text").map_err(db_error)?,
source_context,
source_context_hash: row.try_get("source_context_hash").map_err(db_error)?,
translated_text: row.try_get("translated_text").map_err(db_error)?,
translation_source_kind: parse_source_kind(
row.try_get::<String, _>("translation_source_kind")
.map_err(db_error)?
.as_str(),
)?,
trust_status: parse_trust_status(
row.try_get::<String, _>("trust_status")
.map_err(db_error)?
.as_str(),
)?,
official_release_id: row.try_get("official_release_id").map_err(db_error)?,
source_trace,
provider: row.try_get("provider").map_err(db_error)?,
provider_run_id: row.try_get("provider_run_id").map_err(db_error)?,
created_unix_seconds: i64_to_u64(
row.try_get("created_unix_seconds").map_err(db_error)?,
"created",
)?,
updated_unix_seconds: i64_to_u64(
row.try_get("updated_unix_seconds").map_err(db_error)?,
"updated",
)?,
trusted_unix_seconds: optional_i64_to_u64(
row.try_get("trusted_unix_seconds").map_err(db_error)?,
"trusted",
)?,
trusted_by: row.try_get("trusted_by").map_err(db_error)?,
trusted_reason: row.try_get("trusted_reason").map_err(db_error)?,
supersedes_record_id: row.try_get("supersedes_record_id").map_err(db_error)?,
superseded_by_record_id: row.try_get("superseded_by_record_id").map_err(db_error)?,
})
}
fn parse_json<T: DeserializeOwned>(value: String) -> Result<T> {
serde_json::from_str(&value).map_err(|error| Error::Serialization(error.to_string()))
}
fn parse_source_kind(value: &str) -> Result<TranslationMemorySourceKind> {
match value {
"provider" => Ok(TranslationMemorySourceKind::Provider),
"manual" => Ok(TranslationMemorySourceKind::Manual),
"imported" => Ok(TranslationMemorySourceKind::Imported),
_ => Err(Error::Serialization(format!(
"未知 Translation Memory source kind{value}"
))),
}
}
fn parse_trust_status(value: &str) -> Result<TranslationMemoryTrustStatus> {
match value {
"candidate" => Ok(TranslationMemoryTrustStatus::Candidate),
"trusted" => Ok(TranslationMemoryTrustStatus::Trusted),
"superseded" => Ok(TranslationMemoryTrustStatus::Superseded),
"rejected" => Ok(TranslationMemoryTrustStatus::Rejected),
_ => Err(Error::Serialization(format!(
"未知 Translation Memory trust status{value}"
))),
}
}
fn match_rank(value: &TranslationMemoryMatch) -> u8 {
match value.match_kind {
TranslationMemoryMatchKind::StrongExact if value.can_auto_reuse => 0,
TranslationMemoryMatchKind::TrustedConflict => 1,
TranslationMemoryMatchKind::CandidateExact => 2,
TranslationMemoryMatchKind::SourceOnly => 3,
TranslationMemoryMatchKind::StrongExact => 2,
}
}
fn hash_text(value: &str) -> String {
blake3::hash(value.as_bytes()).to_hex().to_string()
}
fn normalize_source_text(value: &str) -> String {
value.replace("\r\n", "\n").replace('\r', "\n")
}
fn hash_context(context: &TranslationMemoryContext) -> Result<String> {
let bytes =
serde_json::to_vec(context).map_err(|error| Error::Serialization(error.to_string()))?;
Ok(blake3::hash(&bytes).to_hex().to_string())
}
fn record_id(
source_hash: &str,
context_hash: &str,
translated_text: &str,
official_release_id: &str,
source_kind: &TranslationMemorySourceKind,
) -> String {
let mut key = Vec::new();
for value in [
source_hash,
context_hash,
translated_text,
official_release_id,
source_kind.as_str(),
] {
key.extend_from_slice(value.as_bytes());
key.push(0);
}
format!("tm-{}", blake3::hash(&key).to_hex())
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn i64_to_u64(value: i64, label: &str) -> Result<u64> {
u64::try_from(value)
.map_err(|_| Error::Serialization(format!("Translation Memory {label} 时间无效")))
}
fn optional_i64_to_u64(value: Option<i64>, label: &str) -> Result<Option<u64>> {
value.map(|value| i64_to_u64(value, label)).transpose()
}
fn db_error(error: sqlx::Error) -> Error {
Error::Other(error.into())
}
fn bat_infrastructure_absolute(path: &Path) -> Result<PathBuf> {
crate::path_security::lexical_absolute(path).map_err(Error::InvalidArgument)
}
fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
crate::path_security::ensure_safe_directory_path(parent, "Translation Memory 数据库")
.map_err(Error::InvalidArgument)
}
#[cfg(test)]
mod tests {
use super::*;
use bat_core::domain::TranslationMemorySourceTrace;
use sqlx::sqlite::SqlitePoolOptions;
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
let source_trace = TranslationMemorySourceTrace {
official_release_id: release.to_string(),
unit_id: Some(format!("{release}-unit")),
task_id: Some(format!("{release}-task")),
destination: Some("Bundles/story.bundle".to_string()),
archive_entry: None,
serialized_file: Some("CAB-story".to_string()),
path_id: Some(1),
class_id: Some(49),
field_path: Some("m_Text".to_string()),
format: Some("plain".to_string()),
asset_name: Some("Story".to_string()),
text_source_kind: Some("text_asset".to_string()),
source_url: Some("https://example.invalid/story".to_string()),
};
TranslationMemoryDraft {
source_text: source.to_string(),
source_context: translation_memory_context(
"Bundles/story.bundle",
None,
Some("CAB-story"),
Some(1),
Some(49),
Some("m_Text"),
Some("plain"),
Some("Story"),
Some("text_asset"),
&TranslationMemoryContext::new(),
),
translated_text: translated.to_string(),
translation_source_kind: TranslationMemorySourceKind::Provider,
official_release_id: release.to_string(),
source_trace,
provider: Some("mock".to_string()),
provider_run_id: Some(format!("mock:{release}")),
observed_unix_seconds: 1,
}
}
async fn raw_repository(
path: &std::path::Path,
create_if_missing: bool,
) -> SqliteTranslationMemoryRepository {
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", path.display()))
.unwrap()
.create_if_missing(create_if_missing);
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await
.unwrap();
SqliteTranslationMemoryRepository { pool }
}
async fn write_v1_fixture(
path: &std::path::Path,
with_migrations: bool,
records: Vec<(TranslationMemoryDraft, TranslationMemoryTrustStatus)>,
) -> Vec<String> {
let repository = raw_repository(path, true).await;
if with_migrations {
sqlx::query(
"CREATE TABLE schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)",
)
.execute(&repository.pool)
.await
.unwrap();
sqlx::query("INSERT INTO schema_migrations(component, version) VALUES (?1, 1)")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.execute(&repository.pool)
.await
.unwrap();
}
sqlx::query(
"CREATE TABLE translation_memory (
record_id TEXT PRIMARY KEY NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
normalized_source_text TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
translated_text TEXT NOT NULL,
translation_source_kind TEXT NOT NULL,
trust_status TEXT NOT NULL,
official_release_id TEXT NOT NULL,
source_trace_json TEXT NOT NULL,
provider TEXT,
provider_run_id TEXT,
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
trusted_unix_seconds INTEGER,
trusted_by TEXT,
trusted_reason TEXT,
supersedes_record_id TEXT,
superseded_by_record_id TEXT,
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(official_release_id) > 0),
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
)",
)
.execute(&repository.pool)
.await
.unwrap();
for statement in [
"CREATE INDEX idx_translation_memory_source_hash ON translation_memory(source_hash)",
"CREATE INDEX idx_translation_memory_normalized_source ON translation_memory(normalized_source_text)",
"CREATE INDEX idx_translation_memory_context ON translation_memory(source_hash, source_context_hash)",
] {
sqlx::query(statement)
.execute(&repository.pool)
.await
.unwrap();
}
let mut record_ids = Vec::new();
for (draft, status) in records {
let entry = entry_from_draft(draft).unwrap();
let source_context_json = serde_json::to_string(&entry.source_context).unwrap();
let source_trace_json = serde_json::to_string(&entry.source_trace).unwrap();
let trusted = status == TranslationMemoryTrustStatus::Trusted;
sqlx::query(
"INSERT INTO translation_memory (
record_id, source_text, source_hash, normalized_source_text,
source_context_json, source_context_hash, translated_text,
translation_source_kind, trust_status, official_release_id,
source_trace_json, provider, provider_run_id,
created_unix_seconds, updated_unix_seconds,
trusted_unix_seconds, trusted_by, trusted_reason,
supersedes_record_id, superseded_by_record_id
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
?11, ?12, ?13, ?14, ?14, ?15, ?16, ?17, ?18, ?19)",
)
.bind(&entry.record_id)
.bind(&entry.source_text)
.bind(&entry.source_hash)
.bind(&entry.normalized_source_text)
.bind(source_context_json)
.bind(&entry.source_context_hash)
.bind(&entry.translated_text)
.bind(entry.translation_source_kind.as_str())
.bind(status.as_str())
.bind(&entry.official_release_id)
.bind(source_trace_json)
.bind(&entry.provider)
.bind(&entry.provider_run_id)
.bind(i64::try_from(entry.created_unix_seconds).unwrap())
.bind(trusted.then_some(100_i64))
.bind(trusted.then_some("legacy-reviewer"))
.bind(trusted.then_some("legacy fixture"))
.bind(&entry.supersedes_record_id)
.bind(&entry.superseded_by_record_id)
.execute(&repository.pool)
.await
.unwrap();
record_ids.push(entry.record_id);
}
repository.pool.close().await;
record_ids
}
async fn force_trusted(repository: &SqliteTranslationMemoryRepository, record_id: &str) {
sqlx::query(
"UPDATE translation_memory
SET trust_status = 'trusted', trusted_unix_seconds = 100,
trusted_by = 'legacy-reviewer', trusted_reason = 'legacy fixture',
superseded_by_record_id = NULL
WHERE record_id = ?1",
)
.bind(record_id)
.execute(&repository.pool)
.await
.unwrap();
}
async fn trust_event_count(repository: &SqliteTranslationMemoryRepository) -> i64 {
sqlx::query_scalar("SELECT COUNT(*) FROM translation_memory_trust_events")
.fetch_one(&repository.pool)
.await
.unwrap()
}
async fn current_trusted_count(
repository: &SqliteTranslationMemoryRepository,
source_text: &str,
source_context: &TranslationMemoryContext,
) -> i64 {
let source_hash = hash_text(source_text);
let source_context_hash = hash_context(source_context).unwrap();
sqlx::query_scalar(
"SELECT COUNT(*) FROM translation_memory
WHERE source_hash = ?1 AND source_context_hash = ?2
AND source_text = ?3 AND source_context_json = ?4
AND trust_status = 'trusted' AND superseded_by_record_id IS NULL",
)
.bind(source_hash)
.bind(source_context_hash)
.bind(source_text)
.bind(serde_json::to_string(source_context).unwrap())
.fetch_one(&repository.pool)
.await
.unwrap()
}
#[tokio::test]
async fn sqlite_translation_memory_preserves_data_across_reopen_and_missing_row() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
let entry = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let trusted = repository
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
.await
.unwrap();
sqlx::query("DROP TABLE schema_migrations")
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let reopened = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap();
let restored = reopened.find(&trusted.record_id).await.unwrap();
assert_eq!(restored.translated_text, "你好");
assert_eq!(restored.trust_status, TranslationMemoryTrustStatus::Trusted);
assert_eq!(restored.source_trace.official_release_id, "release-1");
assert_eq!(
reopened.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
}
#[tokio::test]
async fn sqlite_translation_memory_migrates_independent_v1_fixture_with_historical_conflict() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("legacy-v1.sqlite");
let first = draft("release-1", "Hello", "你好");
let context = first.source_context.clone();
let ids = write_v1_fixture(
&path,
true,
vec![
(first, TranslationMemoryTrustStatus::Trusted),
(
draft("release-2", "Hello", "您好"),
TranslationMemoryTrustStatus::Trusted,
),
(
draft("release-3", "Hello", "你好呀"),
TranslationMemoryTrustStatus::Candidate,
),
],
)
.await;
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
let summary = repository.summary().await.unwrap();
assert_eq!(summary.schema_version, 2);
assert_eq!(summary.record_count, 3);
assert_eq!(summary.trusted_count, 2);
assert_eq!(summary.trusted_conflict_group_count, 1);
assert_eq!(summary.current_trusted_count, 0);
let matches = repository
.find_matches("Hello", &context, 10)
.await
.unwrap();
assert_eq!(matches.len(), 3);
assert!(matches.iter().all(|item| {
item.match_kind == TranslationMemoryMatchKind::TrustedConflict && !item.can_auto_reuse
}));
assert_eq!(
repository.find(&ids[0]).await.unwrap().translated_text,
"你好"
);
assert_eq!(trust_event_count(&repository).await, 0);
}
#[tokio::test]
async fn sqlite_translation_memory_migrates_v1_without_schema_migrations_row() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("legacy-v1-no-migrations.sqlite");
write_v1_fixture(
&path,
false,
vec![(
draft("release-1", "Hello", "你好"),
TranslationMemoryTrustStatus::Trusted,
)],
)
.await;
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
let version: i64 =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(version, 2);
assert_eq!(repository.summary().await.unwrap().record_count, 1);
}
#[tokio::test]
async fn sqlite_translation_memory_v1_migration_rolls_back_and_retries() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("legacy-v1-rollback.sqlite");
write_v1_fixture(
&path,
true,
vec![(
draft("release-1", "Hello", "你好"),
TranslationMemoryTrustStatus::Candidate,
)],
)
.await;
let repository = raw_repository(&path, false).await;
assert!(repository.init_schema_with_test_failure(1).await.is_err());
let governance_tables: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM sqlite_master
WHERE type = 'table' AND name = 'translation_memory_trust_events'",
)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(governance_tables, 0);
let version: i64 =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(version, 1);
repository.init_schema().await.unwrap();
assert_eq!(repository.summary().await.unwrap().schema_version, 2);
}
#[tokio::test]
async fn sqlite_translation_memory_concurrent_v1_open_migrates_once() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("legacy-v1-concurrent.sqlite");
write_v1_fixture(
&path,
true,
vec![(
draft("release-1", "Hello", "你好"),
TranslationMemoryTrustStatus::Candidate,
)],
)
.await;
let (left, right) = tokio::join!(
SqliteTranslationMemoryRepository::new(&path),
SqliteTranslationMemoryRepository::new(&path)
);
assert!(left.is_ok(), "left migration failed: {left:?}");
assert!(right.is_ok(), "right migration failed: {right:?}");
let repository = left.unwrap();
assert_eq!(repository.summary().await.unwrap().schema_version, 2);
assert_eq!(trust_event_count(&repository).await, 0);
}
#[tokio::test]
async fn sqlite_translation_memory_future_schema_is_read_only_failure() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let before = std::fs::read(&path).unwrap();
let wal_path = std::path::PathBuf::from(format!("{}-wal", path.display()));
let shm_path = std::path::PathBuf::from(format!("{}-shm", path.display()));
let wal_before = std::fs::read(&wal_path).ok();
let shm_before = std::fs::read(&shm_path).ok();
let error = SqliteTranslationMemoryRepository::new(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("不支持的 Translation Memory schema"));
assert_eq!(std::fs::read(&path).unwrap(), before);
assert_eq!(std::fs::read(&wal_path).ok(), wal_before);
assert_eq!(std::fs::read(&shm_path).ok(), shm_before);
}
#[tokio::test]
async fn sqlite_translation_memory_unknown_schema_fails_closed() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = raw_repository(&path, true).await;
sqlx::query(
"CREATE TABLE schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)",
)
.execute(&repository.pool)
.await
.unwrap();
sqlx::query("INSERT INTO schema_migrations(component, version) VALUES (?1, 1)")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.execute(&repository.pool)
.await
.unwrap();
sqlx::query("CREATE TABLE translation_memory (record_id TEXT PRIMARY KEY)")
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let before = std::fs::read(&path).unwrap();
let error = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("schema version 与实际结构不一致"));
assert_eq!(std::fs::read(&path).unwrap(), before);
}
#[tokio::test]
async fn sqlite_translation_memory_failed_new_schema_rolls_back_and_retries() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = raw_repository(&path, true).await;
assert!(repository.init_schema_with_test_failure(2).await.is_err());
let table_count: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'")
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(table_count, 0);
repository.init_schema().await.unwrap();
let version: i64 =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(version, 2);
}
#[tokio::test]
async fn sqlite_translation_memory_concurrent_new_open_has_one_current_schema() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let (left, right) = tokio::join!(
SqliteTranslationMemoryRepository::new(&path),
SqliteTranslationMemoryRepository::new(&path)
);
assert!(left.is_ok(), "left open failed: {left:?}");
assert!(right.is_ok(), "right open failed: {right:?}");
let repository = left.unwrap();
drop(right);
assert_eq!(
repository.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
repository.pool.close().await;
let reopened = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap();
assert_eq!(
reopened.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
}
#[tokio::test]
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(
temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE),
)
.await
.unwrap();
let entry = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
assert_eq!(repository.summary().await.unwrap().candidate_count, 1);
let trusted = repository
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
.await
.unwrap();
assert_eq!(trusted.trust_status, TranslationMemoryTrustStatus::Trusted);
let query = draft("release-2", "Hello", "ignored");
let matches = repository
.find_matches("Hello", &query.source_context, 10)
.await
.unwrap();
assert_eq!(matches.len(), 1);
assert_eq!(
matches[0].match_kind,
TranslationMemoryMatchKind::StrongExact
);
assert!(matches[0].can_auto_reuse);
assert_eq!(matches[0].entry.translated_text, "你好");
assert_eq!(matches[0].entry.official_release_id, "release-1");
}
#[tokio::test]
async fn rejects_future_schema_version() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("tm.sqlite");
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let error = SqliteTranslationMemoryRepository::new(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("不支持的 Translation Memory schema"));
}
#[tokio::test]
async fn different_context_is_candidate_only_and_provider_repeat_is_idempotent() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let first = draft("release-1", "Hello", "你好");
repository.upsert_candidate(first.clone()).await.unwrap();
repository.upsert_candidate(first).await.unwrap();
assert_eq!(repository.summary().await.unwrap().record_count, 1);
let mut different = draft("release-2", "Hello", "你好");
different
.source_context
.insert("field_path".to_string(), "m_Other".to_string());
let matches = repository
.find_matches("Hello", &different.source_context, 10)
.await
.unwrap();
assert_eq!(
matches[0].match_kind,
TranslationMemoryMatchKind::CandidateExact
);
assert!(!matches[0].can_auto_reuse);
}
#[tokio::test]
async fn translation_memory_confirm_is_idempotent_and_requires_explicit_supersede() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let trusted_candidate = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let trusted = repository
.confirm(
&trusted_candidate.record_id,
"reviewer-1",
Some("accepted".to_string()),
)
.await
.unwrap();
assert_eq!(trust_event_count(&repository).await, 1);
let idempotent = repository
.confirm(
&trusted.record_id,
"reviewer-2",
Some("duplicate review".to_string()),
)
.await
.unwrap();
assert_eq!(idempotent.trusted_by.as_deref(), Some("reviewer-1"));
assert_eq!(trust_event_count(&repository).await, 1);
let equivalent = repository
.upsert_candidate(draft("release-2", "Hello", "你好"))
.await
.unwrap();
let duplicate_error = repository
.confirm(&equivalent.record_id, "reviewer", Some("same".to_string()))
.await
.unwrap_err();
assert!(duplicate_error
.to_string()
.contains("trusted_translation_already_exists"));
let replacement = repository
.upsert_candidate(draft("release-3", "Hello", "您好"))
.await
.unwrap();
let required_error = repository
.confirm(
&replacement.record_id,
"reviewer",
Some("replace".to_string()),
)
.await
.unwrap_err();
assert!(required_error
.to_string()
.contains("explicit_supersede_required"));
let stale_error = repository
.confirm_with_supersede(
&replacement.record_id,
"reviewer",
Some("replace".to_string()),
Some("wrong-record"),
)
.await
.unwrap_err();
assert!(stale_error.to_string().contains("stale_supersede_target"));
let superseded = repository
.confirm_with_supersede(
&replacement.record_id,
"reviewer",
Some("approved replacement".to_string()),
Some(&trusted.record_id),
)
.await
.unwrap();
assert_eq!(
superseded.supersedes_record_id.as_deref(),
Some(trusted.record_id.as_str())
);
assert_eq!(
repository
.find(&trusted.record_id)
.await
.unwrap()
.superseded_by_record_id
.as_deref(),
Some(replacement.record_id.as_str())
);
assert_eq!(trust_event_count(&repository).await, 2);
let summary = repository.summary().await.unwrap();
assert_eq!(summary.trusted_conflict_group_count, 0);
assert_eq!(summary.current_trusted_count, 1);
}
#[tokio::test]
async fn translation_memory_historical_conflict_rejects_confirm_and_supports_existing_winner_resolution(
) {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let first = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let second = repository
.upsert_candidate(draft("release-2", "Hello", "您好"))
.await
.unwrap();
let candidate = repository
.upsert_candidate(draft("release-3", "Hello", "你好呀"))
.await
.unwrap();
force_trusted(&repository, &first.record_id).await;
force_trusted(&repository, &second.record_id).await;
let conflict_error = repository
.confirm(
&candidate.record_id,
"reviewer",
Some("resolve".to_string()),
)
.await
.unwrap_err();
assert!(conflict_error
.to_string()
.contains("trusted_conflict_requires_resolution"));
let conflicts = repository.list_conflicts(10).await.unwrap();
assert_eq!(conflicts.len(), 1);
assert_eq!(conflicts[0].trusted_record_ids.len(), 2);
assert_eq!(conflicts[0].records.len(), 3);
let winner = repository
.resolve_conflict(
&first.record_id,
&[second.record_id.clone(), first.record_id.clone()],
"resolver",
"keep established translation",
)
.await
.unwrap();
assert_eq!(winner.record_id, first.record_id);
assert_eq!(
winner.supersedes_record_id.as_deref(),
Some(second.record_id.as_str())
);
let loser = repository.find(&second.record_id).await.unwrap();
assert_eq!(
loser.superseded_by_record_id.as_deref(),
Some(first.record_id.as_str())
);
assert_eq!(
repository
.find(&candidate.record_id)
.await
.unwrap()
.trust_status,
TranslationMemoryTrustStatus::Candidate
);
let stale = repository
.resolve_conflict(
&first.record_id,
&[first.record_id.clone(), second.record_id.clone()],
"resolver",
"stale retry",
)
.await
.unwrap_err();
assert!(stale.to_string().contains("conflict_snapshot_stale"));
assert_eq!(trust_event_count(&repository).await, 1);
let event = sqlx::query(
"SELECT action, affected_record_ids_json
FROM translation_memory_trust_events",
)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(event.get::<String, _>("action"), "resolve_conflict");
let affected: Vec<String> =
serde_json::from_str(&event.get::<String, _>("affected_record_ids_json")).unwrap();
let mut expected_affected = vec![first.record_id.clone(), second.record_id.clone()];
expected_affected.sort();
assert_eq!(affected, expected_affected);
}
#[tokio::test]
async fn translation_memory_conflict_resolution_with_candidate_winner_has_no_fake_chain() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let first = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let second = repository
.upsert_candidate(draft("release-2", "Hello", "您好"))
.await
.unwrap();
let candidate = repository
.upsert_candidate(draft("release-3", "Hello", "你好呀"))
.await
.unwrap();
force_trusted(&repository, &first.record_id).await;
force_trusted(&repository, &second.record_id).await;
let winner = repository
.resolve_conflict(
&candidate.record_id,
&[first.record_id.clone(), second.record_id.clone()],
"resolver",
"select candidate",
)
.await
.unwrap();
assert_eq!(winner.trust_status, TranslationMemoryTrustStatus::Trusted);
assert!(winner.supersedes_record_id.is_none());
for loser in [&first.record_id, &second.record_id] {
assert_eq!(
repository
.find(loser)
.await
.unwrap()
.superseded_by_record_id
.as_deref(),
Some(candidate.record_id.as_str())
);
}
let event_ids: Vec<String> = serde_json::from_str(
&sqlx::query_scalar::<_, String>(
"SELECT affected_record_ids_json
FROM translation_memory_trust_events
WHERE action = 'resolve_conflict'",
)
.fetch_one(&repository.pool)
.await
.unwrap(),
)
.unwrap();
let mut expected_event_ids = vec![
first.record_id.clone(),
second.record_id.clone(),
candidate.record_id.clone(),
];
expected_event_ids.sort();
assert_eq!(event_ids, expected_event_ids);
assert_eq!(repository.summary().await.unwrap().current_trusted_count, 1);
}
#[tokio::test]
async fn translation_memory_concurrent_confirms_and_supersedes_keep_one_current_trusted() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let first = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let second = repository
.upsert_candidate(draft("release-2", "Hello", "您好"))
.await
.unwrap();
let (left, right) = tokio::join!(
repository.confirm(&first.record_id, "reviewer-a", Some("a".to_string())),
repository.confirm(&second.record_id, "reviewer-b", Some("b".to_string()))
);
assert_eq!(left.is_ok() as u8 + right.is_ok() as u8, 1);
let errors = [left.as_ref().err(), right.as_ref().err()];
assert!(errors.iter().flatten().any(|error| {
error.to_string().contains("explicit_supersede_required")
|| error
.to_string()
.contains("trusted_translation_already_exists")
}));
let current_id = if left.is_ok() {
first.record_id.clone()
} else {
second.record_id.clone()
};
let third = repository
.upsert_candidate(draft("release-3", "Hello", "你好呀"))
.await
.unwrap();
let fourth = repository
.upsert_candidate(draft("release-4", "Hello", "你好喔"))
.await
.unwrap();
let (left, right) = tokio::join!(
repository.confirm_with_supersede(
&third.record_id,
"reviewer-c",
Some("replace".to_string()),
Some(&current_id),
),
repository.confirm_with_supersede(
&fourth.record_id,
"reviewer-d",
Some("replace".to_string()),
Some(&current_id),
)
);
assert_eq!(left.is_ok() as u8 + right.is_ok() as u8, 1);
assert!(left.is_ok() || right.is_ok());
assert!(left.as_ref().err().is_none_or(|error| {
error.to_string().contains("stale_supersede_target")
|| error
.to_string()
.contains("trusted_translation_already_exists")
}));
assert!(right.as_ref().err().is_none_or(|error| {
error.to_string().contains("stale_supersede_target")
|| error
.to_string()
.contains("trusted_translation_already_exists")
}));
let identity = draft("release-4", "Hello", "ignored");
assert_eq!(
current_trusted_count(&repository, "Hello", &identity.source_context).await,
1
);
}
#[tokio::test]
async fn rejects_empty_translation_candidates() {
let temp = tempfile::TempDir::new().unwrap();
let repository = SqliteTranslationMemoryRepository::new(temp.path().join("tm.sqlite"))
.await
.unwrap();
let error = repository
.upsert_candidate(draft("release-1", "Hello", " \n"))
.await
.unwrap_err();
assert!(error.to_string().contains("translated_text"));
assert_eq!(repository.summary().await.unwrap().record_count, 0);
}
}