fix(sqlite): 收口长期状态数据库版本化迁移契约
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s

This commit is contained in:
2026-09-16 21:02:22 +08:00
parent 99355effe4
commit 7f7d757f15
10 changed files with 2487 additions and 421 deletions
+546 -89
View File
@@ -1,6 +1,9 @@
//! 跨 official release 的 Translation Memory SQLite 仓储。
use crate::path_security::{set_file_mode, STATE_FILE_MODE};
use crate::sqlite_migration::{
self, ExpectedColumn, ExpectedIndex, ExpectedTable, SqliteSchemaSnapshot,
};
use async_trait::async_trait;
use bat_core::domain::{
TranslationMemoryContext, TranslationMemoryDraft, TranslationMemoryEntry,
@@ -10,7 +13,7 @@ use bat_core::domain::{
use bat_core::repositories::TranslationMemoryRepository;
use bat_core::{Error, Result};
use serde::de::DeserializeOwned;
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode};
use sqlx::{Row, SqlitePool};
use std::fs;
use std::path::{Path, PathBuf};
@@ -85,14 +88,24 @@ impl SqliteTranslationMemoryRepository {
return Err(Error::NotFound(absolute.display().to_string()));
}
if let Ok(metadata) = fs::symlink_metadata(&absolute) {
if metadata.len() > 0 {
let snapshot = sqlite_migration::read_only_preflight(
&absolute,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
classify_translation_memory_schema(&snapshot)?;
}
}
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", absolute.display()))
.map_err(|error| Error::Other(error.into()))?
.create_if_missing(create_if_missing)
.journal_mode(SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30));
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
let pool = sqlite_migration::connect_writable_pool(options)
.await
.map_err(db_error)?;
set_file_mode(&absolute, STATE_FILE_MODE, "Translation Memory 数据库")
@@ -103,98 +116,93 @@ impl SqliteTranslationMemoryRepository {
}
async fn init_schema(&self) -> Result<()> {
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)
"#,
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS translation_memory (
record_id TEXT PRIMARY KEY NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
normalized_source_text TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
translated_text TEXT NOT NULL,
translation_source_kind TEXT NOT NULL,
trust_status TEXT NOT NULL,
official_release_id TEXT NOT NULL,
source_trace_json TEXT NOT NULL,
provider TEXT,
provider_run_id TEXT,
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
trusted_unix_seconds INTEGER,
trusted_by TEXT,
trusted_reason TEXT,
supersedes_record_id TEXT,
superseded_by_record_id TEXT,
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(official_release_id) > 0),
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
)
"#,
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_source_hash \
ON translation_memory(source_hash)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_normalized_source \
ON translation_memory(normalized_source_text)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_translation_memory_context \
ON translation_memory(source_hash, source_context_hash)",
)
.execute(&self.pool)
.await
.map_err(db_error)?;
self.init_schema_with_failure(None).await
}
let current: Option<i64> =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_optional(&self.pool)
#[cfg(test)]
async fn init_schema_with_test_failure(&self, fail_after_step: usize) -> Result<()> {
self.init_schema_with_failure(Some(fail_after_step)).await
}
async fn init_schema_with_failure(&self, fail_after_step: Option<usize>) -> Result<()> {
let mut transaction = sqlite_migration::begin_immediate(&self.pool)
.await
.map_err(db_error)?;
let result = self
.migrate_in_transaction(&mut transaction, fail_after_step)
.await;
match result {
Ok(()) => transaction.commit().await.map_err(db_error),
Err(error) => {
let _ = transaction.rollback().await;
Err(error)
}
}
}
async fn migrate_in_transaction(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
fail_after_step: Option<usize>,
) -> Result<()> {
let snapshot = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.await
.map_err(db_error)?;
match classify_translation_memory_schema(&snapshot)? {
TranslationMemorySchemaState::Empty => {
create_translation_memory_schema(transaction, fail_after_step).await?;
sqlite_migration::write_component_version(
transaction,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
TRANSLATION_MEMORY_SCHEMA_VERSION,
)
.await
.map_err(db_error)?;
if current.is_some_and(|version| version > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION)) {
return Err(Error::InvalidArgument(format!(
"不支持的 Translation Memory schema 版本:{}",
current.unwrap_or_default()
)));
}
TranslationMemorySchemaState::Version(version)
if version == TRANSLATION_MEMORY_SCHEMA_VERSION =>
{
if snapshot.component_version.is_none() {
if !snapshot
.tables
.contains_key(sqlite_migration::SCHEMA_MIGRATIONS_TABLE)
{
sqlite_migration::create_schema_migrations_table(transaction)
.await
.map_err(db_error)?;
}
sqlite_migration::write_component_version(
transaction,
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
TRANSLATION_MEMORY_SCHEMA_VERSION,
)
.await
.map_err(db_error)?;
}
}
TranslationMemorySchemaState::Version(version) => {
return Err(invalid_translation_memory_schema(format!(
"内部不支持的迁移起点 {version}"
)));
}
}
sqlx::query(
r#"
INSERT INTO schema_migrations(component, version)
VALUES (?1, ?2)
ON CONFLICT(component) DO UPDATE SET version = excluded.version
"#,
let final_snapshot = sqlite_migration::snapshot_connection(
transaction.as_mut(),
TRANSLATION_MEMORY_SCHEMA_COMPONENT,
)
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
.execute(&self.pool)
.await
.map_err(db_error)?;
if final_snapshot.component_version != Some(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
|| !matches_translation_memory_fingerprint(&final_snapshot)
{
return Err(invalid_translation_memory_schema(
"migration 结果与当前 schema fingerprint 不一致".to_string(),
));
}
Ok(())
}
@@ -219,6 +227,297 @@ impl SqliteTranslationMemoryRepository {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TranslationMemorySchemaState {
Empty,
Version(u32),
}
fn classify_translation_memory_schema(
snapshot: &SqliteSchemaSnapshot,
) -> Result<TranslationMemorySchemaState> {
if snapshot.is_empty() {
return Ok(TranslationMemorySchemaState::Empty);
}
if let Some(observed) = snapshot.component_version {
if observed > i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) {
return Err(invalid_translation_memory_schema(format!(
"不支持的 Translation Memory schema 版本:observed={observed}, supported={TRANSLATION_MEMORY_SCHEMA_VERSION}"
)));
}
if observed < 1 {
return Err(invalid_translation_memory_schema(format!(
"schema version 无效:observed={observed}, supported=1..={TRANSLATION_MEMORY_SCHEMA_VERSION}"
)));
}
}
if (matches_translation_memory_fingerprint(snapshot)
|| (snapshot.component_version.is_none()
&& matches_translation_memory_component_fingerprint(snapshot)))
&& snapshot
.component_version
.is_none_or(|observed| observed == i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION))
{
return Ok(TranslationMemorySchemaState::Version(
TRANSLATION_MEMORY_SCHEMA_VERSION,
));
}
match snapshot.component_version {
Some(observed) => Err(invalid_translation_memory_schema(format!(
"schema version 与实际结构不一致:observed={observed}, supported={TRANSLATION_MEMORY_SCHEMA_VERSION}"
))),
None => Err(invalid_translation_memory_schema(
"未识别的 legacy schema,拒绝静默修复".to_string(),
)),
}
}
fn invalid_translation_memory_schema(detail: String) -> Error {
Error::InvalidArgument(format!("Translation Memory schema 无效:{detail}"))
}
fn matches_translation_memory_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, true)
}
fn matches_translation_memory_component_fingerprint(snapshot: &SqliteSchemaSnapshot) -> bool {
matches_translation_memory_fingerprint_with_migrations(snapshot, false)
}
fn matches_translation_memory_fingerprint_with_migrations(
snapshot: &SqliteSchemaSnapshot,
include_migrations: bool,
) -> bool {
let tables = [ExpectedTable {
name: "translation_memory",
columns: &TRANSLATION_MEMORY_COLUMNS,
}];
let indexes = [
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_source_hash",
columns: &["source_hash"],
},
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_normalized_source",
columns: &["normalized_source_text"],
},
ExpectedIndex {
table: "translation_memory",
name: "idx_translation_memory_context",
columns: &["source_hash", "source_context_hash"],
},
];
if include_migrations {
let tables = [sqlite_migration::schema_migrations_table(), tables[0]];
return sqlite_migration::matches_fingerprint(snapshot, &tables, &indexes);
}
sqlite_migration::matches_fingerprint(snapshot, &tables, &indexes)
}
const TRANSLATION_MEMORY_COLUMNS: [ExpectedColumn<'static>; 20] = [
ExpectedColumn {
name: "record_id",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: true,
},
ExpectedColumn {
name: "source_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "normalized_source_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_context_hash",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "translated_text",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "translation_source_kind",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trust_status",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "official_release_id",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "source_trace_json",
data_type: "TEXT",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "provider",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "provider_run_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "created_unix_seconds",
data_type: "INTEGER",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "updated_unix_seconds",
data_type: "INTEGER",
not_null: true,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_unix_seconds",
data_type: "INTEGER",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_by",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "trusted_reason",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "supersedes_record_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
ExpectedColumn {
name: "superseded_by_record_id",
data_type: "TEXT",
not_null: false,
default_value: None,
primary_key: false,
},
];
async fn create_translation_memory_schema(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
fail_after_step: Option<usize>,
) -> Result<()> {
let steps = [
"CREATE TABLE schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)",
"CREATE TABLE translation_memory (
record_id TEXT PRIMARY KEY NOT NULL,
source_text TEXT NOT NULL,
source_hash TEXT NOT NULL,
normalized_source_text TEXT NOT NULL,
source_context_json TEXT NOT NULL,
source_context_hash TEXT NOT NULL,
translated_text TEXT NOT NULL,
translation_source_kind TEXT NOT NULL,
trust_status TEXT NOT NULL,
official_release_id TEXT NOT NULL,
source_trace_json TEXT NOT NULL,
provider TEXT,
provider_run_id TEXT,
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
trusted_unix_seconds INTEGER,
trusted_by TEXT,
trusted_reason TEXT,
supersedes_record_id TEXT,
superseded_by_record_id TEXT,
CHECK (length(source_text) > 0),
CHECK (length(source_hash) > 0),
CHECK (length(source_context_hash) > 0),
CHECK (length(official_release_id) > 0),
CHECK (translation_source_kind IN ('provider', 'manual', 'imported')),
CHECK (trust_status IN ('candidate', 'trusted', 'superseded', 'rejected'))
)",
"CREATE INDEX idx_translation_memory_source_hash
ON translation_memory(source_hash)",
"CREATE INDEX idx_translation_memory_normalized_source
ON translation_memory(normalized_source_text)",
"CREATE INDEX idx_translation_memory_context
ON translation_memory(source_hash, source_context_hash)",
];
for (index, statement) in steps.iter().enumerate() {
sqlx::query(statement)
.execute(&mut **transaction)
.await
.map_err(db_error)?;
if fail_after_step == Some(index + 1) {
return Err(Error::Other(anyhow::anyhow!(
"Translation Memory migration failed after step {}",
index + 1
)));
}
}
Ok(())
}
#[async_trait]
impl TranslationMemoryRepository for SqliteTranslationMemoryRepository {
async fn upsert_candidate(
@@ -721,6 +1020,7 @@ fn ensure_safe_tm_parent(parent: &Path) -> Result<()> {
mod tests {
use super::*;
use bat_core::domain::TranslationMemorySourceTrace;
use sqlx::sqlite::SqlitePoolOptions;
fn draft(release: &str, source: &str, translated: &str) -> TranslationMemoryDraft {
let source_trace = TranslationMemorySourceTrace {
@@ -762,6 +1062,163 @@ mod tests {
}
}
async fn raw_repository(
path: &std::path::Path,
create_if_missing: bool,
) -> SqliteTranslationMemoryRepository {
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", path.display()))
.unwrap()
.create_if_missing(create_if_missing);
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await
.unwrap();
SqliteTranslationMemoryRepository { pool }
}
#[tokio::test]
async fn sqlite_translation_memory_preserves_data_across_reopen_and_missing_row() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
let entry = repository
.upsert_candidate(draft("release-1", "Hello", "你好"))
.await
.unwrap();
let trusted = repository
.confirm(&entry.record_id, "reviewer", Some("accepted".to_string()))
.await
.unwrap();
sqlx::query("DROP TABLE schema_migrations")
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let reopened = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap();
let restored = reopened.find(&trusted.record_id).await.unwrap();
assert_eq!(restored.translated_text, "你好");
assert_eq!(restored.trust_status, TranslationMemoryTrustStatus::Trusted);
assert_eq!(restored.source_trace.official_release_id, "release-1");
assert_eq!(
reopened.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
}
#[tokio::test]
async fn sqlite_translation_memory_future_schema_is_read_only_failure() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = SqliteTranslationMemoryRepository::new(&path).await.unwrap();
sqlx::query("UPDATE schema_migrations SET version = ?2 WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.bind(i64::from(TRANSLATION_MEMORY_SCHEMA_VERSION) + 1)
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let before = std::fs::read(&path).unwrap();
let wal_path = std::path::PathBuf::from(format!("{}-wal", path.display()));
let shm_path = std::path::PathBuf::from(format!("{}-shm", path.display()));
let wal_before = std::fs::read(&wal_path).ok();
let shm_before = std::fs::read(&shm_path).ok();
let error = SqliteTranslationMemoryRepository::new(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("不支持的 Translation Memory schema"));
assert_eq!(std::fs::read(&path).unwrap(), before);
assert_eq!(std::fs::read(&wal_path).ok(), wal_before);
assert_eq!(std::fs::read(&shm_path).ok(), shm_before);
}
#[tokio::test]
async fn sqlite_translation_memory_unknown_schema_fails_closed() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = raw_repository(&path, true).await;
sqlx::query(
"CREATE TABLE schema_migrations (
component TEXT PRIMARY KEY NOT NULL,
version INTEGER NOT NULL CHECK(version >= 1)
)",
)
.execute(&repository.pool)
.await
.unwrap();
sqlx::query("INSERT INTO schema_migrations(component, version) VALUES (?1, 1)")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.execute(&repository.pool)
.await
.unwrap();
sqlx::query("CREATE TABLE translation_memory (record_id TEXT PRIMARY KEY)")
.execute(&repository.pool)
.await
.unwrap();
repository.pool.close().await;
let before = std::fs::read(&path).unwrap();
let error = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap_err();
assert!(error
.to_string()
.contains("schema version 与实际结构不一致"));
assert_eq!(std::fs::read(&path).unwrap(), before);
}
#[tokio::test]
async fn sqlite_translation_memory_failed_new_schema_rolls_back_and_retries() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let repository = raw_repository(&path, true).await;
assert!(repository.init_schema_with_test_failure(2).await.is_err());
let table_count: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'")
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(table_count, 0);
repository.init_schema().await.unwrap();
let version: i64 =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
.bind(TRANSLATION_MEMORY_SCHEMA_COMPONENT)
.fetch_one(&repository.pool)
.await
.unwrap();
assert_eq!(version, 1);
}
#[tokio::test]
async fn sqlite_translation_memory_concurrent_new_open_has_one_current_schema() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(TRANSLATION_MEMORY_REPOSITORY_FILE);
let (left, right) = tokio::join!(
SqliteTranslationMemoryRepository::new(&path),
SqliteTranslationMemoryRepository::new(&path)
);
assert!(left.is_ok(), "left open failed: {left:?}");
assert!(right.is_ok(), "right open failed: {right:?}");
let repository = left.unwrap();
drop(right);
assert_eq!(
repository.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
repository.pool.close().await;
let reopened = SqliteTranslationMemoryRepository::open(&path)
.await
.unwrap();
assert_eq!(
reopened.summary().await.unwrap().schema_version,
TRANSLATION_MEMORY_SCHEMA_VERSION
);
}
#[tokio::test]
async fn initializes_schema_and_reuses_trusted_entry_across_releases() {
let temp = tempfile::TempDir::new().unwrap();