feat(i18n): 接入翻译 provider worker
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s

Closes #44
This commit is contained in:
2026-08-30 21:13:31 +08:00
parent 7f465523e1
commit f441f1810e
29 changed files with 3815 additions and 130 deletions
+541 -7
View File
@@ -23,14 +23,16 @@ use bat_infrastructure::{
OfficialUpdateService, OfficialUpdateSnapshot, OfficialUpdateStatus,
OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState, PatchApplyKind,
PatchApplyParams, PatchApplyReport, ReleaseFlowStatusCode, RepackReport,
SqliteResourceRepository, SqliteTranslationTaskRepository, TranslationTaskStatus,
UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams,
UnityFsTextAssetPatchParams, CROWDIN_TEXTUNIT_QUEUE_FILE, LOCALIZED_CURRENT_LINK,
SqliteResourceRepository, SqliteTranslationTaskRepository, TranslationProviderKind,
TranslationTaskStatus, TranslationWorkerConfig, UnityFsFieldPatchParams, UnityFsPatchReport,
UnityFsStringFieldPatchParams, UnityFsTextAssetPatchParams, CROWDIN_TEXTUNIT_QUEUE_FILE,
DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS,
DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF, LOCALIZED_CURRENT_LINK,
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING,
LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING_LABEL, LOCALIZED_VERSIONS_DIR,
LOCALIZED_VERSION_STATE_FILE, MAX_DOWNLOAD_CONCURRENCY, MIN_DOWNLOAD_CONCURRENCY,
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE,
PRIVATE_FILE_MODE,
LOCALIZED_VERSION_STATE_FILE, MAX_DOWNLOAD_CONCURRENCY, MAX_TRANSLATION_CONCURRENCY,
MIN_DOWNLOAD_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, OFFICIAL_PARSE_CACHE_FILE,
OFFICIAL_TEXTUNIT_INDEX_FILE, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE, PRIVATE_FILE_MODE,
};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
@@ -85,6 +87,7 @@ use workflow_commands::{
run_parse_clear_cache, run_parse_once, run_publish_localized, run_repack, run_translate_once,
run_translation_get, run_translation_proofread, run_translation_set,
run_translation_task_update, run_translation_unset, run_translation_validate,
run_translation_worker,
};
const EXIT_ERROR: i32 = 1;
@@ -229,6 +232,10 @@ fn run() -> anyhow::Result<i32> {
run_translation_proofread(&options)?;
Ok(0)
}
CliCommand::TranslationWorker => {
run_repeated_workflow(&options, "translation-worker", run_translation_worker)?;
Ok(0)
}
CliCommand::Repack => {
run_repack(&options)?;
Ok(0)
@@ -378,6 +385,15 @@ struct CliOptions {
translation_text_file: Option<PathBuf>,
translation_failure_reason: Option<String>,
translation_provider_run_id: Option<String>,
translation_provider: Option<String>,
translation_fixture: Option<PathBuf>,
worker_concurrency: usize,
worker_max_attempts: u32,
worker_lease_seconds: u64,
worker_retry_backoff: Duration,
worker_max_tasks: Option<usize>,
worker_id: Option<String>,
translation_worker_option_explicit: bool,
localized_release_id: Option<String>,
repack_spec: Option<PathBuf>,
schedule_group: Option<String>,
@@ -463,6 +479,15 @@ impl Default for CliOptions {
translation_text_file: None,
translation_failure_reason: None,
translation_provider_run_id: None,
translation_provider: None,
translation_fixture: None,
worker_concurrency: DEFAULT_TRANSLATION_CONCURRENCY,
worker_max_attempts: DEFAULT_TRANSLATION_MAX_ATTEMPTS,
worker_lease_seconds: DEFAULT_TRANSLATION_LEASE_SECONDS,
worker_retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
worker_max_tasks: None,
worker_id: None,
translation_worker_option_explicit: false,
localized_release_id: None,
repack_spec: None,
schedule_group: None,
@@ -546,6 +571,7 @@ enum CliCommand {
TranslationGet,
TranslationUnset,
TranslationTaskUpdate,
TranslationWorker,
TranslationProofread,
Repack,
PublishLocalized,
@@ -1025,6 +1051,7 @@ const RPC_METHOD_TRANSLATION_TASKS: &str = "translation.tasks";
const RPC_METHOD_TRANSLATION_HANDOFF: &str = "translation.handoff";
const RPC_METHOD_TRANSLATION_TASK_UPDATE: &str = "translation.task.update";
const RPC_METHOD_TRANSLATION_PROOFREAD: &str = "translation.proofread";
const RPC_METHOD_TRANSLATION_WORKER_RUN: &str = "translation.worker.run";
const RPC_METHOD_LOCALIZED_STATUS: &str = "localized.status";
const RPC_METHOD_CATALOG_STATUS: &str = "catalog.status";
const RPC_METHOD_CATALOG_VERSIONS: &str = "catalog.versions";
@@ -2027,6 +2054,13 @@ fn dispatch_rpc_method(
mark_localized_manual_proofreading_report(state_dir, &tasks.base_config)
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from)),
),
RPC_METHOD_TRANSLATION_WORKER_RUN => {
let config = match rpc_translation_worker_config(request.params.as_ref()) {
Ok(config) => config,
Err(error) => return rpc_envelope_error(request_id, error),
};
enqueue_translation_worker_envelope(tasks, config, request_id)
}
RPC_METHOD_LOCALIZED_STATUS => rpc_envelope_from_result(
request_id,
"localized.status",
@@ -2211,7 +2245,9 @@ fn enqueue_task_envelope(
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
let job = TaskJob {
id: task_id.clone(),
kind,
config,
translation_worker_config: None,
cancel,
};
if tasks.queue.send(job).is_err() {
@@ -2237,6 +2273,50 @@ fn enqueue_task_envelope(
)
}
fn enqueue_translation_worker_envelope(
tasks: &DaemonTaskContext,
worker_config: TranslationWorkerConfig,
request_id: String,
) -> RpcEnvelope {
let config = TaskKind::TranslationWorker.build_config(&tasks.base_config, false);
let task_id = tasks.registry.create(TaskKind::TranslationWorker);
let cancel = tasks
.registry
.cancel_flag(&task_id)
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
let job = TaskJob {
id: task_id.clone(),
kind: TaskKind::TranslationWorker,
config,
translation_worker_config: Some(worker_config.clone()),
cancel,
};
if tasks.queue.send(job).is_err() {
tasks.registry.update(&task_id, |record| {
record.status = "failed";
record.finished_at = Some(unix_seconds_now());
record.error = Some(ApiError::new(
ErrorCode::INTERNAL,
"task.enqueue",
"任务执行器不可用",
));
});
return rpc_envelope_error(
request_id,
ApiError::new(ErrorCode::INTERNAL, "task.enqueue", "任务执行器不可用"),
);
}
rpc_envelope_ok(
request_id,
"accepted",
serde_json::json!({
"task_id": task_id,
"kind": RPC_METHOD_TRANSLATION_WORKER_RUN,
"worker": worker_config,
}),
)
}
fn rpc_ack_value(
command: &'static str,
message: &'static str,
@@ -3070,6 +3150,207 @@ fn rpc_struct_params<T: DeserializeOwned>(
})
}
fn rpc_translation_worker_config(
params: Option<&serde_json::Value>,
) -> Result<TranslationWorkerConfig, ApiError> {
let empty = serde_json::json!({});
let params = params.unwrap_or(&empty);
if !params.is_object() {
return Err(ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
"params 必须是 JSON object",
));
}
let provider = rpc_translation_worker_string_param(
params,
&["provider", "translation_provider"],
"provider",
)?
.unwrap_or_else(|| TranslationProviderKind::Mock.as_str().to_string());
let provider = TranslationProviderKind::parse(&provider).ok_or_else(|| {
ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("不支持的 translation provider{provider}"),
)
})?;
let concurrency = rpc_translation_worker_usize_param(
params,
&[
"concurrency",
"worker_concurrency",
"translation_concurrency",
],
"concurrency",
)?
.unwrap_or(DEFAULT_TRANSLATION_CONCURRENCY);
let max_attempts = rpc_translation_worker_u32_param(
params,
&[
"max_attempts",
"worker_max_attempts",
"translation_max_attempts",
],
"max_attempts",
)?
.unwrap_or(DEFAULT_TRANSLATION_MAX_ATTEMPTS);
let lease_seconds = rpc_translation_worker_u64_param(
params,
&[
"lease_seconds",
"worker_lease_seconds",
"translation_lease_seconds",
],
"lease_seconds",
)?
.unwrap_or(DEFAULT_TRANSLATION_LEASE_SECONDS);
let retry_backoff_seconds = rpc_translation_worker_u64_param(
params,
&[
"retry_backoff_seconds",
"worker_retry_backoff_seconds",
"translation_retry_backoff_seconds",
],
"retry_backoff_seconds",
)?
.unwrap_or(DEFAULT_TRANSLATION_RETRY_BACKOFF.as_secs());
let max_tasks = rpc_translation_worker_usize_param(
params,
&["max_tasks", "worker_max_tasks", "translation_max_tasks"],
"max_tasks",
)?;
let config = TranslationWorkerConfig {
provider,
fixture_path: rpc_translation_worker_string_param(
params,
&[
"fixture_path",
"translation_fixture",
"provider_fixture",
"mock_fixture",
"fixture",
],
"fixture_path",
)?
.map(PathBuf::from),
concurrency,
max_attempts,
lease_seconds,
retry_backoff: Duration::from_secs(retry_backoff_seconds),
max_tasks,
worker_id: rpc_translation_worker_string_param(
params,
&["worker_id", "translation_worker_id"],
"worker_id",
)?
.unwrap_or_else(|| "bat-rpc-worker".to_string()),
};
config.validate().map_err(|error| {
ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
error.to_string(),
)
})?;
Ok(config)
}
fn rpc_translation_worker_string_param(
params: &serde_json::Value,
aliases: &[&str],
label: &str,
) -> Result<Option<String>, ApiError> {
for key in aliases {
let Some(value) = params.get(*key) else {
continue;
};
if value.is_null() {
return Ok(None);
}
let Some(text) = value.as_str() else {
return Err(ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("{label} 必须是字符串"),
));
};
let text = text.trim();
if text.is_empty() {
return Ok(None);
}
return Ok(Some(text.to_string()));
}
Ok(None)
}
fn rpc_translation_worker_u64_param(
params: &serde_json::Value,
aliases: &[&str],
label: &str,
) -> Result<Option<u64>, ApiError> {
for key in aliases {
let Some(value) = params.get(*key) else {
continue;
};
if value.is_null() {
return Ok(None);
}
if value.as_i64().is_some_and(|number| number < 0) {
return Err(ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("{label} 不能为负数"),
));
}
let Some(number) = value.as_u64() else {
return Err(ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("{label} 必须是非负整数 JSON number"),
));
};
return Ok(Some(number));
}
Ok(None)
}
fn rpc_translation_worker_usize_param(
params: &serde_json::Value,
aliases: &[&str],
label: &str,
) -> Result<Option<usize>, ApiError> {
rpc_translation_worker_u64_param(params, aliases, label)?
.map(|number| {
usize::try_from(number).map_err(|error| {
ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("{label} 无效:{error}"),
)
})
})
.transpose()
}
fn rpc_translation_worker_u32_param(
params: &serde_json::Value,
aliases: &[&str],
label: &str,
) -> Result<Option<u32>, ApiError> {
rpc_translation_worker_u64_param(params, aliases, label)?
.map(|number| {
u32::try_from(number).map_err(|error| {
ApiError::new(
ErrorCode::RPC_INVALID_PARAMS,
RPC_METHOD_TRANSLATION_WORKER_RUN,
format!("{label} 无效:{error}"),
)
})
})
.transpose()
}
/// 分页参数:`offset` 默认 0`limit` 默认 100,范围 1..=1000。
fn rpc_page_params(params: Option<&serde_json::Value>) -> anyhow::Result<(usize, usize)> {
let offset = params
@@ -4221,6 +4502,32 @@ impl HumanReport for bat_infrastructure::LocalizedTranslationWorkflowReport {
}
}
impl HumanReport for bat_infrastructure::TranslationWorkerReport {
fn print_human(&self) -> anyhow::Result<()> {
print_title("翻译 provider worker");
print_field("命令", self.command);
print_field("状态", self.status);
print_field("官方 release", &self.official_release_id);
print_field("provider", &self.provider);
print_field("回收 lease", self.recovered_lease_count);
print_field("领取任务", self.claimed_count);
print_field("完成任务", self.completed_count);
print_field("失败任务", self.failed_count);
print_field("已安排重试", self.retry_scheduled_count);
print_field("剩余任务", self.remaining_count);
for failure in &self.failures {
println!(
" - {} [{}] retryable={} {}",
failure.task_id,
failure.failure_class,
format_bool(failure.retryable),
failure.failure_reason
);
}
Ok(())
}
}
fn print_human_json_value(value: &serde_json::Value) -> anyhow::Result<()> {
if value.get("running").is_some() && value.get("state_dir").is_some() {
print_title("后台状态");
@@ -6321,6 +6628,17 @@ BAT_AUTO_DISCOVER=1
#BAT_DOWNLOAD_CONCURRENCY=8
#BAT_UNZIP=unzip
# ---- provider worker ----
# providermock fixture crowdin CROWDIN_*
#BAT_TRANSLATION_PROVIDER=mock
#BAT_TRANSLATION_FIXTURE=
#BAT_TRANSLATION_CONCURRENCY=8
#BAT_TRANSLATION_MAX_ATTEMPTS=3
#BAT_TRANSLATION_LEASE_SECONDS=300
#BAT_TRANSLATION_RETRY_BACKOFF_SECONDS=5
#BAT_TRANSLATION_MAX_TASKS=
#BAT_TRANSLATION_WORKER_ID=
# ---- ----
# 1 JSON
#BAT_JSON=0
@@ -6504,6 +6822,38 @@ fn apply_bat_env_overrides(
if let Some(v) = value("BAT_UNZIP") {
options.config.unzip_command = PathBuf::from(v);
}
if let Some(v) = value("BAT_TRANSLATION_PROVIDER") {
options.translation_provider = Some(v);
}
if let Some(v) = value("BAT_TRANSLATION_FIXTURE") {
options.translation_fixture = Some(PathBuf::from(v));
}
if let Some(v) = value("BAT_TRANSLATION_CONCURRENCY") {
options.worker_concurrency =
parse_translation_worker_concurrency(&v, "环境变量 BAT_TRANSLATION_CONCURRENCY")?;
}
if let Some(v) = value("BAT_TRANSLATION_MAX_ATTEMPTS") {
options.worker_max_attempts =
parse_positive_u32(&v, "环境变量 BAT_TRANSLATION_MAX_ATTEMPTS")?;
}
if let Some(v) = value("BAT_TRANSLATION_LEASE_SECONDS") {
options.worker_lease_seconds =
parse_positive_u64(&v, "环境变量 BAT_TRANSLATION_LEASE_SECONDS")?;
}
if let Some(v) = value("BAT_TRANSLATION_RETRY_BACKOFF_SECONDS") {
options.worker_retry_backoff = Duration::from_secs(v.parse::<u64>().map_err(|error| {
anyhow::anyhow!("环境变量 BAT_TRANSLATION_RETRY_BACKOFF_SECONDS 的秒数无效:{error}")
})?);
}
if let Some(v) = value("BAT_TRANSLATION_MAX_TASKS") {
options.worker_max_tasks = Some(parse_positive_usize(
&v,
"环境变量 BAT_TRANSLATION_MAX_TASKS",
)?);
}
if let Some(v) = value("BAT_TRANSLATION_WORKER_ID") {
options.worker_id = Some(v);
}
if let Some(v) = value("BAT_PROXY") {
options.config.curl_proxy = parse_proxy_config(&v)?;
}
@@ -6725,6 +7075,55 @@ fn parse_args_with_env(
"--provider-run-id" => {
options.translation_provider_run_id = Some(next_option_value(&mut args, &flag)?);
}
"--translation-provider" | "--provider" => {
options.translation_provider = Some(next_option_value(&mut args, &flag)?);
options.translation_worker_option_explicit = true;
}
"--translation-fixture" | "--provider-fixture" | "--mock-fixture" | "--fixture" => {
options.translation_fixture =
Some(PathBuf::from(next_option_value(&mut args, &flag)?));
options.translation_worker_option_explicit = true;
}
"--worker-concurrency" | "--translation-concurrency" => {
options.worker_concurrency = parse_translation_worker_concurrency(
&next_option_value(&mut args, &flag)?,
&flag,
)?;
options.translation_worker_option_explicit = true;
}
"--worker-max-attempts" | "--translation-max-attempts" => {
options.worker_max_attempts =
parse_positive_u32(&next_option_value(&mut args, &flag)?, &flag)?;
options.translation_worker_option_explicit = true;
}
"--worker-lease-seconds" | "--translation-lease-seconds" => {
options.worker_lease_seconds =
parse_positive_u64(&next_option_value(&mut args, &flag)?, &flag)?;
options.translation_worker_option_explicit = true;
}
"--worker-retry-backoff" | "--translation-retry-backoff" => {
options.worker_retry_backoff =
parse_duration(&next_option_value(&mut args, &flag)?)?;
options.translation_worker_option_explicit = true;
}
"--worker-retry-backoff-seconds" | "--translation-retry-backoff-seconds" => {
let seconds = next_option_value(&mut args, &flag)?
.parse::<u64>()
.map_err(|error| anyhow::anyhow!("{flag} 的秒数无效:{error}"))?;
options.worker_retry_backoff = Duration::from_secs(seconds);
options.translation_worker_option_explicit = true;
}
"--worker-max-tasks" | "--translation-max-tasks" => {
options.worker_max_tasks = Some(parse_positive_usize(
&next_option_value(&mut args, &flag)?,
&flag,
)?);
options.translation_worker_option_explicit = true;
}
"--worker-id" | "--translation-worker-id" => {
options.worker_id = Some(next_option_value(&mut args, &flag)?);
options.translation_worker_option_explicit = true;
}
"--localized-release-id" => {
options.localized_release_id = Some(next_option_value(&mut args, &flag)?);
}
@@ -7185,6 +7584,12 @@ fn parse_args_with_env(
));
}
if options.command != CliCommand::TranslationWorker
&& options.translation_worker_option_explicit
{
return Err(anyhow::anyhow!("翻译 worker 参数只适用于 i18n worker run"));
}
match options.command {
CliCommand::Status | CliCommand::Stop | CliCommand::Logs => {
if options.sync_option_explicit
@@ -7406,6 +7811,56 @@ fn parse_args_with_env(
options.progress = false;
options.banner = false;
}
CliCommand::TranslationWorker => {
if options.daemon || options.daemon_child {
return Err(anyhow::anyhow!(
"i18n worker run 使用 --watch 或 --run-count,不支持 daemon"
));
}
if options.config.force
|| options.config.dry_run
|| options.schedule_option_explicit
|| options.query_option_explicit
|| options.write_patch_option_explicit
|| options.translation_file.is_some()
|| options.translation_id.is_some()
|| options.translation_text.is_some()
|| options.translation_text_file.is_some()
|| options.translation_failure_reason.is_some()
|| options.translation_provider_run_id.is_some()
|| options.localized_release_id.is_some()
|| options.repack_spec.is_some()
|| options.proxy_option_explicit
|| tools_are_non_default(&options.config, &options.env_baseline_config)
{
return Err(anyhow::anyhow!(
"i18n worker run 只接受 --output、--resource-root、--state-dir、worker 参数和轮询参数"
));
}
let provider = options
.translation_provider
.as_deref()
.unwrap_or(TranslationProviderKind::Mock.as_str());
if TranslationProviderKind::parse(provider).is_none() {
return Err(anyhow::anyhow!(
"i18n worker run 的 provider 无效:{provider}"
));
}
TranslationWorkerConfig {
provider: TranslationProviderKind::parse(provider).expect("provider 已在上方校验"),
fixture_path: options.translation_fixture.clone(),
concurrency: options.worker_concurrency,
max_attempts: options.worker_max_attempts,
lease_seconds: options.worker_lease_seconds,
retry_backoff: options.worker_retry_backoff,
max_tasks: options.worker_max_tasks,
worker_id: options
.worker_id
.clone()
.unwrap_or_else(|| "bat-worker-validation".to_string()),
}
.validate()?;
}
CliCommand::TranslationProofread => {
if options.watch || options.daemon || options.daemon_child {
return Err(anyhow::anyhow!("i18n proofread 只支持单次执行或 RPC 调用"));
@@ -7506,6 +7961,7 @@ fn parse_args_with_env(
CliCommand::Pull
| CliCommand::Parse
| CliCommand::Translate
| CliCommand::TranslationWorker
| CliCommand::PublishLocalized
)
{
@@ -7515,7 +7971,11 @@ fn parse_args_with_env(
}
if matches!(
options.command,
CliCommand::Pull | CliCommand::Parse | CliCommand::Translate | CliCommand::PublishLocalized
CliCommand::Pull
| CliCommand::Parse
| CliCommand::Translate
| CliCommand::PublishLocalized
| CliCommand::TranslationWorker
) {
if options.watch && options.run_count.is_some() {
return Err(anyhow::anyhow!(
@@ -7632,6 +8092,9 @@ fn parse_translation_command(
if action == "workbench" || action == "wb" {
return parse_translation_workbench_command(args, options);
}
if action == "worker" {
return parse_translation_worker_command(args, options);
}
let command = match action.as_str() {
"run" => CliCommand::Translate,
"export" => CliCommand::Translate,
@@ -7655,6 +8118,20 @@ fn parse_translation_command(
Ok(())
}
fn parse_translation_worker_command(
args: &mut impl Iterator<Item = String>,
options: &mut CliOptions,
) -> anyhow::Result<()> {
let action = next_option_value(args, "translation worker")?;
let command = match action.as_str() {
"run" => CliCommand::TranslationWorker,
other => return Err(anyhow::anyhow!("未知 translation worker 二级命令:{other}")),
};
ensure_command_not_set(options.command, &format!("translation worker {action}"))?;
options.command = command;
Ok(())
}
fn parse_translation_workbench_command(
args: &mut impl Iterator<Item = String>,
options: &mut CliOptions,
@@ -7738,6 +8215,48 @@ fn parse_download_concurrency(value: &str, source: &str) -> anyhow::Result<usize
Ok(parsed)
}
fn parse_translation_worker_concurrency(value: &str, source: &str) -> anyhow::Result<usize> {
let parsed = value
.parse::<usize>()
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
if !(MIN_TRANSLATION_CONCURRENCY..=MAX_TRANSLATION_CONCURRENCY).contains(&parsed) {
return Err(anyhow::anyhow!(
"{source} 必须在 {MIN_TRANSLATION_CONCURRENCY}..={MAX_TRANSLATION_CONCURRENCY} 范围内"
));
}
Ok(parsed)
}
fn parse_positive_usize(value: &str, source: &str) -> anyhow::Result<usize> {
let parsed = value
.parse::<usize>()
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
if parsed == 0 {
return Err(anyhow::anyhow!("{source} 必须大于 0"));
}
Ok(parsed)
}
fn parse_positive_u32(value: &str, source: &str) -> anyhow::Result<u32> {
let parsed = value
.parse::<u32>()
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
if parsed == 0 {
return Err(anyhow::anyhow!("{source} 必须大于 0"));
}
Ok(parsed)
}
fn parse_positive_u64(value: &str, source: &str) -> anyhow::Result<u64> {
let parsed = value
.parse::<u64>()
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
if parsed == 0 {
return Err(anyhow::anyhow!("{source} 必须大于 0"));
}
Ok(parsed)
}
fn next_option_value(
args: &mut impl Iterator<Item = String>,
flag: &str,
@@ -7807,6 +8326,7 @@ fn print_usage(binary: &str) {
eprintln!(" i18n unset Clear one translated workbench entry");
eprintln!(" i18n validate Validate workbench against the current official release");
eprintln!(" i18n proofread Mark localized workflow as manual proofreading");
eprintln!(" i18n worker run Run translation provider worker once or repeatedly");
eprintln!(
" i18n tasks / i18n task list / i18n task status Query current offline TextUnit translation task status"
);
@@ -7854,6 +8374,9 @@ fn print_usage(binary: &str) {
" {binary} i18n unset --translation-file /tmp/bat-workbench.json --translation-id unit-1"
);
eprintln!(" {binary} i18n proofread --json");
eprintln!(
" {binary} i18n worker run --provider mock --worker-concurrency 8 --run-count 2 --interval 30s"
);
eprintln!(" {binary} i18n tasks --json");
eprintln!(" {binary} i18n handoff --json");
eprintln!(" {binary} i18n status --json");
@@ -7908,6 +8431,17 @@ fn print_usage(binary: &str) {
eprintln!(" --translated-file <PATH> UTF-8 translation file for i18n set");
eprintln!(" --failure-reason <TEXT> Provider failure reason for i18n task update");
eprintln!(" --provider-run-id <ID> Provider run ID for i18n task update");
eprintln!(" --translation-provider <NAME> / --provider <NAME> Provider for i18n worker run (mock/crowdin)");
eprintln!(" --translation-fixture <PATH> Mock/provider fixture for i18n worker run");
eprintln!(
" --worker-concurrency <N> Translation worker concurrency (default: 8, range 1..=256)"
);
eprintln!(" --worker-max-attempts <N> Maximum claims per translation task");
eprintln!(" --worker-lease-seconds <N> Lease seconds for one claimed task");
eprintln!(" --worker-retry-backoff <DURATION> Retry backoff after retryable failure");
eprintln!(" --worker-retry-backoff-seconds <N> Retry backoff seconds");
eprintln!(" --worker-max-tasks <N> Maximum tasks claimed in one worker run");
eprintln!(" --worker-id <ID> Worker ID prefix for lease diagnostics");
eprintln!(" --localized-release-id <ID> Explicit localized publication ID");
eprintln!(" --repack-spec <PATH> UnityFS batch repack JSON spec");
eprintln!();
+159
View File
@@ -140,6 +140,165 @@ fn env_invalid_values_error() {
assert!(parse(&["bat", "--download-concurrency", "257"]).is_err());
}
#[test]
fn translation_worker_command_options_are_validated() {
let options = parse(&[
"bat",
"i18n",
"worker",
"run",
"--provider",
"mock",
"--translation-fixture",
"/tmp/mock-translation.json",
"--worker-concurrency",
"8",
"--worker-max-attempts",
"4",
"--worker-lease-seconds",
"60",
"--worker-retry-backoff",
"0s",
"--worker-max-tasks",
"2",
"--worker-id",
"manual-run",
"--run-count",
"2",
"--interval",
"30s",
])
.unwrap();
assert_eq!(options.command, CliCommand::TranslationWorker);
assert_eq!(options.translation_provider.as_deref(), Some("mock"));
assert_eq!(
options.translation_fixture,
Some(PathBuf::from("/tmp/mock-translation.json"))
);
assert_eq!(options.worker_concurrency, 8);
assert_eq!(options.worker_max_attempts, 4);
assert_eq!(options.worker_lease_seconds, 60);
assert_eq!(options.worker_retry_backoff, Duration::ZERO);
assert_eq!(options.worker_max_tasks, Some(2));
assert_eq!(options.worker_id.as_deref(), Some("manual-run"));
assert_eq!(options.run_count, Some(2));
assert!(parse(&["bat", "i18n", "worker", "run"]).is_ok());
assert!(parse(&[
"bat",
"translation",
"worker",
"run",
"--translation-concurrency",
"256"
])
.is_ok());
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-concurrency", "0"]).is_err());
assert!(parse(&[
"bat",
"i18n",
"worker",
"run",
"--worker-concurrency",
"257"
])
.is_err());
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-max-attempts", "0"]).is_err());
assert!(parse(&[
"bat",
"i18n",
"worker",
"run",
"--worker-lease-seconds",
"0"
])
.is_err());
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-max-tasks", "0"]).is_err());
assert!(parse(&["bat", "i18n", "run", "--worker-concurrency", "8"]).is_err());
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
}
#[test]
fn translation_worker_env_defaults_apply() {
let options = parse_with_env(
&["bat", "i18n", "worker", "run"],
&[
("BAT_TRANSLATION_PROVIDER", "mock"),
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
("BAT_TRANSLATION_CONCURRENCY", "16"),
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
("BAT_TRANSLATION_LEASE_SECONDS", "120"),
("BAT_TRANSLATION_RETRY_BACKOFF_SECONDS", "0"),
("BAT_TRANSLATION_MAX_TASKS", "7"),
("BAT_TRANSLATION_WORKER_ID", "env-worker"),
],
)
.unwrap();
assert_eq!(options.command, CliCommand::TranslationWorker);
assert_eq!(options.translation_provider.as_deref(), Some("mock"));
assert_eq!(
options.translation_fixture,
Some(PathBuf::from("/tmp/fixture.json"))
);
assert_eq!(options.worker_concurrency, 16);
assert_eq!(options.worker_max_attempts, 5);
assert_eq!(options.worker_lease_seconds, 120);
assert_eq!(options.worker_retry_backoff, Duration::ZERO);
assert_eq!(options.worker_max_tasks, Some(7));
assert_eq!(options.worker_id.as_deref(), Some("env-worker"));
assert!(parse_with_env(
&["bat", "i18n", "worker", "run"],
&[("BAT_TRANSLATION_CONCURRENCY", "257")]
)
.is_err());
assert!(parse_with_env(
&["bat", "i18n", "worker", "run"],
&[("BAT_TRANSLATION_MAX_ATTEMPTS", "0")]
)
.is_err());
}
#[test]
fn translation_worker_rpc_params_are_strict_and_accept_aliases() {
let default_config = rpc_translation_worker_config(None).unwrap();
assert_eq!(default_config.provider, TranslationProviderKind::Mock);
assert_eq!(default_config.concurrency, DEFAULT_TRANSLATION_CONCURRENCY);
let config = rpc_translation_worker_config(Some(&serde_json::json!({
"translation_provider": "mock",
"provider_fixture": "/tmp/mock-provider.json",
"worker_concurrency": 8,
"worker_max_attempts": 4,
"worker_lease_seconds": 60,
"worker_retry_backoff_seconds": 0,
"worker_max_tasks": 2,
"translation_worker_id": "rpc-worker"
})))
.unwrap();
assert_eq!(config.provider, TranslationProviderKind::Mock);
assert_eq!(
config.fixture_path,
Some(PathBuf::from("/tmp/mock-provider.json"))
);
assert_eq!(config.concurrency, 8);
assert_eq!(config.max_attempts, 4);
assert_eq!(config.lease_seconds, 60);
assert_eq!(config.retry_backoff, Duration::ZERO);
assert_eq!(config.max_tasks, Some(2));
assert_eq!(config.worker_id, "rpc-worker");
assert!(rpc_translation_worker_config(Some(&serde_json::json!([]))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"provider": "bad"}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": "8"}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": -1}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": 0}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": 257}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"max_attempts": 0}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"lease_seconds": 0}))).is_err());
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"max_tasks": 0}))).is_err());
}
#[test]
fn grouped_workflow_commands_use_short_top_level_aliases() {
let options = parse(&[
+68 -26
View File
@@ -4,12 +4,14 @@ pub(super) const MAX_RETAINED_TASKS: usize = 64;
/// 每个任务保留的进度日志行数上限。
pub(super) const MAX_TASK_LOG_LINES: usize = 200;
/// 任务类型:目前覆盖官方同步、校验与 catalog 更新检查。
#[derive(Debug, Clone, Copy)]
/// 任务类型:覆盖资源同步、校验、修复、翻译 worker 与 catalog 更新检查。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum TaskKind {
Sync,
Verify,
Repair,
/// 翻译 provider worker 轮次。
TranslationWorker,
/// catalog 更新检查:只做发现 + 拉取计划(dry-run),不下载不审计。
Refresh,
}
@@ -20,6 +22,7 @@ impl TaskKind {
Self::Sync => RPC_METHOD_RESOURCE_SYNC,
Self::Verify => RPC_METHOD_RESOURCE_VERIFY,
Self::Repair => RPC_METHOD_RESOURCE_REPAIR,
Self::TranslationWorker => RPC_METHOD_TRANSLATION_WORKER_RUN,
Self::Refresh => RPC_METHOD_CATALOG_REFRESH,
}
}
@@ -56,6 +59,10 @@ impl TaskKind {
config.repair = false;
config.force = force;
}
Self::TranslationWorker => {
config.dry_run = false;
config.force = false;
}
}
config
}
@@ -155,6 +162,7 @@ fn task_kind_static(kind: &str) -> Option<&'static str> {
RPC_METHOD_RESOURCE_SYNC => Some(RPC_METHOD_RESOURCE_SYNC),
RPC_METHOD_RESOURCE_VERIFY => Some(RPC_METHOD_RESOURCE_VERIFY),
RPC_METHOD_RESOURCE_REPAIR => Some(RPC_METHOD_RESOURCE_REPAIR),
RPC_METHOD_TRANSLATION_WORKER_RUN => Some(RPC_METHOD_TRANSLATION_WORKER_RUN),
RPC_METHOD_CATALOG_REFRESH => Some(RPC_METHOD_CATALOG_REFRESH),
_ => None,
}
@@ -494,7 +502,9 @@ impl TaskStore {
/// 提交给任务 worker 的作业(配置已按任务类型派生完毕)。
pub(super) struct TaskJob {
pub(super) id: String,
pub(super) kind: TaskKind,
pub(super) config: OfficialUpdateConfig,
pub(super) translation_worker_config: Option<TranslationWorkerConfig>,
/// 与任务记录共享的取消标志。
pub(super) cancel: Arc<AtomicBool>,
}
@@ -528,36 +538,68 @@ pub(super) fn run_task_worker(
});
let cancel = Arc::clone(&job.cancel);
let run_result = {
let _sync_guard = sync_lock
.lock()
.unwrap_or_else(|poison| poison.into_inner());
let progress_registry = registry.clone();
let progress_id = job.id.clone();
let cancel_check = Arc::clone(&cancel);
let stop_control = Arc::clone(&control);
service.run_with_progress_and_cancellation(
&job.config,
|event| {
progress_registry
.append_log(&progress_id, format!("[{}] {}", event.stage, event.message));
progress_registry.update(&progress_id, |record| {
record.stage = Some(event.stage.to_string());
record.message = Some(event.message.clone());
});
},
|| {
cancel_check.load(Ordering::Relaxed)
|| daemon_control_stop_requested(Some(&stop_control))
},
)
let run_result = if job.kind == TaskKind::TranslationWorker {
let worker_config = job
.translation_worker_config
.as_ref()
.ok_or_else(|| anyhow::anyhow!("translation worker 任务缺少运行配置"));
worker_config.and_then(|worker_config| {
let _sync_guard = sync_lock
.lock()
.unwrap_or_else(|poison| poison.into_inner());
registry.append_log(&job.id, "[translation-worker] 开始执行".to_string());
registry.update(&job.id, |record| {
record.stage = Some("translation-worker".to_string());
record.message = Some("翻译 provider worker 正在执行".to_string());
});
let resource_root = active_official_resource_root(&job.config.output_root)?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime
.block_on(bat_infrastructure::run_translation_worker_at(
&resource_root,
worker_config,
))
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
})
} else {
let run_result = {
let _sync_guard = sync_lock
.lock()
.unwrap_or_else(|poison| poison.into_inner());
let progress_registry = registry.clone();
let progress_id = job.id.clone();
let cancel_check = Arc::clone(&cancel);
let stop_control = Arc::clone(&control);
service.run_with_progress_and_cancellation(
&job.config,
|event| {
progress_registry.append_log(
&progress_id,
format!("[{}] {}", event.stage, event.message),
);
progress_registry.update(&progress_id, |record| {
record.stage = Some(event.stage.to_string());
record.message = Some(event.message.clone());
});
},
|| {
cancel_check.load(Ordering::Relaxed)
|| daemon_control_stop_requested(Some(&stop_control))
},
)
};
run_result
.map(|report| serde_json::to_value(&report).map_err(anyhow::Error::from))
.and_then(|result| result)
};
match run_result {
Ok(report) => registry.update(&job.id, |record| {
record.status = "succeeded";
record.finished_at = Some(unix_seconds_now());
record.result = serde_json::to_value(&report).ok();
record.result = Some(report);
}),
Err(error) => {
let cancelled = cancel.load(Ordering::Relaxed);
@@ -253,6 +253,43 @@ pub(super) fn run_translation_proofread(options: &CliOptions) -> anyhow::Result<
print_report(options.output_format, &report)
}
pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()> {
let resource_root = options
.resource_root
.clone()
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
.transpose()?
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
let provider = options
.translation_provider
.as_deref()
.unwrap_or(TranslationProviderKind::Mock.as_str());
let provider = TranslationProviderKind::parse(provider)
.ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效:{provider}"))?;
let config = TranslationWorkerConfig {
provider,
fixture_path: options.translation_fixture.clone(),
concurrency: options.worker_concurrency,
max_attempts: options.worker_max_attempts,
lease_seconds: options.worker_lease_seconds,
retry_backoff: options.worker_retry_backoff,
max_tasks: options.worker_max_tasks,
worker_id: options
.worker_id
.clone()
.unwrap_or_else(|| format!("bat-worker-{}", std::process::id())),
};
config.validate()?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let report = runtime.block_on(bat_infrastructure::run_translation_worker_at(
&resource_root,
&config,
))?;
print_report(options.output_format, &report)
}
pub(super) fn run_repack(options: &CliOptions) -> anyhow::Result<()> {
let spec = options
.repack_spec
+10 -2
View File
@@ -156,7 +156,7 @@ mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread;
use std::time::Duration;
use std::time::{Duration, Instant};
struct TestBackend {
active: AtomicUsize,
@@ -222,7 +222,15 @@ mod tests {
fn download(&self, task: usize) -> Result<Self::Output, Self::Error> {
if task == 0 {
self.active.fetch_add(1, Ordering::SeqCst);
thread::sleep(Duration::from_millis(20));
let deadline = Instant::now() + Duration::from_millis(500);
while self
.task_two_started_while_task_zero_active
.load(Ordering::SeqCst)
== 0
&& Instant::now() < deadline
{
thread::sleep(Duration::from_millis(1));
}
self.active.fetch_sub(1, Ordering::SeqCst);
} else {
if task == 1 {
+14 -3
View File
@@ -30,6 +30,7 @@ pub mod path_security;
pub mod release_flow;
pub mod resources;
pub mod translation_tasks;
pub mod translation_worker;
pub mod translation_workflow;
mod zip_validation;
@@ -138,11 +139,21 @@ pub use translation_tasks::{
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
TranslationJob, TranslationJobStatus, TranslationTaskStatus, TranslationTaskSyncReport,
TranslationUnit, TranslationUnitStatus, TRANSLATION_HANDOFF_FILE,
TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus,
TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus,
TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
TRANSLATION_TASK_SCHEMA_VERSION,
};
pub use translation_worker::{
run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider,
MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass,
TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse,
TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig,
TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY,
DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS,
DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY,
MOCK_TRANSLATION_FIXTURE_VERSION,
};
pub use translation_workflow::{
export_translation_workbench, get_translation_entry, localized_text_asset_patches,
read_translation_workbench, repack_bundle, set_translation, unset_translation,
+672 -33
View File
@@ -14,15 +14,15 @@ use crate::path_security::{
};
use bat_core::Result;
use serde::{Deserialize, Serialize};
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::str::FromStr;
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// SQLite schema version for durable translation task state.
pub const TRANSLATION_TASK_SCHEMA_VERSION: u32 = 1;
pub const TRANSLATION_TASK_SCHEMA_VERSION: u32 = 2;
const TRANSLATION_TASK_SCHEMA_COMPONENT: &str = "translation_tasks";
/// SQLite file name stored under a published official release root.
pub const TRANSLATION_TASK_REPOSITORY_FILE: &str = "translation-tasks.sqlite";
@@ -147,6 +147,26 @@ pub struct TranslationUnit {
pub status: TranslationUnitStatus,
/// Provider failure diagnostic, when present.
pub failure_reason: Option<String>,
/// TextUnit-level provider results associated with this resource task.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub translation_results: Vec<TranslationTaskUnitResult>,
}
/// One provider result linked to an immutable TextUnit.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TranslationTaskUnitResult {
/// Stable TextUnit ID from `official-textunit-index.json`.
pub unit_id: String,
/// Source text submitted to the provider.
pub source_text: String,
/// Provider-produced or human-supplied translation.
pub translated_text: String,
/// Provider identifier.
pub provider: String,
/// Provider run that produced this result.
pub provider_run_id: String,
/// Result persistence time.
pub translated_unix_seconds: u64,
}
/// One provider execution associated with one or more translation units.
@@ -203,7 +223,16 @@ pub fn build_translation_handoff(
let state = persisted
.get(task.task_id.as_str())
.copied()
.map(|task| (task.task_status, task.failure_reason.clone(), task.attempt_count, task.provider_run_id.clone()))
.map(|task| {
(
task.task_status,
task.failure_reason.clone(),
task.attempt_count,
task.provider_run_id.clone(),
task.provider.clone(),
task.translation_results.clone(),
)
})
.unwrap_or_else(|| {
(
initial_task_status(task),
@@ -214,6 +243,8 @@ pub fn build_translation_handoff(
},
0,
None,
None,
Vec::new(),
)
});
let unit_status = match state.0 {
@@ -234,6 +265,7 @@ pub fn build_translation_handoff(
text_unit_formats: task.text_unit_formats.clone(),
status: unit_status,
failure_reason: state.1.clone(),
translation_results: state.5.clone(),
};
if let Some(provider_run_id) = state.3 {
let run = provider_runs
@@ -241,7 +273,7 @@ pub fn build_translation_handoff(
.or_insert_with(|| ProviderRun {
provider_run_id,
job_id: job_id.clone(),
provider: "worker".to_string(),
provider: state.4.unwrap_or_else(|| "worker".to_string()),
status: ProviderRunStatus::Queued,
unit_ids: Vec::new(),
attempt_count: 0,
@@ -403,6 +435,26 @@ pub struct PersistedTranslationTask {
/// Provider-side run identifier, if known.
#[serde(skip_serializing_if = "Option::is_none")]
pub provider_run_id: Option<String>,
/// Provider-produced results keyed by TextUnit ID.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub translation_results: Vec<TranslationTaskUnitResult>,
/// Provider identifier used by the latest run.
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
/// Worker currently holding the lease.
#[serde(skip_serializing_if = "Option::is_none")]
pub lease_owner: Option<String>,
/// Lease expiry as Unix seconds.
#[serde(skip_serializing_if = "Option::is_none")]
pub lease_expires_unix_seconds: Option<u64>,
/// Stable failure classification.
#[serde(skip_serializing_if = "Option::is_none")]
pub failure_class: Option<String>,
/// Whether the latest failure can be retried.
pub failure_retryable: bool,
/// Earliest Unix time at which a failed task may be claimed again.
#[serde(skip_serializing_if = "Option::is_none")]
pub next_attempt_unix_seconds: Option<u64>,
}
/// Mutable persistence metadata associated with an official TextUnit task.
@@ -422,6 +474,39 @@ pub struct PersistedTranslationTaskState {
pub completed_unix_seconds: Option<u64>,
/// Provider-side run identifier, if known.
pub provider_run_id: Option<String>,
/// Provider-produced results keyed by TextUnit ID.
pub translation_results: Vec<TranslationTaskUnitResult>,
/// Provider identifier used by the latest run.
pub provider: Option<String>,
/// Worker currently holding the lease.
pub lease_owner: Option<String>,
/// Lease expiry as Unix seconds.
pub lease_expires_unix_seconds: Option<u64>,
/// Stable failure classification.
pub failure_class: Option<String>,
/// Whether the latest failure can be retried.
pub failure_retryable: bool,
/// Earliest Unix time at which a failed task may be claimed again.
pub next_attempt_unix_seconds: Option<u64>,
}
/// Provider failure data used to atomically release a task lease.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TranslationTaskFailure {
/// Task being failed.
pub task_id: String,
/// Worker that owns the lease.
pub worker_id: String,
/// Provider run associated with the lease.
pub provider_run_id: String,
/// Stable failure classification.
pub failure_class: String,
/// Redacted diagnostic message.
pub failure_reason: String,
/// Whether a future attempt may retry the task.
pub retryable: bool,
/// Earliest retry time, when retryable.
pub next_attempt_unix_seconds: Option<u64>,
}
/// Result of synchronizing an immutable release queue into SQLite.
@@ -463,7 +548,9 @@ impl SqliteTranslationTaskRepository {
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", path.display()))
.map_err(|error| bat_core::Error::Other(error.into()))?
.create_if_missing(create_if_missing);
.create_if_missing(create_if_missing)
.journal_mode(SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30));
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
@@ -510,7 +597,14 @@ impl SqliteTranslationTaskRepository {
created_unix_seconds INTEGER NOT NULL,
updated_unix_seconds INTEGER NOT NULL,
completed_unix_seconds INTEGER,
provider_run_id TEXT
provider_run_id TEXT,
translation_results_json TEXT NOT NULL DEFAULT '[]',
provider TEXT,
lease_owner TEXT,
lease_expires_unix_seconds INTEGER,
failure_class TEXT,
failure_retryable INTEGER NOT NULL DEFAULT 0,
next_attempt_unix_seconds INTEGER
)
"#,
)
@@ -573,6 +667,37 @@ impl SqliteTranslationTaskRepository {
)
.await?;
ensure_column(&self.pool, "translation_tasks", "provider_run_id", "TEXT").await?;
ensure_column(
&self.pool,
"translation_tasks",
"translation_results_json",
"TEXT NOT NULL DEFAULT '[]'",
)
.await?;
ensure_column(&self.pool, "translation_tasks", "provider", "TEXT").await?;
ensure_column(&self.pool, "translation_tasks", "lease_owner", "TEXT").await?;
ensure_column(
&self.pool,
"translation_tasks",
"lease_expires_unix_seconds",
"INTEGER",
)
.await?;
ensure_column(&self.pool, "translation_tasks", "failure_class", "TEXT").await?;
ensure_column(
&self.pool,
"translation_tasks",
"failure_retryable",
"INTEGER NOT NULL DEFAULT 0",
)
.await?;
ensure_column(
&self.pool,
"translation_tasks",
"next_attempt_unix_seconds",
"INTEGER",
)
.await?;
let current: Option<i64> =
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
@@ -615,7 +740,10 @@ impl SqliteTranslationTaskRepository {
let existing: Option<ExistingTaskRow> = sqlx::query_as(
r#"
SELECT task_json, worker_status, failure_reason, attempt_count,
created_unix_seconds, completed_unix_seconds, provider_run_id
created_unix_seconds, completed_unix_seconds, provider_run_id,
translation_results_json, provider, lease_owner,
lease_expires_unix_seconds, failure_class, failure_retryable,
next_attempt_unix_seconds
FROM translation_tasks
WHERE task_id = ?1
"#,
@@ -642,30 +770,64 @@ impl SqliteTranslationTaskRepository {
created,
completed,
provider_run_id,
translation_results_json,
provider,
lease_owner,
lease_expires,
failure_class,
failure_retryable,
next_attempt,
)) = existing
{
let immutable_unchanged = previous_task_json == task_json;
let (status, failure_reason, attempt_count, created, completed, provider_run_id) =
if immutable_unchanged {
(
worker_status,
failure_reason,
attempt_count,
created,
completed,
provider_run_id,
)
} else {
report.refreshed_count += 1;
(
initial_status.as_str().to_string(),
initial_failure_reason.clone(),
0_i64,
now,
None,
None,
)
};
let (
status,
failure_reason,
attempt_count,
created,
completed,
provider_run_id,
translation_results_json,
provider,
lease_owner,
lease_expires,
failure_class,
failure_retryable,
next_attempt,
) = if immutable_unchanged {
(
worker_status,
failure_reason,
attempt_count,
created,
completed,
provider_run_id,
translation_results_json,
provider,
lease_owner,
lease_expires,
failure_class,
failure_retryable,
next_attempt,
)
} else {
report.refreshed_count += 1;
(
initial_status.as_str().to_string(),
initial_failure_reason.clone(),
0_i64,
now,
None,
None,
"[]".to_string(),
None,
None,
None,
None,
0_i64,
None,
)
};
if immutable_unchanged {
report.preserved_state_count += 1;
}
@@ -677,7 +839,11 @@ impl SqliteTranslationTaskRepository {
text_unit_formats_json = ?8, task_json = ?9,
worker_status = ?10, failure_reason = ?11, attempt_count = ?12,
created_unix_seconds = ?13, updated_unix_seconds = ?14,
completed_unix_seconds = ?15, provider_run_id = ?16
completed_unix_seconds = ?15, provider_run_id = ?16,
translation_results_json = ?17, provider = ?18,
lease_owner = ?19, lease_expires_unix_seconds = ?20,
failure_class = ?21, failure_retryable = ?22,
next_attempt_unix_seconds = ?23
WHERE task_id = ?1
"#,
)
@@ -697,6 +863,13 @@ impl SqliteTranslationTaskRepository {
.bind(now)
.bind(completed)
.bind(provider_run_id)
.bind(translation_results_json)
.bind(provider)
.bind(lease_owner)
.bind(lease_expires)
.bind(failure_class)
.bind(failure_retryable)
.bind(next_attempt)
.execute(&mut *transaction)
.await
.map_err(db_error)?;
@@ -770,7 +943,9 @@ impl SqliteTranslationTaskRepository {
r#"
SELECT task_json, worker_status, failure_reason, attempt_count,
created_unix_seconds, updated_unix_seconds,
completed_unix_seconds, provider_run_id
completed_unix_seconds, provider_run_id, translation_results_json,
provider, lease_owner, lease_expires_unix_seconds, failure_class,
failure_retryable, next_attempt_unix_seconds
FROM translation_tasks
ORDER BY task_id
"#,
@@ -794,6 +969,247 @@ impl SqliteTranslationTaskRepository {
Ok(self.list(query).await?.len() as u64)
}
/// Requeues tasks whose worker lease expired.
pub async fn recover_expired_leases(&self, now_unix_seconds: u64) -> Result<u64> {
let now = i64::try_from(now_unix_seconds).unwrap_or(i64::MAX);
let result = sqlx::query(
r#"
UPDATE translation_tasks
SET worker_status = CASE
WHEN attempt_count > 0 THEN 'failed'
ELSE 'queued'
END,
failure_reason = 'worker lease expired; task recovered',
failure_class = 'lease_expired',
failure_retryable = 1,
next_attempt_unix_seconds = ?1,
updated_unix_seconds = ?1,
lease_owner = NULL,
lease_expires_unix_seconds = NULL
WHERE worker_status = 'running'
AND lease_expires_unix_seconds IS NOT NULL
AND lease_expires_unix_seconds <= ?1
"#,
)
.bind(now)
.execute(&self.pool)
.await
.map_err(db_error)?;
Ok(result.rows_affected())
}
/// Claims one queued or retryable failed task and assigns an exclusive
/// lease to one worker.
pub async fn claim_next(
&self,
worker_id: &str,
provider: &str,
lease_seconds: u64,
max_attempts: u32,
) -> Result<Option<PersistedTranslationTask>> {
if worker_id.trim().is_empty() || provider.trim().is_empty() {
return Err(bat_core::Error::InvalidArgument(
"translation worker_id/provider 不能为空".to_string(),
));
}
if max_attempts == 0 {
return Err(bat_core::Error::InvalidArgument(
"translation worker max_attempts 必须大于 0".to_string(),
));
}
if lease_seconds == 0 {
return Err(bat_core::Error::InvalidArgument(
"translation worker lease_seconds 必须大于 0".to_string(),
));
}
let now = unix_seconds_now_i64();
let lease_expires = now.saturating_add(i64::try_from(lease_seconds).unwrap_or(i64::MAX));
let mut transaction = self.pool.begin().await.map_err(db_error)?;
let row: Option<TranslationTaskRow> = sqlx::query_as(
r#"
SELECT task_json, worker_status, failure_reason, attempt_count,
created_unix_seconds, updated_unix_seconds,
completed_unix_seconds, provider_run_id, translation_results_json,
provider, lease_owner, lease_expires_unix_seconds, failure_class,
failure_retryable, next_attempt_unix_seconds
FROM translation_tasks
WHERE queue_status = 'queued_offline'
AND (
worker_status = 'queued'
OR (
worker_status = 'failed'
AND failure_retryable = 1
AND attempt_count < ?1
AND (next_attempt_unix_seconds IS NULL
OR next_attempt_unix_seconds <= ?2)
)
)
AND (lease_expires_unix_seconds IS NULL OR lease_expires_unix_seconds <= ?2)
ORDER BY task_id
LIMIT 1
"#,
)
.bind(i64::from(max_attempts))
.bind(now)
.fetch_optional(&mut *transaction)
.await
.map_err(db_error)?;
let Some(row) = row else {
transaction.commit().await.map_err(db_error)?;
return Ok(None);
};
let current = PersistedTranslationTask::from_row(row)?;
let attempt_count = current.attempt_count.saturating_add(1);
let provider_run_id = format!(
"{provider}:{}:attempt-{attempt_count}",
current.task.task_id
);
let result = sqlx::query(
r#"
UPDATE translation_tasks
SET worker_status = 'running',
failure_reason = NULL,
attempt_count = ?2,
updated_unix_seconds = ?3,
completed_unix_seconds = NULL,
provider_run_id = ?4,
provider = ?5,
lease_owner = ?6,
lease_expires_unix_seconds = ?7,
failure_class = NULL,
failure_retryable = 0,
next_attempt_unix_seconds = NULL
WHERE task_id = ?1
AND (
worker_status = 'queued'
OR (
worker_status = 'failed'
AND failure_retryable = 1
AND attempt_count < ?8
AND (next_attempt_unix_seconds IS NULL
OR next_attempt_unix_seconds <= ?3)
)
)
AND queue_status = 'queued_offline'
AND (lease_expires_unix_seconds IS NULL OR lease_expires_unix_seconds <= ?3)
"#,
)
.bind(&current.task.task_id)
.bind(i64::from(attempt_count))
.bind(now)
.bind(&provider_run_id)
.bind(provider)
.bind(worker_id)
.bind(lease_expires)
.bind(i64::from(max_attempts))
.execute(&mut *transaction)
.await
.map_err(db_error)?;
if result.rows_affected() != 1 {
transaction.rollback().await.map_err(db_error)?;
return Ok(None);
}
transaction.commit().await.map_err(db_error)?;
self.find(&current.task.task_id).await.map(Some)
}
/// Stores an idempotent provider result while the worker still owns its lease.
pub async fn complete_claim(
&self,
task_id: &str,
worker_id: &str,
provider_run_id: &str,
provider: &str,
translation_results: &[TranslationTaskUnitResult],
) -> Result<PersistedTranslationTask> {
let results_json = serde_json::to_string(translation_results)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
let now = unix_seconds_now_i64();
let result = sqlx::query(
r#"
UPDATE translation_tasks
SET worker_status = 'completed',
failure_reason = NULL,
updated_unix_seconds = ?2,
completed_unix_seconds = ?2,
provider_run_id = ?3,
provider = ?4,
translation_results_json = ?5,
lease_owner = NULL,
lease_expires_unix_seconds = NULL,
failure_class = NULL,
failure_retryable = 0,
next_attempt_unix_seconds = NULL
WHERE task_id = ?1
AND worker_status = 'running'
AND lease_owner = ?6
AND provider_run_id = ?3
"#,
)
.bind(task_id)
.bind(now)
.bind(provider_run_id)
.bind(provider)
.bind(results_json)
.bind(worker_id)
.execute(&self.pool)
.await
.map_err(db_error)?;
if result.rows_affected() != 1 {
return Err(bat_core::Error::InvalidArgument(format!(
"翻译任务 {} 的 lease 已失效,拒绝写入 provider 结果",
task_id
)));
}
self.find(task_id).await
}
/// Records one provider failure and releases the worker lease.
pub async fn fail_claim(
&self,
failure: TranslationTaskFailure,
) -> Result<PersistedTranslationTask> {
let now = unix_seconds_now_i64();
let next_attempt = failure
.next_attempt_unix_seconds
.map(|value| i64::try_from(value).unwrap_or(i64::MAX));
let result = sqlx::query(
r#"
UPDATE translation_tasks
SET worker_status = 'failed',
failure_reason = ?2,
updated_unix_seconds = ?3,
lease_owner = NULL,
lease_expires_unix_seconds = NULL,
failure_class = ?4,
failure_retryable = ?5,
next_attempt_unix_seconds = ?6
WHERE task_id = ?1
AND worker_status = 'running'
AND lease_owner = ?7
AND provider_run_id = ?8
"#,
)
.bind(&failure.task_id)
.bind(&failure.failure_reason)
.bind(now)
.bind(&failure.failure_class)
.bind(if failure.retryable { 1_i64 } else { 0_i64 })
.bind(next_attempt)
.bind(&failure.worker_id)
.bind(&failure.provider_run_id)
.execute(&self.pool)
.await
.map_err(db_error)?;
if result.rows_affected() != 1 {
return Err(bat_core::Error::InvalidArgument(format!(
"翻译任务 {} 的 lease 已失效,拒绝写入 provider 失败状态",
failure.task_id
)));
}
self.find(&failure.task_id).await
}
/// Updates provider state and returns the durable task record.
pub async fn update_status(
&self,
@@ -818,7 +1234,10 @@ impl SqliteTranslationTaskRepository {
UPDATE translation_tasks
SET worker_status = ?2, failure_reason = ?3, attempt_count = ?4,
updated_unix_seconds = ?5, completed_unix_seconds = ?6,
provider_run_id = COALESCE(?7, provider_run_id)
provider_run_id = COALESCE(?7, provider_run_id),
lease_owner = NULL, lease_expires_unix_seconds = NULL,
failure_class = NULL, failure_retryable = 0,
next_attempt_unix_seconds = NULL
WHERE task_id = ?1
"#,
)
@@ -841,7 +1260,9 @@ impl SqliteTranslationTaskRepository {
r#"
SELECT task_json, worker_status, failure_reason, attempt_count,
created_unix_seconds, updated_unix_seconds,
completed_unix_seconds, provider_run_id
completed_unix_seconds, provider_run_id, translation_results_json,
provider, lease_owner, lease_expires_unix_seconds, failure_class,
failure_retryable, next_attempt_unix_seconds
FROM translation_tasks
WHERE task_id = ?1
"#,
@@ -864,6 +1285,13 @@ type ExistingTaskRow = (
i64,
Option<i64>,
Option<String>,
String,
Option<String>,
Option<String>,
Option<i64>,
Option<String>,
i64,
Option<i64>,
);
type TranslationTaskRow = (
@@ -875,6 +1303,13 @@ type TranslationTaskRow = (
i64,
Option<i64>,
Option<String>,
String,
Option<String>,
Option<String>,
Option<i64>,
Option<String>,
i64,
Option<i64>,
);
impl PersistedTranslationTask {
@@ -889,6 +1324,13 @@ impl PersistedTranslationTask {
updated_unix_seconds: state.updated_unix_seconds,
completed_unix_seconds: state.completed_unix_seconds,
provider_run_id: state.provider_run_id,
translation_results: state.translation_results,
provider: state.provider,
lease_owner: state.lease_owner,
lease_expires_unix_seconds: state.lease_expires_unix_seconds,
failure_class: state.failure_class,
failure_retryable: state.failure_retryable,
next_attempt_unix_seconds: state.next_attempt_unix_seconds,
}
}
@@ -910,6 +1352,13 @@ impl PersistedTranslationTask {
updated_unix_seconds: generated_unix_seconds,
completed_unix_seconds: None,
provider_run_id: None,
translation_results: Vec::new(),
provider: None,
lease_owner: None,
lease_expires_unix_seconds: None,
failure_class: None,
failure_retryable: false,
next_attempt_unix_seconds: None,
},
)
}
@@ -924,12 +1373,21 @@ impl PersistedTranslationTask {
updated_unix_seconds,
completed_unix_seconds,
provider_run_id,
translation_results_json,
provider,
lease_owner,
lease_expires_unix_seconds,
failure_class,
failure_retryable,
next_attempt_unix_seconds,
) = row;
let task = serde_json::from_str(&task_json)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
let task_status = TranslationTaskStatus::parse(&worker_status).ok_or_else(|| {
bat_core::Error::Serialization(format!("未知翻译任务 worker 状态:{worker_status}"))
})?;
let translation_results = serde_json::from_str(&translation_results_json)
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
Ok(Self {
task,
task_status,
@@ -951,6 +1409,25 @@ impl PersistedTranslationTask {
})
.transpose()?,
provider_run_id,
translation_results,
provider,
lease_owner,
lease_expires_unix_seconds: lease_expires_unix_seconds
.map(|value| {
u64::try_from(value).map_err(|_| {
bat_core::Error::Serialization("翻译任务 lease 时间无效".to_string())
})
})
.transpose()?,
failure_class,
failure_retryable: failure_retryable != 0,
next_attempt_unix_seconds: next_attempt_unix_seconds
.map(|value| {
u64::try_from(value).map_err(|_| {
bat_core::Error::Serialization("翻译任务 next_attempt 时间无效".to_string())
})
})
.transpose()?,
})
}
}
@@ -1219,6 +1696,140 @@ mod tests {
assert_eq!(retrievable[0].attempt_count, 1);
}
#[tokio::test]
async fn sqlite_translation_tasks_recover_expired_leases_for_retry() {
let temp = tempfile::TempDir::new().unwrap();
let repository =
SqliteTranslationTaskRepository::new(temp.path().join("translation-tasks.sqlite"))
.await
.unwrap();
let queue = queue(vec![task(
"task-a",
"Bundles/a.bundle",
OfficialTextUnitTaskStatus::QueuedOffline,
Some(OfficialParseStatus::Parsed),
None,
)]);
repository.sync_queue(&queue).await.unwrap();
let claimed = repository
.claim_next("worker-a", "mock", 1, 3)
.await
.unwrap()
.unwrap();
assert_eq!(claimed.task_status, TranslationTaskStatus::Running);
assert_eq!(claimed.lease_owner.as_deref(), Some("worker-a"));
force_expire_lease(&repository, "task-a").await;
let recovered = repository
.recover_expired_leases(unix_seconds_now())
.await
.unwrap();
assert_eq!(recovered, 1);
let recovered_task = repository.find("task-a").await.unwrap();
assert_eq!(recovered_task.task_status, TranslationTaskStatus::Failed);
assert_eq!(
recovered_task.failure_reason.as_deref(),
Some("worker lease expired; task recovered")
);
assert_eq!(
recovered_task.failure_class.as_deref(),
Some("lease_expired")
);
assert!(recovered_task.failure_retryable);
assert_eq!(recovered_task.lease_owner, None);
let reclaimed = repository
.claim_next("worker-b", "mock", 30, 3)
.await
.unwrap()
.unwrap();
assert_eq!(reclaimed.task_status, TranslationTaskStatus::Running);
assert_eq!(reclaimed.attempt_count, 2);
assert_eq!(reclaimed.lease_owner.as_deref(), Some("worker-b"));
}
#[tokio::test]
async fn sqlite_translation_tasks_reject_stale_worker_writes() {
let temp = tempfile::TempDir::new().unwrap();
let repository =
SqliteTranslationTaskRepository::new(temp.path().join("translation-tasks.sqlite"))
.await
.unwrap();
let queue = queue(vec![task(
"task-a",
"Bundles/a.bundle",
OfficialTextUnitTaskStatus::QueuedOffline,
Some(OfficialParseStatus::Parsed),
None,
)]);
repository.sync_queue(&queue).await.unwrap();
let first_claim = repository
.claim_next("worker-a", "mock", 1, 3)
.await
.unwrap()
.unwrap();
let first_run = first_claim.provider_run_id.clone().unwrap();
force_expire_lease(&repository, "task-a").await;
repository
.recover_expired_leases(unix_seconds_now())
.await
.unwrap();
let second_claim = repository
.claim_next("worker-b", "mock", 30, 3)
.await
.unwrap()
.unwrap();
let second_run = second_claim.provider_run_id.clone().unwrap();
let result = TranslationTaskUnitResult {
unit_id: "unit-a".to_string(),
source_text: "source".to_string(),
translated_text: "translated".to_string(),
provider: "mock".to_string(),
provider_run_id: second_run.clone(),
translated_unix_seconds: 1,
};
assert!(repository
.complete_claim(
"task-a",
"worker-a",
&first_run,
"mock",
std::slice::from_ref(&result)
)
.await
.is_err());
let completed = repository
.complete_claim(
"task-a",
"worker-b",
&second_run,
"mock",
std::slice::from_ref(&result),
)
.await
.unwrap();
assert_eq!(completed.task_status, TranslationTaskStatus::Completed);
assert_eq!(completed.translation_results, vec![result.clone()]);
assert!(repository
.fail_claim(TranslationTaskFailure {
task_id: "task-a".to_string(),
worker_id: "worker-a".to_string(),
provider_run_id: first_run,
failure_class: "network".to_string(),
failure_reason: "late failure".to_string(),
retryable: true,
next_attempt_unix_seconds: Some(2),
})
.await
.is_err());
let task = repository.find("task-a").await.unwrap();
assert_eq!(task.task_status, TranslationTaskStatus::Completed);
assert_eq!(task.translation_results, vec![result]);
}
#[test]
fn translation_handoff_maps_worker_state_and_preserves_provider_progress() {
let queue = queue(vec![
@@ -1254,6 +1865,13 @@ mod tests {
updated_unix_seconds: 124,
completed_unix_seconds: None,
provider_run_id: Some("provider-run-1".to_string()),
translation_results: Vec::new(),
provider: Some("fixture".to_string()),
lease_owner: None,
lease_expires_unix_seconds: None,
failure_class: None,
failure_retryable: false,
next_attempt_unix_seconds: None,
},
PersistedTranslationTask {
task: queue.tasks[1].clone(),
@@ -1264,6 +1882,13 @@ mod tests {
updated_unix_seconds: 125,
completed_unix_seconds: Some(125),
provider_run_id: Some("provider-run-1".to_string()),
translation_results: Vec::new(),
provider: Some("fixture".to_string()),
lease_owner: None,
lease_expires_unix_seconds: None,
failure_class: None,
failure_retryable: false,
next_attempt_unix_seconds: None,
},
];
@@ -1286,6 +1911,20 @@ mod tests {
assert_eq!(handoff.provider_runs[0].attempt_count, 2);
}
async fn force_expire_lease(repository: &SqliteTranslationTaskRepository, task_id: &str) {
sqlx::query(
r#"
UPDATE translation_tasks
SET lease_expires_unix_seconds = 0
WHERE task_id = ?1
"#,
)
.bind(task_id)
.execute(&repository.pool)
.await
.unwrap();
}
#[test]
fn translation_handoff_file_round_trips_with_version_check() {
let temp = tempfile::TempDir::new().unwrap();
File diff suppressed because it is too large Load Diff