mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-09-18 06:34:54 +08:00
@@ -23,14 +23,16 @@ use bat_infrastructure::{
|
||||
OfficialUpdateService, OfficialUpdateSnapshot, OfficialUpdateStatus,
|
||||
OfficialVerificationSummary, OfficialVersionRecord, OfficialVersionState, PatchApplyKind,
|
||||
PatchApplyParams, PatchApplyReport, ReleaseFlowStatusCode, RepackReport,
|
||||
SqliteResourceRepository, SqliteTranslationTaskRepository, TranslationTaskStatus,
|
||||
UnityFsFieldPatchParams, UnityFsPatchReport, UnityFsStringFieldPatchParams,
|
||||
UnityFsTextAssetPatchParams, CROWDIN_TEXTUNIT_QUEUE_FILE, LOCALIZED_CURRENT_LINK,
|
||||
SqliteResourceRepository, SqliteTranslationTaskRepository, TranslationProviderKind,
|
||||
TranslationTaskStatus, TranslationWorkerConfig, UnityFsFieldPatchParams, UnityFsPatchReport,
|
||||
UnityFsStringFieldPatchParams, UnityFsTextAssetPatchParams, CROWDIN_TEXTUNIT_QUEUE_FILE,
|
||||
DEFAULT_TRANSLATION_CONCURRENCY, DEFAULT_TRANSLATION_LEASE_SECONDS,
|
||||
DEFAULT_TRANSLATION_MAX_ATTEMPTS, DEFAULT_TRANSLATION_RETRY_BACKOFF, LOCALIZED_CURRENT_LINK,
|
||||
LOCALIZED_PATCH_MANIFEST_FILE, LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING,
|
||||
LOCALIZED_TRANSLATION_STATUS_MANUAL_PROOFREADING_LABEL, LOCALIZED_VERSIONS_DIR,
|
||||
LOCALIZED_VERSION_STATE_FILE, MAX_DOWNLOAD_CONCURRENCY, MIN_DOWNLOAD_CONCURRENCY,
|
||||
OFFICIAL_PARSE_CACHE_FILE, OFFICIAL_TEXTUNIT_INDEX_FILE, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE,
|
||||
PRIVATE_FILE_MODE,
|
||||
LOCALIZED_VERSION_STATE_FILE, MAX_DOWNLOAD_CONCURRENCY, MAX_TRANSLATION_CONCURRENCY,
|
||||
MIN_DOWNLOAD_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY, OFFICIAL_PARSE_CACHE_FILE,
|
||||
OFFICIAL_TEXTUNIT_INDEX_FILE, OFFICIAL_TEXTUNIT_TASK_QUEUE_FILE, PRIVATE_FILE_MODE,
|
||||
};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -85,6 +87,7 @@ use workflow_commands::{
|
||||
run_parse_clear_cache, run_parse_once, run_publish_localized, run_repack, run_translate_once,
|
||||
run_translation_get, run_translation_proofread, run_translation_set,
|
||||
run_translation_task_update, run_translation_unset, run_translation_validate,
|
||||
run_translation_worker,
|
||||
};
|
||||
|
||||
const EXIT_ERROR: i32 = 1;
|
||||
@@ -229,6 +232,10 @@ fn run() -> anyhow::Result<i32> {
|
||||
run_translation_proofread(&options)?;
|
||||
Ok(0)
|
||||
}
|
||||
CliCommand::TranslationWorker => {
|
||||
run_repeated_workflow(&options, "translation-worker", run_translation_worker)?;
|
||||
Ok(0)
|
||||
}
|
||||
CliCommand::Repack => {
|
||||
run_repack(&options)?;
|
||||
Ok(0)
|
||||
@@ -378,6 +385,15 @@ struct CliOptions {
|
||||
translation_text_file: Option<PathBuf>,
|
||||
translation_failure_reason: Option<String>,
|
||||
translation_provider_run_id: Option<String>,
|
||||
translation_provider: Option<String>,
|
||||
translation_fixture: Option<PathBuf>,
|
||||
worker_concurrency: usize,
|
||||
worker_max_attempts: u32,
|
||||
worker_lease_seconds: u64,
|
||||
worker_retry_backoff: Duration,
|
||||
worker_max_tasks: Option<usize>,
|
||||
worker_id: Option<String>,
|
||||
translation_worker_option_explicit: bool,
|
||||
localized_release_id: Option<String>,
|
||||
repack_spec: Option<PathBuf>,
|
||||
schedule_group: Option<String>,
|
||||
@@ -463,6 +479,15 @@ impl Default for CliOptions {
|
||||
translation_text_file: None,
|
||||
translation_failure_reason: None,
|
||||
translation_provider_run_id: None,
|
||||
translation_provider: None,
|
||||
translation_fixture: None,
|
||||
worker_concurrency: DEFAULT_TRANSLATION_CONCURRENCY,
|
||||
worker_max_attempts: DEFAULT_TRANSLATION_MAX_ATTEMPTS,
|
||||
worker_lease_seconds: DEFAULT_TRANSLATION_LEASE_SECONDS,
|
||||
worker_retry_backoff: DEFAULT_TRANSLATION_RETRY_BACKOFF,
|
||||
worker_max_tasks: None,
|
||||
worker_id: None,
|
||||
translation_worker_option_explicit: false,
|
||||
localized_release_id: None,
|
||||
repack_spec: None,
|
||||
schedule_group: None,
|
||||
@@ -546,6 +571,7 @@ enum CliCommand {
|
||||
TranslationGet,
|
||||
TranslationUnset,
|
||||
TranslationTaskUpdate,
|
||||
TranslationWorker,
|
||||
TranslationProofread,
|
||||
Repack,
|
||||
PublishLocalized,
|
||||
@@ -1025,6 +1051,7 @@ const RPC_METHOD_TRANSLATION_TASKS: &str = "translation.tasks";
|
||||
const RPC_METHOD_TRANSLATION_HANDOFF: &str = "translation.handoff";
|
||||
const RPC_METHOD_TRANSLATION_TASK_UPDATE: &str = "translation.task.update";
|
||||
const RPC_METHOD_TRANSLATION_PROOFREAD: &str = "translation.proofread";
|
||||
const RPC_METHOD_TRANSLATION_WORKER_RUN: &str = "translation.worker.run";
|
||||
const RPC_METHOD_LOCALIZED_STATUS: &str = "localized.status";
|
||||
const RPC_METHOD_CATALOG_STATUS: &str = "catalog.status";
|
||||
const RPC_METHOD_CATALOG_VERSIONS: &str = "catalog.versions";
|
||||
@@ -2027,6 +2054,13 @@ fn dispatch_rpc_method(
|
||||
mark_localized_manual_proofreading_report(state_dir, &tasks.base_config)
|
||||
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from)),
|
||||
),
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN => {
|
||||
let config = match rpc_translation_worker_config(request.params.as_ref()) {
|
||||
Ok(config) => config,
|
||||
Err(error) => return rpc_envelope_error(request_id, error),
|
||||
};
|
||||
enqueue_translation_worker_envelope(tasks, config, request_id)
|
||||
}
|
||||
RPC_METHOD_LOCALIZED_STATUS => rpc_envelope_from_result(
|
||||
request_id,
|
||||
"localized.status",
|
||||
@@ -2211,7 +2245,9 @@ fn enqueue_task_envelope(
|
||||
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
|
||||
let job = TaskJob {
|
||||
id: task_id.clone(),
|
||||
kind,
|
||||
config,
|
||||
translation_worker_config: None,
|
||||
cancel,
|
||||
};
|
||||
if tasks.queue.send(job).is_err() {
|
||||
@@ -2237,6 +2273,50 @@ fn enqueue_task_envelope(
|
||||
)
|
||||
}
|
||||
|
||||
fn enqueue_translation_worker_envelope(
|
||||
tasks: &DaemonTaskContext,
|
||||
worker_config: TranslationWorkerConfig,
|
||||
request_id: String,
|
||||
) -> RpcEnvelope {
|
||||
let config = TaskKind::TranslationWorker.build_config(&tasks.base_config, false);
|
||||
let task_id = tasks.registry.create(TaskKind::TranslationWorker);
|
||||
let cancel = tasks
|
||||
.registry
|
||||
.cancel_flag(&task_id)
|
||||
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
|
||||
let job = TaskJob {
|
||||
id: task_id.clone(),
|
||||
kind: TaskKind::TranslationWorker,
|
||||
config,
|
||||
translation_worker_config: Some(worker_config.clone()),
|
||||
cancel,
|
||||
};
|
||||
if tasks.queue.send(job).is_err() {
|
||||
tasks.registry.update(&task_id, |record| {
|
||||
record.status = "failed";
|
||||
record.finished_at = Some(unix_seconds_now());
|
||||
record.error = Some(ApiError::new(
|
||||
ErrorCode::INTERNAL,
|
||||
"task.enqueue",
|
||||
"任务执行器不可用",
|
||||
));
|
||||
});
|
||||
return rpc_envelope_error(
|
||||
request_id,
|
||||
ApiError::new(ErrorCode::INTERNAL, "task.enqueue", "任务执行器不可用"),
|
||||
);
|
||||
}
|
||||
rpc_envelope_ok(
|
||||
request_id,
|
||||
"accepted",
|
||||
serde_json::json!({
|
||||
"task_id": task_id,
|
||||
"kind": RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
"worker": worker_config,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn rpc_ack_value(
|
||||
command: &'static str,
|
||||
message: &'static str,
|
||||
@@ -3070,6 +3150,207 @@ fn rpc_struct_params<T: DeserializeOwned>(
|
||||
})
|
||||
}
|
||||
|
||||
fn rpc_translation_worker_config(
|
||||
params: Option<&serde_json::Value>,
|
||||
) -> Result<TranslationWorkerConfig, ApiError> {
|
||||
let empty = serde_json::json!({});
|
||||
let params = params.unwrap_or(&empty);
|
||||
if !params.is_object() {
|
||||
return Err(ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
"params 必须是 JSON object",
|
||||
));
|
||||
}
|
||||
let provider = rpc_translation_worker_string_param(
|
||||
params,
|
||||
&["provider", "translation_provider"],
|
||||
"provider",
|
||||
)?
|
||||
.unwrap_or_else(|| TranslationProviderKind::Mock.as_str().to_string());
|
||||
let provider = TranslationProviderKind::parse(&provider).ok_or_else(|| {
|
||||
ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("不支持的 translation provider:{provider}"),
|
||||
)
|
||||
})?;
|
||||
let concurrency = rpc_translation_worker_usize_param(
|
||||
params,
|
||||
&[
|
||||
"concurrency",
|
||||
"worker_concurrency",
|
||||
"translation_concurrency",
|
||||
],
|
||||
"concurrency",
|
||||
)?
|
||||
.unwrap_or(DEFAULT_TRANSLATION_CONCURRENCY);
|
||||
let max_attempts = rpc_translation_worker_u32_param(
|
||||
params,
|
||||
&[
|
||||
"max_attempts",
|
||||
"worker_max_attempts",
|
||||
"translation_max_attempts",
|
||||
],
|
||||
"max_attempts",
|
||||
)?
|
||||
.unwrap_or(DEFAULT_TRANSLATION_MAX_ATTEMPTS);
|
||||
let lease_seconds = rpc_translation_worker_u64_param(
|
||||
params,
|
||||
&[
|
||||
"lease_seconds",
|
||||
"worker_lease_seconds",
|
||||
"translation_lease_seconds",
|
||||
],
|
||||
"lease_seconds",
|
||||
)?
|
||||
.unwrap_or(DEFAULT_TRANSLATION_LEASE_SECONDS);
|
||||
let retry_backoff_seconds = rpc_translation_worker_u64_param(
|
||||
params,
|
||||
&[
|
||||
"retry_backoff_seconds",
|
||||
"worker_retry_backoff_seconds",
|
||||
"translation_retry_backoff_seconds",
|
||||
],
|
||||
"retry_backoff_seconds",
|
||||
)?
|
||||
.unwrap_or(DEFAULT_TRANSLATION_RETRY_BACKOFF.as_secs());
|
||||
let max_tasks = rpc_translation_worker_usize_param(
|
||||
params,
|
||||
&["max_tasks", "worker_max_tasks", "translation_max_tasks"],
|
||||
"max_tasks",
|
||||
)?;
|
||||
let config = TranslationWorkerConfig {
|
||||
provider,
|
||||
fixture_path: rpc_translation_worker_string_param(
|
||||
params,
|
||||
&[
|
||||
"fixture_path",
|
||||
"translation_fixture",
|
||||
"provider_fixture",
|
||||
"mock_fixture",
|
||||
"fixture",
|
||||
],
|
||||
"fixture_path",
|
||||
)?
|
||||
.map(PathBuf::from),
|
||||
concurrency,
|
||||
max_attempts,
|
||||
lease_seconds,
|
||||
retry_backoff: Duration::from_secs(retry_backoff_seconds),
|
||||
max_tasks,
|
||||
worker_id: rpc_translation_worker_string_param(
|
||||
params,
|
||||
&["worker_id", "translation_worker_id"],
|
||||
"worker_id",
|
||||
)?
|
||||
.unwrap_or_else(|| "bat-rpc-worker".to_string()),
|
||||
};
|
||||
config.validate().map_err(|error| {
|
||||
ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
error.to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn rpc_translation_worker_string_param(
|
||||
params: &serde_json::Value,
|
||||
aliases: &[&str],
|
||||
label: &str,
|
||||
) -> Result<Option<String>, ApiError> {
|
||||
for key in aliases {
|
||||
let Some(value) = params.get(*key) else {
|
||||
continue;
|
||||
};
|
||||
if value.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(text) = value.as_str() else {
|
||||
return Err(ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("{label} 必须是字符串"),
|
||||
));
|
||||
};
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(Some(text.to_string()));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn rpc_translation_worker_u64_param(
|
||||
params: &serde_json::Value,
|
||||
aliases: &[&str],
|
||||
label: &str,
|
||||
) -> Result<Option<u64>, ApiError> {
|
||||
for key in aliases {
|
||||
let Some(value) = params.get(*key) else {
|
||||
continue;
|
||||
};
|
||||
if value.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
if value.as_i64().is_some_and(|number| number < 0) {
|
||||
return Err(ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("{label} 不能为负数"),
|
||||
));
|
||||
}
|
||||
let Some(number) = value.as_u64() else {
|
||||
return Err(ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("{label} 必须是非负整数 JSON number"),
|
||||
));
|
||||
};
|
||||
return Ok(Some(number));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn rpc_translation_worker_usize_param(
|
||||
params: &serde_json::Value,
|
||||
aliases: &[&str],
|
||||
label: &str,
|
||||
) -> Result<Option<usize>, ApiError> {
|
||||
rpc_translation_worker_u64_param(params, aliases, label)?
|
||||
.map(|number| {
|
||||
usize::try_from(number).map_err(|error| {
|
||||
ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("{label} 无效:{error}"),
|
||||
)
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn rpc_translation_worker_u32_param(
|
||||
params: &serde_json::Value,
|
||||
aliases: &[&str],
|
||||
label: &str,
|
||||
) -> Result<Option<u32>, ApiError> {
|
||||
rpc_translation_worker_u64_param(params, aliases, label)?
|
||||
.map(|number| {
|
||||
u32::try_from(number).map_err(|error| {
|
||||
ApiError::new(
|
||||
ErrorCode::RPC_INVALID_PARAMS,
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
format!("{label} 无效:{error}"),
|
||||
)
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// 分页参数:`offset` 默认 0;`limit` 默认 100,范围 1..=1000。
|
||||
fn rpc_page_params(params: Option<&serde_json::Value>) -> anyhow::Result<(usize, usize)> {
|
||||
let offset = params
|
||||
@@ -4221,6 +4502,32 @@ impl HumanReport for bat_infrastructure::LocalizedTranslationWorkflowReport {
|
||||
}
|
||||
}
|
||||
|
||||
impl HumanReport for bat_infrastructure::TranslationWorkerReport {
|
||||
fn print_human(&self) -> anyhow::Result<()> {
|
||||
print_title("翻译 provider worker");
|
||||
print_field("命令", self.command);
|
||||
print_field("状态", self.status);
|
||||
print_field("官方 release", &self.official_release_id);
|
||||
print_field("provider", &self.provider);
|
||||
print_field("回收 lease", self.recovered_lease_count);
|
||||
print_field("领取任务", self.claimed_count);
|
||||
print_field("完成任务", self.completed_count);
|
||||
print_field("失败任务", self.failed_count);
|
||||
print_field("已安排重试", self.retry_scheduled_count);
|
||||
print_field("剩余任务", self.remaining_count);
|
||||
for failure in &self.failures {
|
||||
println!(
|
||||
" - {} [{}] retryable={} {}",
|
||||
failure.task_id,
|
||||
failure.failure_class,
|
||||
format_bool(failure.retryable),
|
||||
failure.failure_reason
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn print_human_json_value(value: &serde_json::Value) -> anyhow::Result<()> {
|
||||
if value.get("running").is_some() && value.get("state_dir").is_some() {
|
||||
print_title("后台状态");
|
||||
@@ -6321,6 +6628,17 @@ BAT_AUTO_DISCOVER=1
|
||||
#BAT_DOWNLOAD_CONCURRENCY=8
|
||||
#BAT_UNZIP=unzip
|
||||
|
||||
# ---- 翻译 provider worker ----
|
||||
# provider:mock(本地 fixture)或 crowdin(读取 CROWDIN_* 环境变量)
|
||||
#BAT_TRANSLATION_PROVIDER=mock
|
||||
#BAT_TRANSLATION_FIXTURE=
|
||||
#BAT_TRANSLATION_CONCURRENCY=8
|
||||
#BAT_TRANSLATION_MAX_ATTEMPTS=3
|
||||
#BAT_TRANSLATION_LEASE_SECONDS=300
|
||||
#BAT_TRANSLATION_RETRY_BACKOFF_SECONDS=5
|
||||
#BAT_TRANSLATION_MAX_TASKS=
|
||||
#BAT_TRANSLATION_WORKER_ID=
|
||||
|
||||
# ---- 输出 ----
|
||||
# 设为 1 时输出机器可读 JSON(默认人类可读)
|
||||
#BAT_JSON=0
|
||||
@@ -6504,6 +6822,38 @@ fn apply_bat_env_overrides(
|
||||
if let Some(v) = value("BAT_UNZIP") {
|
||||
options.config.unzip_command = PathBuf::from(v);
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_PROVIDER") {
|
||||
options.translation_provider = Some(v);
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_FIXTURE") {
|
||||
options.translation_fixture = Some(PathBuf::from(v));
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_CONCURRENCY") {
|
||||
options.worker_concurrency =
|
||||
parse_translation_worker_concurrency(&v, "环境变量 BAT_TRANSLATION_CONCURRENCY")?;
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_MAX_ATTEMPTS") {
|
||||
options.worker_max_attempts =
|
||||
parse_positive_u32(&v, "环境变量 BAT_TRANSLATION_MAX_ATTEMPTS")?;
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_LEASE_SECONDS") {
|
||||
options.worker_lease_seconds =
|
||||
parse_positive_u64(&v, "环境变量 BAT_TRANSLATION_LEASE_SECONDS")?;
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_RETRY_BACKOFF_SECONDS") {
|
||||
options.worker_retry_backoff = Duration::from_secs(v.parse::<u64>().map_err(|error| {
|
||||
anyhow::anyhow!("环境变量 BAT_TRANSLATION_RETRY_BACKOFF_SECONDS 的秒数无效:{error}")
|
||||
})?);
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_MAX_TASKS") {
|
||||
options.worker_max_tasks = Some(parse_positive_usize(
|
||||
&v,
|
||||
"环境变量 BAT_TRANSLATION_MAX_TASKS",
|
||||
)?);
|
||||
}
|
||||
if let Some(v) = value("BAT_TRANSLATION_WORKER_ID") {
|
||||
options.worker_id = Some(v);
|
||||
}
|
||||
if let Some(v) = value("BAT_PROXY") {
|
||||
options.config.curl_proxy = parse_proxy_config(&v)?;
|
||||
}
|
||||
@@ -6725,6 +7075,55 @@ fn parse_args_with_env(
|
||||
"--provider-run-id" => {
|
||||
options.translation_provider_run_id = Some(next_option_value(&mut args, &flag)?);
|
||||
}
|
||||
"--translation-provider" | "--provider" => {
|
||||
options.translation_provider = Some(next_option_value(&mut args, &flag)?);
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--translation-fixture" | "--provider-fixture" | "--mock-fixture" | "--fixture" => {
|
||||
options.translation_fixture =
|
||||
Some(PathBuf::from(next_option_value(&mut args, &flag)?));
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-concurrency" | "--translation-concurrency" => {
|
||||
options.worker_concurrency = parse_translation_worker_concurrency(
|
||||
&next_option_value(&mut args, &flag)?,
|
||||
&flag,
|
||||
)?;
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-max-attempts" | "--translation-max-attempts" => {
|
||||
options.worker_max_attempts =
|
||||
parse_positive_u32(&next_option_value(&mut args, &flag)?, &flag)?;
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-lease-seconds" | "--translation-lease-seconds" => {
|
||||
options.worker_lease_seconds =
|
||||
parse_positive_u64(&next_option_value(&mut args, &flag)?, &flag)?;
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-retry-backoff" | "--translation-retry-backoff" => {
|
||||
options.worker_retry_backoff =
|
||||
parse_duration(&next_option_value(&mut args, &flag)?)?;
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-retry-backoff-seconds" | "--translation-retry-backoff-seconds" => {
|
||||
let seconds = next_option_value(&mut args, &flag)?
|
||||
.parse::<u64>()
|
||||
.map_err(|error| anyhow::anyhow!("{flag} 的秒数无效:{error}"))?;
|
||||
options.worker_retry_backoff = Duration::from_secs(seconds);
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-max-tasks" | "--translation-max-tasks" => {
|
||||
options.worker_max_tasks = Some(parse_positive_usize(
|
||||
&next_option_value(&mut args, &flag)?,
|
||||
&flag,
|
||||
)?);
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--worker-id" | "--translation-worker-id" => {
|
||||
options.worker_id = Some(next_option_value(&mut args, &flag)?);
|
||||
options.translation_worker_option_explicit = true;
|
||||
}
|
||||
"--localized-release-id" => {
|
||||
options.localized_release_id = Some(next_option_value(&mut args, &flag)?);
|
||||
}
|
||||
@@ -7185,6 +7584,12 @@ fn parse_args_with_env(
|
||||
));
|
||||
}
|
||||
|
||||
if options.command != CliCommand::TranslationWorker
|
||||
&& options.translation_worker_option_explicit
|
||||
{
|
||||
return Err(anyhow::anyhow!("翻译 worker 参数只适用于 i18n worker run"));
|
||||
}
|
||||
|
||||
match options.command {
|
||||
CliCommand::Status | CliCommand::Stop | CliCommand::Logs => {
|
||||
if options.sync_option_explicit
|
||||
@@ -7406,6 +7811,56 @@ fn parse_args_with_env(
|
||||
options.progress = false;
|
||||
options.banner = false;
|
||||
}
|
||||
CliCommand::TranslationWorker => {
|
||||
if options.daemon || options.daemon_child {
|
||||
return Err(anyhow::anyhow!(
|
||||
"i18n worker run 使用 --watch 或 --run-count,不支持 daemon"
|
||||
));
|
||||
}
|
||||
if options.config.force
|
||||
|| options.config.dry_run
|
||||
|| options.schedule_option_explicit
|
||||
|| options.query_option_explicit
|
||||
|| options.write_patch_option_explicit
|
||||
|| options.translation_file.is_some()
|
||||
|| options.translation_id.is_some()
|
||||
|| options.translation_text.is_some()
|
||||
|| options.translation_text_file.is_some()
|
||||
|| options.translation_failure_reason.is_some()
|
||||
|| options.translation_provider_run_id.is_some()
|
||||
|| options.localized_release_id.is_some()
|
||||
|| options.repack_spec.is_some()
|
||||
|| options.proxy_option_explicit
|
||||
|| tools_are_non_default(&options.config, &options.env_baseline_config)
|
||||
{
|
||||
return Err(anyhow::anyhow!(
|
||||
"i18n worker run 只接受 --output、--resource-root、--state-dir、worker 参数和轮询参数"
|
||||
));
|
||||
}
|
||||
let provider = options
|
||||
.translation_provider
|
||||
.as_deref()
|
||||
.unwrap_or(TranslationProviderKind::Mock.as_str());
|
||||
if TranslationProviderKind::parse(provider).is_none() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"i18n worker run 的 provider 无效:{provider}"
|
||||
));
|
||||
}
|
||||
TranslationWorkerConfig {
|
||||
provider: TranslationProviderKind::parse(provider).expect("provider 已在上方校验"),
|
||||
fixture_path: options.translation_fixture.clone(),
|
||||
concurrency: options.worker_concurrency,
|
||||
max_attempts: options.worker_max_attempts,
|
||||
lease_seconds: options.worker_lease_seconds,
|
||||
retry_backoff: options.worker_retry_backoff,
|
||||
max_tasks: options.worker_max_tasks,
|
||||
worker_id: options
|
||||
.worker_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "bat-worker-validation".to_string()),
|
||||
}
|
||||
.validate()?;
|
||||
}
|
||||
CliCommand::TranslationProofread => {
|
||||
if options.watch || options.daemon || options.daemon_child {
|
||||
return Err(anyhow::anyhow!("i18n proofread 只支持单次执行或 RPC 调用"));
|
||||
@@ -7506,6 +7961,7 @@ fn parse_args_with_env(
|
||||
CliCommand::Pull
|
||||
| CliCommand::Parse
|
||||
| CliCommand::Translate
|
||||
| CliCommand::TranslationWorker
|
||||
| CliCommand::PublishLocalized
|
||||
)
|
||||
{
|
||||
@@ -7515,7 +7971,11 @@ fn parse_args_with_env(
|
||||
}
|
||||
if matches!(
|
||||
options.command,
|
||||
CliCommand::Pull | CliCommand::Parse | CliCommand::Translate | CliCommand::PublishLocalized
|
||||
CliCommand::Pull
|
||||
| CliCommand::Parse
|
||||
| CliCommand::Translate
|
||||
| CliCommand::PublishLocalized
|
||||
| CliCommand::TranslationWorker
|
||||
) {
|
||||
if options.watch && options.run_count.is_some() {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -7632,6 +8092,9 @@ fn parse_translation_command(
|
||||
if action == "workbench" || action == "wb" {
|
||||
return parse_translation_workbench_command(args, options);
|
||||
}
|
||||
if action == "worker" {
|
||||
return parse_translation_worker_command(args, options);
|
||||
}
|
||||
let command = match action.as_str() {
|
||||
"run" => CliCommand::Translate,
|
||||
"export" => CliCommand::Translate,
|
||||
@@ -7655,6 +8118,20 @@ fn parse_translation_command(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_translation_worker_command(
|
||||
args: &mut impl Iterator<Item = String>,
|
||||
options: &mut CliOptions,
|
||||
) -> anyhow::Result<()> {
|
||||
let action = next_option_value(args, "translation worker")?;
|
||||
let command = match action.as_str() {
|
||||
"run" => CliCommand::TranslationWorker,
|
||||
other => return Err(anyhow::anyhow!("未知 translation worker 二级命令:{other}")),
|
||||
};
|
||||
ensure_command_not_set(options.command, &format!("translation worker {action}"))?;
|
||||
options.command = command;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_translation_workbench_command(
|
||||
args: &mut impl Iterator<Item = String>,
|
||||
options: &mut CliOptions,
|
||||
@@ -7738,6 +8215,48 @@ fn parse_download_concurrency(value: &str, source: &str) -> anyhow::Result<usize
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_translation_worker_concurrency(value: &str, source: &str) -> anyhow::Result<usize> {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
|
||||
if !(MIN_TRANSLATION_CONCURRENCY..=MAX_TRANSLATION_CONCURRENCY).contains(&parsed) {
|
||||
return Err(anyhow::anyhow!(
|
||||
"{source} 必须在 {MIN_TRANSLATION_CONCURRENCY}..={MAX_TRANSLATION_CONCURRENCY} 范围内"
|
||||
));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_positive_usize(value: &str, source: &str) -> anyhow::Result<usize> {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow::anyhow!("{source} 必须大于 0"));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_positive_u32(value: &str, source: &str) -> anyhow::Result<u32> {
|
||||
let parsed = value
|
||||
.parse::<u32>()
|
||||
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow::anyhow!("{source} 必须大于 0"));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_positive_u64(value: &str, source: &str) -> anyhow::Result<u64> {
|
||||
let parsed = value
|
||||
.parse::<u64>()
|
||||
.map_err(|error| anyhow::anyhow!("{source} 无效:{error}"))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow::anyhow!("{source} 必须大于 0"));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn next_option_value(
|
||||
args: &mut impl Iterator<Item = String>,
|
||||
flag: &str,
|
||||
@@ -7807,6 +8326,7 @@ fn print_usage(binary: &str) {
|
||||
eprintln!(" i18n unset Clear one translated workbench entry");
|
||||
eprintln!(" i18n validate Validate workbench against the current official release");
|
||||
eprintln!(" i18n proofread Mark localized workflow as manual proofreading");
|
||||
eprintln!(" i18n worker run Run translation provider worker once or repeatedly");
|
||||
eprintln!(
|
||||
" i18n tasks / i18n task list / i18n task status Query current offline TextUnit translation task status"
|
||||
);
|
||||
@@ -7854,6 +8374,9 @@ fn print_usage(binary: &str) {
|
||||
" {binary} i18n unset --translation-file /tmp/bat-workbench.json --translation-id unit-1"
|
||||
);
|
||||
eprintln!(" {binary} i18n proofread --json");
|
||||
eprintln!(
|
||||
" {binary} i18n worker run --provider mock --worker-concurrency 8 --run-count 2 --interval 30s"
|
||||
);
|
||||
eprintln!(" {binary} i18n tasks --json");
|
||||
eprintln!(" {binary} i18n handoff --json");
|
||||
eprintln!(" {binary} i18n status --json");
|
||||
@@ -7908,6 +8431,17 @@ fn print_usage(binary: &str) {
|
||||
eprintln!(" --translated-file <PATH> UTF-8 translation file for i18n set");
|
||||
eprintln!(" --failure-reason <TEXT> Provider failure reason for i18n task update");
|
||||
eprintln!(" --provider-run-id <ID> Provider run ID for i18n task update");
|
||||
eprintln!(" --translation-provider <NAME> / --provider <NAME> Provider for i18n worker run (mock/crowdin)");
|
||||
eprintln!(" --translation-fixture <PATH> Mock/provider fixture for i18n worker run");
|
||||
eprintln!(
|
||||
" --worker-concurrency <N> Translation worker concurrency (default: 8, range 1..=256)"
|
||||
);
|
||||
eprintln!(" --worker-max-attempts <N> Maximum claims per translation task");
|
||||
eprintln!(" --worker-lease-seconds <N> Lease seconds for one claimed task");
|
||||
eprintln!(" --worker-retry-backoff <DURATION> Retry backoff after retryable failure");
|
||||
eprintln!(" --worker-retry-backoff-seconds <N> Retry backoff seconds");
|
||||
eprintln!(" --worker-max-tasks <N> Maximum tasks claimed in one worker run");
|
||||
eprintln!(" --worker-id <ID> Worker ID prefix for lease diagnostics");
|
||||
eprintln!(" --localized-release-id <ID> Explicit localized publication ID");
|
||||
eprintln!(" --repack-spec <PATH> UnityFS batch repack JSON spec");
|
||||
eprintln!();
|
||||
|
||||
@@ -140,6 +140,165 @@ fn env_invalid_values_error() {
|
||||
assert!(parse(&["bat", "--download-concurrency", "257"]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_worker_command_options_are_validated() {
|
||||
let options = parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"worker",
|
||||
"run",
|
||||
"--provider",
|
||||
"mock",
|
||||
"--translation-fixture",
|
||||
"/tmp/mock-translation.json",
|
||||
"--worker-concurrency",
|
||||
"8",
|
||||
"--worker-max-attempts",
|
||||
"4",
|
||||
"--worker-lease-seconds",
|
||||
"60",
|
||||
"--worker-retry-backoff",
|
||||
"0s",
|
||||
"--worker-max-tasks",
|
||||
"2",
|
||||
"--worker-id",
|
||||
"manual-run",
|
||||
"--run-count",
|
||||
"2",
|
||||
"--interval",
|
||||
"30s",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(options.command, CliCommand::TranslationWorker);
|
||||
assert_eq!(options.translation_provider.as_deref(), Some("mock"));
|
||||
assert_eq!(
|
||||
options.translation_fixture,
|
||||
Some(PathBuf::from("/tmp/mock-translation.json"))
|
||||
);
|
||||
assert_eq!(options.worker_concurrency, 8);
|
||||
assert_eq!(options.worker_max_attempts, 4);
|
||||
assert_eq!(options.worker_lease_seconds, 60);
|
||||
assert_eq!(options.worker_retry_backoff, Duration::ZERO);
|
||||
assert_eq!(options.worker_max_tasks, Some(2));
|
||||
assert_eq!(options.worker_id.as_deref(), Some("manual-run"));
|
||||
assert_eq!(options.run_count, Some(2));
|
||||
|
||||
assert!(parse(&["bat", "i18n", "worker", "run"]).is_ok());
|
||||
assert!(parse(&[
|
||||
"bat",
|
||||
"translation",
|
||||
"worker",
|
||||
"run",
|
||||
"--translation-concurrency",
|
||||
"256"
|
||||
])
|
||||
.is_ok());
|
||||
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-concurrency", "0"]).is_err());
|
||||
assert!(parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"worker",
|
||||
"run",
|
||||
"--worker-concurrency",
|
||||
"257"
|
||||
])
|
||||
.is_err());
|
||||
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-max-attempts", "0"]).is_err());
|
||||
assert!(parse(&[
|
||||
"bat",
|
||||
"i18n",
|
||||
"worker",
|
||||
"run",
|
||||
"--worker-lease-seconds",
|
||||
"0"
|
||||
])
|
||||
.is_err());
|
||||
assert!(parse(&["bat", "i18n", "worker", "run", "--worker-max-tasks", "0"]).is_err());
|
||||
assert!(parse(&["bat", "i18n", "run", "--worker-concurrency", "8"]).is_err());
|
||||
assert!(parse(&["bat", "i18n", "worker", "run", "--provider", "unknown"]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_worker_env_defaults_apply() {
|
||||
let options = parse_with_env(
|
||||
&["bat", "i18n", "worker", "run"],
|
||||
&[
|
||||
("BAT_TRANSLATION_PROVIDER", "mock"),
|
||||
("BAT_TRANSLATION_FIXTURE", "/tmp/fixture.json"),
|
||||
("BAT_TRANSLATION_CONCURRENCY", "16"),
|
||||
("BAT_TRANSLATION_MAX_ATTEMPTS", "5"),
|
||||
("BAT_TRANSLATION_LEASE_SECONDS", "120"),
|
||||
("BAT_TRANSLATION_RETRY_BACKOFF_SECONDS", "0"),
|
||||
("BAT_TRANSLATION_MAX_TASKS", "7"),
|
||||
("BAT_TRANSLATION_WORKER_ID", "env-worker"),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(options.command, CliCommand::TranslationWorker);
|
||||
assert_eq!(options.translation_provider.as_deref(), Some("mock"));
|
||||
assert_eq!(
|
||||
options.translation_fixture,
|
||||
Some(PathBuf::from("/tmp/fixture.json"))
|
||||
);
|
||||
assert_eq!(options.worker_concurrency, 16);
|
||||
assert_eq!(options.worker_max_attempts, 5);
|
||||
assert_eq!(options.worker_lease_seconds, 120);
|
||||
assert_eq!(options.worker_retry_backoff, Duration::ZERO);
|
||||
assert_eq!(options.worker_max_tasks, Some(7));
|
||||
assert_eq!(options.worker_id.as_deref(), Some("env-worker"));
|
||||
|
||||
assert!(parse_with_env(
|
||||
&["bat", "i18n", "worker", "run"],
|
||||
&[("BAT_TRANSLATION_CONCURRENCY", "257")]
|
||||
)
|
||||
.is_err());
|
||||
assert!(parse_with_env(
|
||||
&["bat", "i18n", "worker", "run"],
|
||||
&[("BAT_TRANSLATION_MAX_ATTEMPTS", "0")]
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_worker_rpc_params_are_strict_and_accept_aliases() {
|
||||
let default_config = rpc_translation_worker_config(None).unwrap();
|
||||
assert_eq!(default_config.provider, TranslationProviderKind::Mock);
|
||||
assert_eq!(default_config.concurrency, DEFAULT_TRANSLATION_CONCURRENCY);
|
||||
|
||||
let config = rpc_translation_worker_config(Some(&serde_json::json!({
|
||||
"translation_provider": "mock",
|
||||
"provider_fixture": "/tmp/mock-provider.json",
|
||||
"worker_concurrency": 8,
|
||||
"worker_max_attempts": 4,
|
||||
"worker_lease_seconds": 60,
|
||||
"worker_retry_backoff_seconds": 0,
|
||||
"worker_max_tasks": 2,
|
||||
"translation_worker_id": "rpc-worker"
|
||||
})))
|
||||
.unwrap();
|
||||
assert_eq!(config.provider, TranslationProviderKind::Mock);
|
||||
assert_eq!(
|
||||
config.fixture_path,
|
||||
Some(PathBuf::from("/tmp/mock-provider.json"))
|
||||
);
|
||||
assert_eq!(config.concurrency, 8);
|
||||
assert_eq!(config.max_attempts, 4);
|
||||
assert_eq!(config.lease_seconds, 60);
|
||||
assert_eq!(config.retry_backoff, Duration::ZERO);
|
||||
assert_eq!(config.max_tasks, Some(2));
|
||||
assert_eq!(config.worker_id, "rpc-worker");
|
||||
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!([]))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"provider": "bad"}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": "8"}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": -1}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": 0}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"concurrency": 257}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"max_attempts": 0}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"lease_seconds": 0}))).is_err());
|
||||
assert!(rpc_translation_worker_config(Some(&serde_json::json!({"max_tasks": 0}))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grouped_workflow_commands_use_short_top_level_aliases() {
|
||||
let options = parse(&[
|
||||
|
||||
@@ -4,12 +4,14 @@ pub(super) const MAX_RETAINED_TASKS: usize = 64;
|
||||
/// 每个任务保留的进度日志行数上限。
|
||||
pub(super) const MAX_TASK_LOG_LINES: usize = 200;
|
||||
|
||||
/// 任务类型:目前覆盖官方同步、校验与 catalog 更新检查。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
/// 任务类型:覆盖资源同步、校验、修复、翻译 worker 与 catalog 更新检查。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum TaskKind {
|
||||
Sync,
|
||||
Verify,
|
||||
Repair,
|
||||
/// 翻译 provider worker 轮次。
|
||||
TranslationWorker,
|
||||
/// catalog 更新检查:只做发现 + 拉取计划(dry-run),不下载不审计。
|
||||
Refresh,
|
||||
}
|
||||
@@ -20,6 +22,7 @@ impl TaskKind {
|
||||
Self::Sync => RPC_METHOD_RESOURCE_SYNC,
|
||||
Self::Verify => RPC_METHOD_RESOURCE_VERIFY,
|
||||
Self::Repair => RPC_METHOD_RESOURCE_REPAIR,
|
||||
Self::TranslationWorker => RPC_METHOD_TRANSLATION_WORKER_RUN,
|
||||
Self::Refresh => RPC_METHOD_CATALOG_REFRESH,
|
||||
}
|
||||
}
|
||||
@@ -56,6 +59,10 @@ impl TaskKind {
|
||||
config.repair = false;
|
||||
config.force = force;
|
||||
}
|
||||
Self::TranslationWorker => {
|
||||
config.dry_run = false;
|
||||
config.force = false;
|
||||
}
|
||||
}
|
||||
config
|
||||
}
|
||||
@@ -155,6 +162,7 @@ fn task_kind_static(kind: &str) -> Option<&'static str> {
|
||||
RPC_METHOD_RESOURCE_SYNC => Some(RPC_METHOD_RESOURCE_SYNC),
|
||||
RPC_METHOD_RESOURCE_VERIFY => Some(RPC_METHOD_RESOURCE_VERIFY),
|
||||
RPC_METHOD_RESOURCE_REPAIR => Some(RPC_METHOD_RESOURCE_REPAIR),
|
||||
RPC_METHOD_TRANSLATION_WORKER_RUN => Some(RPC_METHOD_TRANSLATION_WORKER_RUN),
|
||||
RPC_METHOD_CATALOG_REFRESH => Some(RPC_METHOD_CATALOG_REFRESH),
|
||||
_ => None,
|
||||
}
|
||||
@@ -494,7 +502,9 @@ impl TaskStore {
|
||||
/// 提交给任务 worker 的作业(配置已按任务类型派生完毕)。
|
||||
pub(super) struct TaskJob {
|
||||
pub(super) id: String,
|
||||
pub(super) kind: TaskKind,
|
||||
pub(super) config: OfficialUpdateConfig,
|
||||
pub(super) translation_worker_config: Option<TranslationWorkerConfig>,
|
||||
/// 与任务记录共享的取消标志。
|
||||
pub(super) cancel: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -528,36 +538,68 @@ pub(super) fn run_task_worker(
|
||||
});
|
||||
|
||||
let cancel = Arc::clone(&job.cancel);
|
||||
let run_result = {
|
||||
let _sync_guard = sync_lock
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let progress_registry = registry.clone();
|
||||
let progress_id = job.id.clone();
|
||||
let cancel_check = Arc::clone(&cancel);
|
||||
let stop_control = Arc::clone(&control);
|
||||
service.run_with_progress_and_cancellation(
|
||||
&job.config,
|
||||
|event| {
|
||||
progress_registry
|
||||
.append_log(&progress_id, format!("[{}] {}", event.stage, event.message));
|
||||
progress_registry.update(&progress_id, |record| {
|
||||
record.stage = Some(event.stage.to_string());
|
||||
record.message = Some(event.message.clone());
|
||||
});
|
||||
},
|
||||
|| {
|
||||
cancel_check.load(Ordering::Relaxed)
|
||||
|| daemon_control_stop_requested(Some(&stop_control))
|
||||
},
|
||||
)
|
||||
let run_result = if job.kind == TaskKind::TranslationWorker {
|
||||
let worker_config = job
|
||||
.translation_worker_config
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow::anyhow!("translation worker 任务缺少运行配置"));
|
||||
worker_config.and_then(|worker_config| {
|
||||
let _sync_guard = sync_lock
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
registry.append_log(&job.id, "[translation-worker] 开始执行".to_string());
|
||||
registry.update(&job.id, |record| {
|
||||
record.stage = Some("translation-worker".to_string());
|
||||
record.message = Some("翻译 provider worker 正在执行".to_string());
|
||||
});
|
||||
let resource_root = active_official_resource_root(&job.config.output_root)?;
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
runtime
|
||||
.block_on(bat_infrastructure::run_translation_worker_at(
|
||||
&resource_root,
|
||||
worker_config,
|
||||
))
|
||||
.and_then(|report| serde_json::to_value(report).map_err(anyhow::Error::from))
|
||||
})
|
||||
} else {
|
||||
let run_result = {
|
||||
let _sync_guard = sync_lock
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let progress_registry = registry.clone();
|
||||
let progress_id = job.id.clone();
|
||||
let cancel_check = Arc::clone(&cancel);
|
||||
let stop_control = Arc::clone(&control);
|
||||
service.run_with_progress_and_cancellation(
|
||||
&job.config,
|
||||
|event| {
|
||||
progress_registry.append_log(
|
||||
&progress_id,
|
||||
format!("[{}] {}", event.stage, event.message),
|
||||
);
|
||||
progress_registry.update(&progress_id, |record| {
|
||||
record.stage = Some(event.stage.to_string());
|
||||
record.message = Some(event.message.clone());
|
||||
});
|
||||
},
|
||||
|| {
|
||||
cancel_check.load(Ordering::Relaxed)
|
||||
|| daemon_control_stop_requested(Some(&stop_control))
|
||||
},
|
||||
)
|
||||
};
|
||||
run_result
|
||||
.map(|report| serde_json::to_value(&report).map_err(anyhow::Error::from))
|
||||
.and_then(|result| result)
|
||||
};
|
||||
|
||||
match run_result {
|
||||
Ok(report) => registry.update(&job.id, |record| {
|
||||
record.status = "succeeded";
|
||||
record.finished_at = Some(unix_seconds_now());
|
||||
record.result = serde_json::to_value(&report).ok();
|
||||
record.result = Some(report);
|
||||
}),
|
||||
Err(error) => {
|
||||
let cancelled = cancel.load(Ordering::Relaxed);
|
||||
|
||||
@@ -253,6 +253,43 @@ pub(super) fn run_translation_proofread(options: &CliOptions) -> anyhow::Result<
|
||||
print_report(options.output_format, &report)
|
||||
}
|
||||
|
||||
pub(super) fn run_translation_worker(options: &CliOptions) -> anyhow::Result<()> {
|
||||
let resource_root = options
|
||||
.resource_root
|
||||
.clone()
|
||||
.map(|path| lexical_absolute(&path).map_err(anyhow::Error::msg))
|
||||
.transpose()?
|
||||
.unwrap_or(active_official_resource_root(&options.config.output_root)?);
|
||||
let provider = options
|
||||
.translation_provider
|
||||
.as_deref()
|
||||
.unwrap_or(TranslationProviderKind::Mock.as_str());
|
||||
let provider = TranslationProviderKind::parse(provider)
|
||||
.ok_or_else(|| anyhow::anyhow!("i18n worker run 的 provider 无效:{provider}"))?;
|
||||
let config = TranslationWorkerConfig {
|
||||
provider,
|
||||
fixture_path: options.translation_fixture.clone(),
|
||||
concurrency: options.worker_concurrency,
|
||||
max_attempts: options.worker_max_attempts,
|
||||
lease_seconds: options.worker_lease_seconds,
|
||||
retry_backoff: options.worker_retry_backoff,
|
||||
max_tasks: options.worker_max_tasks,
|
||||
worker_id: options
|
||||
.worker_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("bat-worker-{}", std::process::id())),
|
||||
};
|
||||
config.validate()?;
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let report = runtime.block_on(bat_infrastructure::run_translation_worker_at(
|
||||
&resource_root,
|
||||
&config,
|
||||
))?;
|
||||
print_report(options.output_format, &report)
|
||||
}
|
||||
|
||||
pub(super) fn run_repack(options: &CliOptions) -> anyhow::Result<()> {
|
||||
let spec = options
|
||||
.repack_spec
|
||||
|
||||
@@ -156,7 +156,7 @@ mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
struct TestBackend {
|
||||
active: AtomicUsize,
|
||||
@@ -222,7 +222,15 @@ mod tests {
|
||||
fn download(&self, task: usize) -> Result<Self::Output, Self::Error> {
|
||||
if task == 0 {
|
||||
self.active.fetch_add(1, Ordering::SeqCst);
|
||||
thread::sleep(Duration::from_millis(20));
|
||||
let deadline = Instant::now() + Duration::from_millis(500);
|
||||
while self
|
||||
.task_two_started_while_task_zero_active
|
||||
.load(Ordering::SeqCst)
|
||||
== 0
|
||||
&& Instant::now() < deadline
|
||||
{
|
||||
thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
self.active.fetch_sub(1, Ordering::SeqCst);
|
||||
} else {
|
||||
if task == 1 {
|
||||
|
||||
@@ -30,6 +30,7 @@ pub mod path_security;
|
||||
pub mod release_flow;
|
||||
pub mod resources;
|
||||
pub mod translation_tasks;
|
||||
pub mod translation_worker;
|
||||
pub mod translation_workflow;
|
||||
mod zip_validation;
|
||||
|
||||
@@ -138,11 +139,21 @@ pub use translation_tasks::{
|
||||
build_translation_handoff, read_translation_handoff_at, sync_translation_task_repository_at,
|
||||
write_translation_handoff_at, PersistedTranslationTask, PersistedTranslationTaskState,
|
||||
ProviderRun, ProviderRunStatus, SqliteTranslationTaskRepository, TranslationHandoff,
|
||||
TranslationJob, TranslationJobStatus, TranslationTaskStatus, TranslationTaskSyncReport,
|
||||
TranslationUnit, TranslationUnitStatus, TRANSLATION_HANDOFF_FILE,
|
||||
TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
|
||||
TranslationJob, TranslationJobStatus, TranslationTaskFailure, TranslationTaskStatus,
|
||||
TranslationTaskSyncReport, TranslationTaskUnitResult, TranslationUnit, TranslationUnitStatus,
|
||||
TRANSLATION_HANDOFF_FILE, TRANSLATION_HANDOFF_SCHEMA_VERSION, TRANSLATION_TASK_REPOSITORY_FILE,
|
||||
TRANSLATION_TASK_SCHEMA_VERSION,
|
||||
};
|
||||
pub use translation_worker::{
|
||||
run_translation_worker_at, run_translation_worker_with_provider, CrowdinProvider,
|
||||
MockTranslationProvider, TranslationProvider, TranslationProviderFailureClass,
|
||||
TranslationProviderKind, TranslationProviderRequest, TranslationProviderResponse,
|
||||
TranslationProviderUnit, TranslationProviderUnitResult, TranslationWorkerConfig,
|
||||
TranslationWorkerFailure, TranslationWorkerReport, DEFAULT_TRANSLATION_CONCURRENCY,
|
||||
DEFAULT_TRANSLATION_LEASE_SECONDS, DEFAULT_TRANSLATION_MAX_ATTEMPTS,
|
||||
DEFAULT_TRANSLATION_RETRY_BACKOFF, MAX_TRANSLATION_CONCURRENCY, MIN_TRANSLATION_CONCURRENCY,
|
||||
MOCK_TRANSLATION_FIXTURE_VERSION,
|
||||
};
|
||||
pub use translation_workflow::{
|
||||
export_translation_workbench, get_translation_entry, localized_text_asset_patches,
|
||||
read_translation_workbench, repack_bundle, set_translation, unset_translation,
|
||||
|
||||
@@ -14,15 +14,15 @@ use crate::path_security::{
|
||||
};
|
||||
use bat_core::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions};
|
||||
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// SQLite schema version for durable translation task state.
|
||||
pub const TRANSLATION_TASK_SCHEMA_VERSION: u32 = 1;
|
||||
pub const TRANSLATION_TASK_SCHEMA_VERSION: u32 = 2;
|
||||
const TRANSLATION_TASK_SCHEMA_COMPONENT: &str = "translation_tasks";
|
||||
/// SQLite file name stored under a published official release root.
|
||||
pub const TRANSLATION_TASK_REPOSITORY_FILE: &str = "translation-tasks.sqlite";
|
||||
@@ -147,6 +147,26 @@ pub struct TranslationUnit {
|
||||
pub status: TranslationUnitStatus,
|
||||
/// Provider failure diagnostic, when present.
|
||||
pub failure_reason: Option<String>,
|
||||
/// TextUnit-level provider results associated with this resource task.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub translation_results: Vec<TranslationTaskUnitResult>,
|
||||
}
|
||||
|
||||
/// One provider result linked to an immutable TextUnit.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TranslationTaskUnitResult {
|
||||
/// Stable TextUnit ID from `official-textunit-index.json`.
|
||||
pub unit_id: String,
|
||||
/// Source text submitted to the provider.
|
||||
pub source_text: String,
|
||||
/// Provider-produced or human-supplied translation.
|
||||
pub translated_text: String,
|
||||
/// Provider identifier.
|
||||
pub provider: String,
|
||||
/// Provider run that produced this result.
|
||||
pub provider_run_id: String,
|
||||
/// Result persistence time.
|
||||
pub translated_unix_seconds: u64,
|
||||
}
|
||||
|
||||
/// One provider execution associated with one or more translation units.
|
||||
@@ -203,7 +223,16 @@ pub fn build_translation_handoff(
|
||||
let state = persisted
|
||||
.get(task.task_id.as_str())
|
||||
.copied()
|
||||
.map(|task| (task.task_status, task.failure_reason.clone(), task.attempt_count, task.provider_run_id.clone()))
|
||||
.map(|task| {
|
||||
(
|
||||
task.task_status,
|
||||
task.failure_reason.clone(),
|
||||
task.attempt_count,
|
||||
task.provider_run_id.clone(),
|
||||
task.provider.clone(),
|
||||
task.translation_results.clone(),
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
(
|
||||
initial_task_status(task),
|
||||
@@ -214,6 +243,8 @@ pub fn build_translation_handoff(
|
||||
},
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
Vec::new(),
|
||||
)
|
||||
});
|
||||
let unit_status = match state.0 {
|
||||
@@ -234,6 +265,7 @@ pub fn build_translation_handoff(
|
||||
text_unit_formats: task.text_unit_formats.clone(),
|
||||
status: unit_status,
|
||||
failure_reason: state.1.clone(),
|
||||
translation_results: state.5.clone(),
|
||||
};
|
||||
if let Some(provider_run_id) = state.3 {
|
||||
let run = provider_runs
|
||||
@@ -241,7 +273,7 @@ pub fn build_translation_handoff(
|
||||
.or_insert_with(|| ProviderRun {
|
||||
provider_run_id,
|
||||
job_id: job_id.clone(),
|
||||
provider: "worker".to_string(),
|
||||
provider: state.4.unwrap_or_else(|| "worker".to_string()),
|
||||
status: ProviderRunStatus::Queued,
|
||||
unit_ids: Vec::new(),
|
||||
attempt_count: 0,
|
||||
@@ -403,6 +435,26 @@ pub struct PersistedTranslationTask {
|
||||
/// Provider-side run identifier, if known.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub provider_run_id: Option<String>,
|
||||
/// Provider-produced results keyed by TextUnit ID.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub translation_results: Vec<TranslationTaskUnitResult>,
|
||||
/// Provider identifier used by the latest run.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub provider: Option<String>,
|
||||
/// Worker currently holding the lease.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lease_owner: Option<String>,
|
||||
/// Lease expiry as Unix seconds.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lease_expires_unix_seconds: Option<u64>,
|
||||
/// Stable failure classification.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub failure_class: Option<String>,
|
||||
/// Whether the latest failure can be retried.
|
||||
pub failure_retryable: bool,
|
||||
/// Earliest Unix time at which a failed task may be claimed again.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub next_attempt_unix_seconds: Option<u64>,
|
||||
}
|
||||
|
||||
/// Mutable persistence metadata associated with an official TextUnit task.
|
||||
@@ -422,6 +474,39 @@ pub struct PersistedTranslationTaskState {
|
||||
pub completed_unix_seconds: Option<u64>,
|
||||
/// Provider-side run identifier, if known.
|
||||
pub provider_run_id: Option<String>,
|
||||
/// Provider-produced results keyed by TextUnit ID.
|
||||
pub translation_results: Vec<TranslationTaskUnitResult>,
|
||||
/// Provider identifier used by the latest run.
|
||||
pub provider: Option<String>,
|
||||
/// Worker currently holding the lease.
|
||||
pub lease_owner: Option<String>,
|
||||
/// Lease expiry as Unix seconds.
|
||||
pub lease_expires_unix_seconds: Option<u64>,
|
||||
/// Stable failure classification.
|
||||
pub failure_class: Option<String>,
|
||||
/// Whether the latest failure can be retried.
|
||||
pub failure_retryable: bool,
|
||||
/// Earliest Unix time at which a failed task may be claimed again.
|
||||
pub next_attempt_unix_seconds: Option<u64>,
|
||||
}
|
||||
|
||||
/// Provider failure data used to atomically release a task lease.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TranslationTaskFailure {
|
||||
/// Task being failed.
|
||||
pub task_id: String,
|
||||
/// Worker that owns the lease.
|
||||
pub worker_id: String,
|
||||
/// Provider run associated with the lease.
|
||||
pub provider_run_id: String,
|
||||
/// Stable failure classification.
|
||||
pub failure_class: String,
|
||||
/// Redacted diagnostic message.
|
||||
pub failure_reason: String,
|
||||
/// Whether a future attempt may retry the task.
|
||||
pub retryable: bool,
|
||||
/// Earliest retry time, when retryable.
|
||||
pub next_attempt_unix_seconds: Option<u64>,
|
||||
}
|
||||
|
||||
/// Result of synchronizing an immutable release queue into SQLite.
|
||||
@@ -463,7 +548,9 @@ impl SqliteTranslationTaskRepository {
|
||||
|
||||
let options = SqliteConnectOptions::from_str(&format!("sqlite://{}", path.display()))
|
||||
.map_err(|error| bat_core::Error::Other(error.into()))?
|
||||
.create_if_missing(create_if_missing);
|
||||
.create_if_missing(create_if_missing)
|
||||
.journal_mode(SqliteJournalMode::Wal)
|
||||
.busy_timeout(Duration::from_secs(30));
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
@@ -510,7 +597,14 @@ impl SqliteTranslationTaskRepository {
|
||||
created_unix_seconds INTEGER NOT NULL,
|
||||
updated_unix_seconds INTEGER NOT NULL,
|
||||
completed_unix_seconds INTEGER,
|
||||
provider_run_id TEXT
|
||||
provider_run_id TEXT,
|
||||
translation_results_json TEXT NOT NULL DEFAULT '[]',
|
||||
provider TEXT,
|
||||
lease_owner TEXT,
|
||||
lease_expires_unix_seconds INTEGER,
|
||||
failure_class TEXT,
|
||||
failure_retryable INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_unix_seconds INTEGER
|
||||
)
|
||||
"#,
|
||||
)
|
||||
@@ -573,6 +667,37 @@ impl SqliteTranslationTaskRepository {
|
||||
)
|
||||
.await?;
|
||||
ensure_column(&self.pool, "translation_tasks", "provider_run_id", "TEXT").await?;
|
||||
ensure_column(
|
||||
&self.pool,
|
||||
"translation_tasks",
|
||||
"translation_results_json",
|
||||
"TEXT NOT NULL DEFAULT '[]'",
|
||||
)
|
||||
.await?;
|
||||
ensure_column(&self.pool, "translation_tasks", "provider", "TEXT").await?;
|
||||
ensure_column(&self.pool, "translation_tasks", "lease_owner", "TEXT").await?;
|
||||
ensure_column(
|
||||
&self.pool,
|
||||
"translation_tasks",
|
||||
"lease_expires_unix_seconds",
|
||||
"INTEGER",
|
||||
)
|
||||
.await?;
|
||||
ensure_column(&self.pool, "translation_tasks", "failure_class", "TEXT").await?;
|
||||
ensure_column(
|
||||
&self.pool,
|
||||
"translation_tasks",
|
||||
"failure_retryable",
|
||||
"INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
.await?;
|
||||
ensure_column(
|
||||
&self.pool,
|
||||
"translation_tasks",
|
||||
"next_attempt_unix_seconds",
|
||||
"INTEGER",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let current: Option<i64> =
|
||||
sqlx::query_scalar("SELECT version FROM schema_migrations WHERE component = ?1")
|
||||
@@ -615,7 +740,10 @@ impl SqliteTranslationTaskRepository {
|
||||
let existing: Option<ExistingTaskRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT task_json, worker_status, failure_reason, attempt_count,
|
||||
created_unix_seconds, completed_unix_seconds, provider_run_id
|
||||
created_unix_seconds, completed_unix_seconds, provider_run_id,
|
||||
translation_results_json, provider, lease_owner,
|
||||
lease_expires_unix_seconds, failure_class, failure_retryable,
|
||||
next_attempt_unix_seconds
|
||||
FROM translation_tasks
|
||||
WHERE task_id = ?1
|
||||
"#,
|
||||
@@ -642,30 +770,64 @@ impl SqliteTranslationTaskRepository {
|
||||
created,
|
||||
completed,
|
||||
provider_run_id,
|
||||
translation_results_json,
|
||||
provider,
|
||||
lease_owner,
|
||||
lease_expires,
|
||||
failure_class,
|
||||
failure_retryable,
|
||||
next_attempt,
|
||||
)) = existing
|
||||
{
|
||||
let immutable_unchanged = previous_task_json == task_json;
|
||||
let (status, failure_reason, attempt_count, created, completed, provider_run_id) =
|
||||
if immutable_unchanged {
|
||||
(
|
||||
worker_status,
|
||||
failure_reason,
|
||||
attempt_count,
|
||||
created,
|
||||
completed,
|
||||
provider_run_id,
|
||||
)
|
||||
} else {
|
||||
report.refreshed_count += 1;
|
||||
(
|
||||
initial_status.as_str().to_string(),
|
||||
initial_failure_reason.clone(),
|
||||
0_i64,
|
||||
now,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
};
|
||||
let (
|
||||
status,
|
||||
failure_reason,
|
||||
attempt_count,
|
||||
created,
|
||||
completed,
|
||||
provider_run_id,
|
||||
translation_results_json,
|
||||
provider,
|
||||
lease_owner,
|
||||
lease_expires,
|
||||
failure_class,
|
||||
failure_retryable,
|
||||
next_attempt,
|
||||
) = if immutable_unchanged {
|
||||
(
|
||||
worker_status,
|
||||
failure_reason,
|
||||
attempt_count,
|
||||
created,
|
||||
completed,
|
||||
provider_run_id,
|
||||
translation_results_json,
|
||||
provider,
|
||||
lease_owner,
|
||||
lease_expires,
|
||||
failure_class,
|
||||
failure_retryable,
|
||||
next_attempt,
|
||||
)
|
||||
} else {
|
||||
report.refreshed_count += 1;
|
||||
(
|
||||
initial_status.as_str().to_string(),
|
||||
initial_failure_reason.clone(),
|
||||
0_i64,
|
||||
now,
|
||||
None,
|
||||
None,
|
||||
"[]".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
0_i64,
|
||||
None,
|
||||
)
|
||||
};
|
||||
if immutable_unchanged {
|
||||
report.preserved_state_count += 1;
|
||||
}
|
||||
@@ -677,7 +839,11 @@ impl SqliteTranslationTaskRepository {
|
||||
text_unit_formats_json = ?8, task_json = ?9,
|
||||
worker_status = ?10, failure_reason = ?11, attempt_count = ?12,
|
||||
created_unix_seconds = ?13, updated_unix_seconds = ?14,
|
||||
completed_unix_seconds = ?15, provider_run_id = ?16
|
||||
completed_unix_seconds = ?15, provider_run_id = ?16,
|
||||
translation_results_json = ?17, provider = ?18,
|
||||
lease_owner = ?19, lease_expires_unix_seconds = ?20,
|
||||
failure_class = ?21, failure_retryable = ?22,
|
||||
next_attempt_unix_seconds = ?23
|
||||
WHERE task_id = ?1
|
||||
"#,
|
||||
)
|
||||
@@ -697,6 +863,13 @@ impl SqliteTranslationTaskRepository {
|
||||
.bind(now)
|
||||
.bind(completed)
|
||||
.bind(provider_run_id)
|
||||
.bind(translation_results_json)
|
||||
.bind(provider)
|
||||
.bind(lease_owner)
|
||||
.bind(lease_expires)
|
||||
.bind(failure_class)
|
||||
.bind(failure_retryable)
|
||||
.bind(next_attempt)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
@@ -770,7 +943,9 @@ impl SqliteTranslationTaskRepository {
|
||||
r#"
|
||||
SELECT task_json, worker_status, failure_reason, attempt_count,
|
||||
created_unix_seconds, updated_unix_seconds,
|
||||
completed_unix_seconds, provider_run_id
|
||||
completed_unix_seconds, provider_run_id, translation_results_json,
|
||||
provider, lease_owner, lease_expires_unix_seconds, failure_class,
|
||||
failure_retryable, next_attempt_unix_seconds
|
||||
FROM translation_tasks
|
||||
ORDER BY task_id
|
||||
"#,
|
||||
@@ -794,6 +969,247 @@ impl SqliteTranslationTaskRepository {
|
||||
Ok(self.list(query).await?.len() as u64)
|
||||
}
|
||||
|
||||
/// Requeues tasks whose worker lease expired.
|
||||
pub async fn recover_expired_leases(&self, now_unix_seconds: u64) -> Result<u64> {
|
||||
let now = i64::try_from(now_unix_seconds).unwrap_or(i64::MAX);
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
SET worker_status = CASE
|
||||
WHEN attempt_count > 0 THEN 'failed'
|
||||
ELSE 'queued'
|
||||
END,
|
||||
failure_reason = 'worker lease expired; task recovered',
|
||||
failure_class = 'lease_expired',
|
||||
failure_retryable = 1,
|
||||
next_attempt_unix_seconds = ?1,
|
||||
updated_unix_seconds = ?1,
|
||||
lease_owner = NULL,
|
||||
lease_expires_unix_seconds = NULL
|
||||
WHERE worker_status = 'running'
|
||||
AND lease_expires_unix_seconds IS NOT NULL
|
||||
AND lease_expires_unix_seconds <= ?1
|
||||
"#,
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// Claims one queued or retryable failed task and assigns an exclusive
|
||||
/// lease to one worker.
|
||||
pub async fn claim_next(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
provider: &str,
|
||||
lease_seconds: u64,
|
||||
max_attempts: u32,
|
||||
) -> Result<Option<PersistedTranslationTask>> {
|
||||
if worker_id.trim().is_empty() || provider.trim().is_empty() {
|
||||
return Err(bat_core::Error::InvalidArgument(
|
||||
"translation worker_id/provider 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if max_attempts == 0 {
|
||||
return Err(bat_core::Error::InvalidArgument(
|
||||
"translation worker max_attempts 必须大于 0".to_string(),
|
||||
));
|
||||
}
|
||||
if lease_seconds == 0 {
|
||||
return Err(bat_core::Error::InvalidArgument(
|
||||
"translation worker lease_seconds 必须大于 0".to_string(),
|
||||
));
|
||||
}
|
||||
let now = unix_seconds_now_i64();
|
||||
let lease_expires = now.saturating_add(i64::try_from(lease_seconds).unwrap_or(i64::MAX));
|
||||
let mut transaction = self.pool.begin().await.map_err(db_error)?;
|
||||
let row: Option<TranslationTaskRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT task_json, worker_status, failure_reason, attempt_count,
|
||||
created_unix_seconds, updated_unix_seconds,
|
||||
completed_unix_seconds, provider_run_id, translation_results_json,
|
||||
provider, lease_owner, lease_expires_unix_seconds, failure_class,
|
||||
failure_retryable, next_attempt_unix_seconds
|
||||
FROM translation_tasks
|
||||
WHERE queue_status = 'queued_offline'
|
||||
AND (
|
||||
worker_status = 'queued'
|
||||
OR (
|
||||
worker_status = 'failed'
|
||||
AND failure_retryable = 1
|
||||
AND attempt_count < ?1
|
||||
AND (next_attempt_unix_seconds IS NULL
|
||||
OR next_attempt_unix_seconds <= ?2)
|
||||
)
|
||||
)
|
||||
AND (lease_expires_unix_seconds IS NULL OR lease_expires_unix_seconds <= ?2)
|
||||
ORDER BY task_id
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(i64::from(max_attempts))
|
||||
.bind(now)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
let Some(row) = row else {
|
||||
transaction.commit().await.map_err(db_error)?;
|
||||
return Ok(None);
|
||||
};
|
||||
let current = PersistedTranslationTask::from_row(row)?;
|
||||
let attempt_count = current.attempt_count.saturating_add(1);
|
||||
let provider_run_id = format!(
|
||||
"{provider}:{}:attempt-{attempt_count}",
|
||||
current.task.task_id
|
||||
);
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
SET worker_status = 'running',
|
||||
failure_reason = NULL,
|
||||
attempt_count = ?2,
|
||||
updated_unix_seconds = ?3,
|
||||
completed_unix_seconds = NULL,
|
||||
provider_run_id = ?4,
|
||||
provider = ?5,
|
||||
lease_owner = ?6,
|
||||
lease_expires_unix_seconds = ?7,
|
||||
failure_class = NULL,
|
||||
failure_retryable = 0,
|
||||
next_attempt_unix_seconds = NULL
|
||||
WHERE task_id = ?1
|
||||
AND (
|
||||
worker_status = 'queued'
|
||||
OR (
|
||||
worker_status = 'failed'
|
||||
AND failure_retryable = 1
|
||||
AND attempt_count < ?8
|
||||
AND (next_attempt_unix_seconds IS NULL
|
||||
OR next_attempt_unix_seconds <= ?3)
|
||||
)
|
||||
)
|
||||
AND queue_status = 'queued_offline'
|
||||
AND (lease_expires_unix_seconds IS NULL OR lease_expires_unix_seconds <= ?3)
|
||||
"#,
|
||||
)
|
||||
.bind(¤t.task.task_id)
|
||||
.bind(i64::from(attempt_count))
|
||||
.bind(now)
|
||||
.bind(&provider_run_id)
|
||||
.bind(provider)
|
||||
.bind(worker_id)
|
||||
.bind(lease_expires)
|
||||
.bind(i64::from(max_attempts))
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if result.rows_affected() != 1 {
|
||||
transaction.rollback().await.map_err(db_error)?;
|
||||
return Ok(None);
|
||||
}
|
||||
transaction.commit().await.map_err(db_error)?;
|
||||
self.find(¤t.task.task_id).await.map(Some)
|
||||
}
|
||||
|
||||
/// Stores an idempotent provider result while the worker still owns its lease.
|
||||
pub async fn complete_claim(
|
||||
&self,
|
||||
task_id: &str,
|
||||
worker_id: &str,
|
||||
provider_run_id: &str,
|
||||
provider: &str,
|
||||
translation_results: &[TranslationTaskUnitResult],
|
||||
) -> Result<PersistedTranslationTask> {
|
||||
let results_json = serde_json::to_string(translation_results)
|
||||
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
|
||||
let now = unix_seconds_now_i64();
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
SET worker_status = 'completed',
|
||||
failure_reason = NULL,
|
||||
updated_unix_seconds = ?2,
|
||||
completed_unix_seconds = ?2,
|
||||
provider_run_id = ?3,
|
||||
provider = ?4,
|
||||
translation_results_json = ?5,
|
||||
lease_owner = NULL,
|
||||
lease_expires_unix_seconds = NULL,
|
||||
failure_class = NULL,
|
||||
failure_retryable = 0,
|
||||
next_attempt_unix_seconds = NULL
|
||||
WHERE task_id = ?1
|
||||
AND worker_status = 'running'
|
||||
AND lease_owner = ?6
|
||||
AND provider_run_id = ?3
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(now)
|
||||
.bind(provider_run_id)
|
||||
.bind(provider)
|
||||
.bind(results_json)
|
||||
.bind(worker_id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if result.rows_affected() != 1 {
|
||||
return Err(bat_core::Error::InvalidArgument(format!(
|
||||
"翻译任务 {} 的 lease 已失效,拒绝写入 provider 结果",
|
||||
task_id
|
||||
)));
|
||||
}
|
||||
self.find(task_id).await
|
||||
}
|
||||
|
||||
/// Records one provider failure and releases the worker lease.
|
||||
pub async fn fail_claim(
|
||||
&self,
|
||||
failure: TranslationTaskFailure,
|
||||
) -> Result<PersistedTranslationTask> {
|
||||
let now = unix_seconds_now_i64();
|
||||
let next_attempt = failure
|
||||
.next_attempt_unix_seconds
|
||||
.map(|value| i64::try_from(value).unwrap_or(i64::MAX));
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
SET worker_status = 'failed',
|
||||
failure_reason = ?2,
|
||||
updated_unix_seconds = ?3,
|
||||
lease_owner = NULL,
|
||||
lease_expires_unix_seconds = NULL,
|
||||
failure_class = ?4,
|
||||
failure_retryable = ?5,
|
||||
next_attempt_unix_seconds = ?6
|
||||
WHERE task_id = ?1
|
||||
AND worker_status = 'running'
|
||||
AND lease_owner = ?7
|
||||
AND provider_run_id = ?8
|
||||
"#,
|
||||
)
|
||||
.bind(&failure.task_id)
|
||||
.bind(&failure.failure_reason)
|
||||
.bind(now)
|
||||
.bind(&failure.failure_class)
|
||||
.bind(if failure.retryable { 1_i64 } else { 0_i64 })
|
||||
.bind(next_attempt)
|
||||
.bind(&failure.worker_id)
|
||||
.bind(&failure.provider_run_id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(db_error)?;
|
||||
if result.rows_affected() != 1 {
|
||||
return Err(bat_core::Error::InvalidArgument(format!(
|
||||
"翻译任务 {} 的 lease 已失效,拒绝写入 provider 失败状态",
|
||||
failure.task_id
|
||||
)));
|
||||
}
|
||||
self.find(&failure.task_id).await
|
||||
}
|
||||
|
||||
/// Updates provider state and returns the durable task record.
|
||||
pub async fn update_status(
|
||||
&self,
|
||||
@@ -818,7 +1234,10 @@ impl SqliteTranslationTaskRepository {
|
||||
UPDATE translation_tasks
|
||||
SET worker_status = ?2, failure_reason = ?3, attempt_count = ?4,
|
||||
updated_unix_seconds = ?5, completed_unix_seconds = ?6,
|
||||
provider_run_id = COALESCE(?7, provider_run_id)
|
||||
provider_run_id = COALESCE(?7, provider_run_id),
|
||||
lease_owner = NULL, lease_expires_unix_seconds = NULL,
|
||||
failure_class = NULL, failure_retryable = 0,
|
||||
next_attempt_unix_seconds = NULL
|
||||
WHERE task_id = ?1
|
||||
"#,
|
||||
)
|
||||
@@ -841,7 +1260,9 @@ impl SqliteTranslationTaskRepository {
|
||||
r#"
|
||||
SELECT task_json, worker_status, failure_reason, attempt_count,
|
||||
created_unix_seconds, updated_unix_seconds,
|
||||
completed_unix_seconds, provider_run_id
|
||||
completed_unix_seconds, provider_run_id, translation_results_json,
|
||||
provider, lease_owner, lease_expires_unix_seconds, failure_class,
|
||||
failure_retryable, next_attempt_unix_seconds
|
||||
FROM translation_tasks
|
||||
WHERE task_id = ?1
|
||||
"#,
|
||||
@@ -864,6 +1285,13 @@ type ExistingTaskRow = (
|
||||
i64,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
i64,
|
||||
Option<i64>,
|
||||
);
|
||||
|
||||
type TranslationTaskRow = (
|
||||
@@ -875,6 +1303,13 @@ type TranslationTaskRow = (
|
||||
i64,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
i64,
|
||||
Option<i64>,
|
||||
);
|
||||
|
||||
impl PersistedTranslationTask {
|
||||
@@ -889,6 +1324,13 @@ impl PersistedTranslationTask {
|
||||
updated_unix_seconds: state.updated_unix_seconds,
|
||||
completed_unix_seconds: state.completed_unix_seconds,
|
||||
provider_run_id: state.provider_run_id,
|
||||
translation_results: state.translation_results,
|
||||
provider: state.provider,
|
||||
lease_owner: state.lease_owner,
|
||||
lease_expires_unix_seconds: state.lease_expires_unix_seconds,
|
||||
failure_class: state.failure_class,
|
||||
failure_retryable: state.failure_retryable,
|
||||
next_attempt_unix_seconds: state.next_attempt_unix_seconds,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -910,6 +1352,13 @@ impl PersistedTranslationTask {
|
||||
updated_unix_seconds: generated_unix_seconds,
|
||||
completed_unix_seconds: None,
|
||||
provider_run_id: None,
|
||||
translation_results: Vec::new(),
|
||||
provider: None,
|
||||
lease_owner: None,
|
||||
lease_expires_unix_seconds: None,
|
||||
failure_class: None,
|
||||
failure_retryable: false,
|
||||
next_attempt_unix_seconds: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -924,12 +1373,21 @@ impl PersistedTranslationTask {
|
||||
updated_unix_seconds,
|
||||
completed_unix_seconds,
|
||||
provider_run_id,
|
||||
translation_results_json,
|
||||
provider,
|
||||
lease_owner,
|
||||
lease_expires_unix_seconds,
|
||||
failure_class,
|
||||
failure_retryable,
|
||||
next_attempt_unix_seconds,
|
||||
) = row;
|
||||
let task = serde_json::from_str(&task_json)
|
||||
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
|
||||
let task_status = TranslationTaskStatus::parse(&worker_status).ok_or_else(|| {
|
||||
bat_core::Error::Serialization(format!("未知翻译任务 worker 状态:{worker_status}"))
|
||||
})?;
|
||||
let translation_results = serde_json::from_str(&translation_results_json)
|
||||
.map_err(|error| bat_core::Error::Serialization(error.to_string()))?;
|
||||
Ok(Self {
|
||||
task,
|
||||
task_status,
|
||||
@@ -951,6 +1409,25 @@ impl PersistedTranslationTask {
|
||||
})
|
||||
.transpose()?,
|
||||
provider_run_id,
|
||||
translation_results,
|
||||
provider,
|
||||
lease_owner,
|
||||
lease_expires_unix_seconds: lease_expires_unix_seconds
|
||||
.map(|value| {
|
||||
u64::try_from(value).map_err(|_| {
|
||||
bat_core::Error::Serialization("翻译任务 lease 时间无效".to_string())
|
||||
})
|
||||
})
|
||||
.transpose()?,
|
||||
failure_class,
|
||||
failure_retryable: failure_retryable != 0,
|
||||
next_attempt_unix_seconds: next_attempt_unix_seconds
|
||||
.map(|value| {
|
||||
u64::try_from(value).map_err(|_| {
|
||||
bat_core::Error::Serialization("翻译任务 next_attempt 时间无效".to_string())
|
||||
})
|
||||
})
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1219,6 +1696,140 @@ mod tests {
|
||||
assert_eq!(retrievable[0].attempt_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_translation_tasks_recover_expired_leases_for_retry() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository =
|
||||
SqliteTranslationTaskRepository::new(temp.path().join("translation-tasks.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let queue = queue(vec![task(
|
||||
"task-a",
|
||||
"Bundles/a.bundle",
|
||||
OfficialTextUnitTaskStatus::QueuedOffline,
|
||||
Some(OfficialParseStatus::Parsed),
|
||||
None,
|
||||
)]);
|
||||
repository.sync_queue(&queue).await.unwrap();
|
||||
|
||||
let claimed = repository
|
||||
.claim_next("worker-a", "mock", 1, 3)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(claimed.task_status, TranslationTaskStatus::Running);
|
||||
assert_eq!(claimed.lease_owner.as_deref(), Some("worker-a"));
|
||||
force_expire_lease(&repository, "task-a").await;
|
||||
|
||||
let recovered = repository
|
||||
.recover_expired_leases(unix_seconds_now())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(recovered, 1);
|
||||
let recovered_task = repository.find("task-a").await.unwrap();
|
||||
assert_eq!(recovered_task.task_status, TranslationTaskStatus::Failed);
|
||||
assert_eq!(
|
||||
recovered_task.failure_reason.as_deref(),
|
||||
Some("worker lease expired; task recovered")
|
||||
);
|
||||
assert_eq!(
|
||||
recovered_task.failure_class.as_deref(),
|
||||
Some("lease_expired")
|
||||
);
|
||||
assert!(recovered_task.failure_retryable);
|
||||
assert_eq!(recovered_task.lease_owner, None);
|
||||
|
||||
let reclaimed = repository
|
||||
.claim_next("worker-b", "mock", 30, 3)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(reclaimed.task_status, TranslationTaskStatus::Running);
|
||||
assert_eq!(reclaimed.attempt_count, 2);
|
||||
assert_eq!(reclaimed.lease_owner.as_deref(), Some("worker-b"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_translation_tasks_reject_stale_worker_writes() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
let repository =
|
||||
SqliteTranslationTaskRepository::new(temp.path().join("translation-tasks.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let queue = queue(vec![task(
|
||||
"task-a",
|
||||
"Bundles/a.bundle",
|
||||
OfficialTextUnitTaskStatus::QueuedOffline,
|
||||
Some(OfficialParseStatus::Parsed),
|
||||
None,
|
||||
)]);
|
||||
repository.sync_queue(&queue).await.unwrap();
|
||||
|
||||
let first_claim = repository
|
||||
.claim_next("worker-a", "mock", 1, 3)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let first_run = first_claim.provider_run_id.clone().unwrap();
|
||||
force_expire_lease(&repository, "task-a").await;
|
||||
repository
|
||||
.recover_expired_leases(unix_seconds_now())
|
||||
.await
|
||||
.unwrap();
|
||||
let second_claim = repository
|
||||
.claim_next("worker-b", "mock", 30, 3)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let second_run = second_claim.provider_run_id.clone().unwrap();
|
||||
let result = TranslationTaskUnitResult {
|
||||
unit_id: "unit-a".to_string(),
|
||||
source_text: "source".to_string(),
|
||||
translated_text: "translated".to_string(),
|
||||
provider: "mock".to_string(),
|
||||
provider_run_id: second_run.clone(),
|
||||
translated_unix_seconds: 1,
|
||||
};
|
||||
|
||||
assert!(repository
|
||||
.complete_claim(
|
||||
"task-a",
|
||||
"worker-a",
|
||||
&first_run,
|
||||
"mock",
|
||||
std::slice::from_ref(&result)
|
||||
)
|
||||
.await
|
||||
.is_err());
|
||||
let completed = repository
|
||||
.complete_claim(
|
||||
"task-a",
|
||||
"worker-b",
|
||||
&second_run,
|
||||
"mock",
|
||||
std::slice::from_ref(&result),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(completed.task_status, TranslationTaskStatus::Completed);
|
||||
assert_eq!(completed.translation_results, vec![result.clone()]);
|
||||
assert!(repository
|
||||
.fail_claim(TranslationTaskFailure {
|
||||
task_id: "task-a".to_string(),
|
||||
worker_id: "worker-a".to_string(),
|
||||
provider_run_id: first_run,
|
||||
failure_class: "network".to_string(),
|
||||
failure_reason: "late failure".to_string(),
|
||||
retryable: true,
|
||||
next_attempt_unix_seconds: Some(2),
|
||||
})
|
||||
.await
|
||||
.is_err());
|
||||
let task = repository.find("task-a").await.unwrap();
|
||||
assert_eq!(task.task_status, TranslationTaskStatus::Completed);
|
||||
assert_eq!(task.translation_results, vec![result]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_handoff_maps_worker_state_and_preserves_provider_progress() {
|
||||
let queue = queue(vec![
|
||||
@@ -1254,6 +1865,13 @@ mod tests {
|
||||
updated_unix_seconds: 124,
|
||||
completed_unix_seconds: None,
|
||||
provider_run_id: Some("provider-run-1".to_string()),
|
||||
translation_results: Vec::new(),
|
||||
provider: Some("fixture".to_string()),
|
||||
lease_owner: None,
|
||||
lease_expires_unix_seconds: None,
|
||||
failure_class: None,
|
||||
failure_retryable: false,
|
||||
next_attempt_unix_seconds: None,
|
||||
},
|
||||
PersistedTranslationTask {
|
||||
task: queue.tasks[1].clone(),
|
||||
@@ -1264,6 +1882,13 @@ mod tests {
|
||||
updated_unix_seconds: 125,
|
||||
completed_unix_seconds: Some(125),
|
||||
provider_run_id: Some("provider-run-1".to_string()),
|
||||
translation_results: Vec::new(),
|
||||
provider: Some("fixture".to_string()),
|
||||
lease_owner: None,
|
||||
lease_expires_unix_seconds: None,
|
||||
failure_class: None,
|
||||
failure_retryable: false,
|
||||
next_attempt_unix_seconds: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -1286,6 +1911,20 @@ mod tests {
|
||||
assert_eq!(handoff.provider_runs[0].attempt_count, 2);
|
||||
}
|
||||
|
||||
async fn force_expire_lease(repository: &SqliteTranslationTaskRepository, task_id: &str) {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE translation_tasks
|
||||
SET lease_expires_unix_seconds = 0
|
||||
WHERE task_id = ?1
|
||||
"#,
|
||||
)
|
||||
.bind(task_id)
|
||||
.execute(&repository.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translation_handoff_file_round_trips_with_version_check() {
|
||||
let temp = tempfile::TempDir::new().unwrap();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user