fix: harden official resource sync retries

infrastructure/src/bin/bat_official_sync.rs

@@ -11,6 +11,7 @@ use std::time::Duration;
 const EXIT_ERROR: i32 = 1;
 const EXIT_LOCKED: i32 = 75;
 const DEFAULT_WATCH_INTERVAL_SECONDS: u64 = 60 * 60;
+const DEFAULT_ERROR_RETRY_SECONDS: u64 = 60;
 
 fn main() {
     match run() {
@@ -64,6 +65,7 @@ struct CliOptions {
     config: OfficialUpdateConfig,
     watch: bool,
     interval: Duration,
+    error_retry_interval: Duration,
     quiet_up_to_date: bool,
     quiet_up_to_date_explicit: bool,
 }
@@ -74,6 +76,7 @@ impl Default for CliOptions {
             config: OfficialUpdateConfig::default(),
             watch: false,
             interval: Duration::from_secs(DEFAULT_WATCH_INTERVAL_SECONDS),
+            error_retry_interval: Duration::from_secs(DEFAULT_ERROR_RETRY_SECONDS),
             quiet_up_to_date: false,
             quiet_up_to_date_explicit: false,
         }
@@ -87,8 +90,14 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
     if options.interval.is_zero() {
         return Err(anyhow::anyhow!("watch interval must be greater than zero"));
     }
+    if options.error_retry_interval.is_zero() {
+        return Err(anyhow::anyhow!(
+            "watch error retry interval must be greater than zero"
+        ));
+    }
     let service = OfficialUpdateService::new();
     loop {
+        let mut sleep_for = options.interval;
         match service.run(&options.config) {
             Ok(report) => {
                 if should_print_status(report.update_status, options.quiet_up_to_date) {
@@ -96,17 +105,18 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
                 }
             }
             Err(error) => {
+                sleep_for = options.error_retry_interval;
                 let payload = ErrorReport {
                     status: "error",
                     exit_code: EXIT_ERROR,
                     error: error.to_string(),
-                    next_retry_seconds: Some(options.interval.as_secs()),
+                    next_retry_seconds: Some(sleep_for.as_secs()),
                 };
                 eprintln!("{}", serde_json::to_string_pretty(&payload)?);
             }
         }
 
-        thread::sleep(options.interval);
+        thread::sleep(sleep_for);
     }
 }
 
@@ -205,6 +215,16 @@ fn parse_args_from(raw_args: impl IntoIterator<Item = String>) -> anyhow::Result
                     .map_err(|error| anyhow::anyhow!("invalid seconds for {flag}: {error}"))?;
                 options.interval = Duration::from_secs(seconds);
             }
+            "--error-retry" => {
+                options.error_retry_interval =
+                    parse_duration(&next_option_value(&mut args, &flag)?)?;
+            }
+            "--error-retry-seconds" => {
+                let seconds = next_option_value(&mut args, &flag)?
+                    .parse::<u64>()
+                    .map_err(|error| anyhow::anyhow!("invalid seconds for {flag}: {error}"))?;
+                options.error_retry_interval = Duration::from_secs(seconds);
+            }
             "--quiet-up-to-date" => {
                 options.quiet_up_to_date = true;
                 options.quiet_up_to_date_explicit = true;
@@ -227,6 +247,11 @@ fn parse_args_from(raw_args: impl IntoIterator<Item = String>) -> anyhow::Result
     if options.interval.is_zero() {
         return Err(anyhow::anyhow!("watch interval must be greater than zero"));
     }
+    if options.error_retry_interval.is_zero() {
+        return Err(anyhow::anyhow!(
+            "watch error retry interval must be greater than zero"
+        ));
+    }
     if options.watch && !options.quiet_up_to_date_explicit {
         options.quiet_up_to_date = true;
     }
@@ -248,7 +273,7 @@ fn print_usage(binary: &str) {
          [--app-version 1.70.0] [--connection-group <name>] [--launcher-version 1.7.2] \
          [--platforms Windows,Android] [--output official_update_output] [--snapshot official-sync-snapshot.json] \
          [--curl curl] [--unzip unzip] [--dry-run] [--plan] [--force] [--audit-local|--no-audit-local] [--repair|--no-repair] \
-         [--watch] [--interval 1h|60m|3600s] [--quiet-up-to-date|--no-quiet-up-to-date]"
+         [--watch] [--interval 1h|60m|3600s] [--error-retry 60s] [--quiet-up-to-date|--no-quiet-up-to-date]"
     );
 }
 
@@ -337,6 +362,10 @@ mod tests {
             options.interval,
             Duration::from_secs(DEFAULT_WATCH_INTERVAL_SECONDS)
         );
+        assert_eq!(
+            options.error_retry_interval,
+            Duration::from_secs(DEFAULT_ERROR_RETRY_SECONDS)
+        );
         assert!(!options.quiet_up_to_date);
         assert!(!options.quiet_up_to_date_explicit);
     }
@@ -385,10 +414,38 @@ mod tests {
 
         assert!(options.watch);
         assert_eq!(options.interval, Duration::from_secs(30 * 60));
+        assert_eq!(
+            options.error_retry_interval,
+            Duration::from_secs(DEFAULT_ERROR_RETRY_SECONDS)
+        );
         assert!(options.quiet_up_to_date);
         assert!(!options.quiet_up_to_date_explicit);
     }
 
+    #[test]
+    fn parses_explicit_watch_error_retry_interval() {
+        let options = parse(&[
+            "bat-official-sync",
+            "--watch",
+            "--interval",
+            "1h",
+            "--error-retry",
+            "45s",
+        ])
+        .unwrap();
+        assert_eq!(options.interval, Duration::from_secs(3600));
+        assert_eq!(options.error_retry_interval, Duration::from_secs(45));
+
+        let options = parse(&[
+            "bat-official-sync",
+            "--watch",
+            "--error-retry-seconds",
+            "30",
+        ])
+        .unwrap();
+        assert_eq!(options.error_retry_interval, Duration::from_secs(30));
+    }
+
     #[test]
     fn explicit_no_quiet_up_to_date_overrides_watch_default() {
         let options = parse(&[
@@ -413,6 +470,10 @@ mod tests {
         let error =
             parse(&["bat-official-sync", "--watch", "--interval-seconds", "0"]).unwrap_err();
         assert!(error.to_string().contains("greater than zero"));
+
+        let error =
+            parse(&["bat-official-sync", "--watch", "--error-retry-seconds", "0"]).unwrap_err();
+        assert!(error.to_string().contains("error retry interval"));
     }
 
     #[test]

infrastructure/src/official_download.rs

@@ -329,6 +329,21 @@ impl OfficialResourcePullService {
             return Err(format!("Refusing to fetch non-official URL: {url}"));
         }
 
+        let attempts = self.retry_attempts.max(1);
+        let mut last_error = None;
+        for attempt in 1..=attempts {
+            match self.fetch_bytes_once(url) {
+                Ok(bytes) => return Ok(bytes),
+                Err(error) => {
+                    last_error = Some(format!("attempt {attempt}/{attempts}: {error}"));
+                }
+            }
+        }
+
+        Err(last_error.unwrap_or_else(|| format!("curl failed for {url} without an attempt")))
+    }
+
+    fn fetch_bytes_once(&self, url: &str) -> Result<Vec<u8>, String> {
         let output = Command::new(&self.curl_command)
             .arg("--fail")
             .arg("--location")
@@ -1078,7 +1093,7 @@ mod tests {
     fn inventory() -> YostarJpDownloadInventory {
         YostarJpDownloadInventory::from_catalog_bytes(
             b"FullPatch_000.zip",
-            b"ExcelDB.db",
+            b"ExcelDB.db ExcelDB.db",
             b"JP_Airi.zip",
         )
     }
@@ -1680,6 +1695,31 @@ fi
         );
     }
 
+    #[test]
+    fn retries_transient_fetch_bytes_failures() {
+        let out_dir = TempDir::new().unwrap();
+        let bin_dir = TempDir::new().unwrap();
+        let curl_path = bin_dir.path().join("curl");
+        write_fake_flaky_curl(&curl_path);
+
+        let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path)
+            .with_retry_attempts(2);
+        let bytes = service
+            .fetch_bytes(
+                "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes",
+            )
+            .unwrap();
+
+        assert_eq!(
+            String::from_utf8(bytes).unwrap(),
+            "https://prod-clientpatch.bluearchiveyostar.com/r93_token/TableBundles/TableCatalog.bytes"
+        );
+        assert_eq!(
+            fs::read_to_string(curl_path.with_extension("state")).unwrap(),
+            "2"
+        );
+    }
+
     #[test]
     fn rejects_non_official_urls() {
         let service = OfficialResourcePullService::new("/tmp/unused");

infrastructure/src/official_game_main_config.rs

@@ -16,6 +16,8 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use tempfile::TempDir;
 
+const DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS: usize = 3;
+
 /// Combined official bootstrap data for the latest client package.
 #[derive(Debug, Clone)]
 pub struct OfficialGameMainConfigBootstrap {
@@ -113,6 +115,21 @@ impl OfficialGameMainConfigBootstrapService {
     }
 
     fn download_file(&self, url: &str, destination: &Path) -> Result<(), String> {
+        let attempts = DEFAULT_BOOTSTRAP_DOWNLOAD_RETRY_ATTEMPTS;
+        let mut last_error = None;
+        for attempt in 1..=attempts {
+            match self.download_file_once(url, destination) {
+                Ok(()) => return Ok(()),
+                Err(error) => {
+                    last_error = Some(format!("attempt {attempt}/{attempts}: {error}"));
+                }
+            }
+        }
+
+        Err(last_error.unwrap_or_else(|| format!("curl failed for {url} without an attempt")))
+    }
+
+    fn download_file_once(&self, url: &str, destination: &Path) -> Result<(), String> {
         let output = Command::new(&self.curl_command)
             .arg("--fail")
             .arg("--location")

infrastructure/src/official_launcher.rs

@@ -8,7 +8,7 @@
 use bat_adapters::official::launcher::{YostarJpLauncherManifestFile, YOSTAR_JP_GAME_TAG};
 use md5::{Digest, Md5};
 use serde::Deserialize;
-use std::process::Command;
+use std::process::{Command, Output};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 /// Official JP launcher API root.
@@ -23,6 +23,8 @@ pub const YOSTAR_JP_LAUNCHER_PRIMARY_CDN_HOST: &str = "launcher-pkg-ba-jp.yo-sta
 /// Official backup JP PC launcher package CDN host.
 pub const YOSTAR_JP_LAUNCHER_BACKUP_CDN_HOST: &str = "launcher-pkg-ba-jp-bk.yo-star.com";
 
+const DEFAULT_LAUNCHER_RETRY_ATTEMPTS: usize = 3;
+
 /// Latest official PC client metadata returned by `/api/launcher/game/config`.
 #[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
 pub struct YostarJpLauncherGameConfig {
@@ -77,6 +79,7 @@ pub struct YostarJpLauncherRemoteManifest {
 pub struct OfficialLauncherBootstrapService {
     curl_command: String,
     launcher_version: String,
+    retry_attempts: usize,
 }
 
 impl OfficialLauncherBootstrapService {
@@ -93,9 +96,16 @@ impl OfficialLauncherBootstrapService {
         Self {
             curl_command: curl_command.into(),
             launcher_version: launcher_version.into(),
+            retry_attempts: DEFAULT_LAUNCHER_RETRY_ATTEMPTS,
         }
     }
 
+    /// Sets the number of attempts for each launcher `curl` transfer.
+    pub fn with_retry_attempts(mut self, retry_attempts: usize) -> Self {
+        self.retry_attempts = retry_attempts.max(1);
+        self
+    }
+
     /// Fetches latest official PC game config.
     pub fn fetch_game_config(&self) -> Result<YostarJpLauncherGameConfig, String> {
         let envelope = self.fetch_launcher_envelope("/api/launcher/game/config")?;
@@ -161,28 +171,26 @@ impl OfficialLauncherBootstrapService {
             ));
         }
 
-        let output = Command::new(&self.curl_command)
-            .arg("--fail")
-            .arg("--location")
-            .arg("--silent")
-            .arg("--show-error")
-            .arg("--url")
-            .arg(manifest_url)
-            .output()
-            .map_err(|error| {
+        let output = self.run_curl_with_retry(
+            || {
+                let mut command = Command::new(&self.curl_command);
+                command
+                    .arg("--fail")
+                    .arg("--location")
+                    .arg("--silent")
+                    .arg("--show-error")
+                    .arg("--url")
+                    .arg(manifest_url);
+                command
+            },
+            |output| {
+                let stderr = String::from_utf8_lossy(&output.stderr);
                 format!(
-                    "Failed to launch curl command {}: {error}",
-                    self.curl_command
+                    "curl failed for launcher manifest {manifest_url}: {}",
+                    stderr.trim()
                 )
-            })?;
-
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            return Err(format!(
-                "curl failed for launcher manifest {manifest_url}: {}",
-                stderr.trim()
-            ));
-        }
+            },
+        )?;
 
         let manifest: YostarJpLauncherRemoteManifest = serde_json::from_slice(&output.stdout)
             .map_err(|error| format!("Failed to parse official launcher manifest: {error}"))?;
@@ -220,29 +228,27 @@ impl OfficialLauncherBootstrapService {
         let url = launcher_api_url(path)?;
         let authorization = launcher_authorization_header(&self.launcher_version, "", None)?;
 
-        let output = Command::new(&self.curl_command)
-            .arg("--fail")
-            .arg("--location")
-            .arg("--silent")
-            .arg("--show-error")
-            .arg("--header")
-            .arg("Content-Type: application/json;charset=UTF-8")
-            .arg("--header")
-            .arg(format!("Authorization: {authorization}"))
-            .arg("--url")
-            .arg(&url)
-            .output()
-            .map_err(|error| {
-                format!(
-                    "Failed to launch curl command {}: {error}",
-                    self.curl_command
-                )
-            })?;
-
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            return Err(format!("curl failed for {url}: {}", stderr.trim()));
-        }
+        let output = self.run_curl_with_retry(
+            || {
+                let mut command = Command::new(&self.curl_command);
+                command
+                    .arg("--fail")
+                    .arg("--location")
+                    .arg("--silent")
+                    .arg("--show-error")
+                    .arg("--header")
+                    .arg("Content-Type: application/json;charset=UTF-8")
+                    .arg("--header")
+                    .arg(format!("Authorization: {authorization}"))
+                    .arg("--url")
+                    .arg(&url);
+                command
+            },
+            |output| {
+                let stderr = String::from_utf8_lossy(&output.stderr);
+                format!("curl failed for {url}: {}", stderr.trim())
+            },
+        )?;
 
         let envelope: LauncherEnvelope = serde_json::from_slice(&output.stdout)
             .map_err(|error| format!("Failed to parse official launcher API response: {error}"))?;
@@ -261,6 +267,34 @@ impl OfficialLauncherBootstrapService {
 
         Ok(envelope)
     }
+
+    fn run_curl_with_retry(
+        &self,
+        mut build_command: impl FnMut() -> Command,
+        failure_message: impl Fn(&Output) -> String,
+    ) -> Result<Output, String> {
+        let attempts = self.retry_attempts.max(1);
+        let mut last_error = None;
+        for attempt in 1..=attempts {
+            match build_command().output() {
+                Ok(output) if output.status.success() => return Ok(output),
+                Ok(output) => {
+                    last_error = Some(format!(
+                        "attempt {attempt}/{attempts}: {}",
+                        failure_message(&output)
+                    ));
+                }
+                Err(error) => {
+                    last_error = Some(format!(
+                        "attempt {attempt}/{attempts}: Failed to launch curl command {}: {error}",
+                        self.curl_command
+                    ));
+                }
+            }
+        }
+
+        Err(last_error.unwrap_or_else(|| format!("curl command {} did not run", self.curl_command)))
+    }
 }
 
 /// Generic official launcher API response envelope.
@@ -394,6 +428,58 @@ pub fn launcher_authorization_header(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::fs;
+    use std::path::Path;
+    use tempfile::TempDir;
+
+    fn write_shell_script(path: &Path, script: &str) {
+        fs::write(path, script).unwrap();
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            let mut permissions = fs::metadata(path).unwrap().permissions();
+            permissions.set_mode(0o755);
+            fs::set_permissions(path, permissions).unwrap();
+        }
+    }
+
+    fn fake_flaky_launcher_curl_script() -> &'static str {
+        r#"#!/bin/sh
+set -eu
+url=""
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        --url)
+            url="$2"
+            shift 2
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+state="$0.state"
+count=0
+if [ -f "$state" ]; then
+    count="$(cat "$state")"
+fi
+count=$((count + 1))
+printf '%s' "$count" > "$state"
+if [ "$count" -eq 1 ]; then
+    printf '%s\n' "temporary launcher failure" >&2
+    exit 35
+fi
+case "$url" in
+    */api/launcher/game/config)
+        printf '%s' '{"code":200,"data":{"game_latest_version":"1.70.436321","game_latest_file_path":"prod/manifest/BlueArchive_JP_TEMP","game_lowest_version":"1.69.0","game_start_params":[]}}'
+        ;;
+    *)
+        printf '%s\n' "unexpected url: $url" >&2
+        exit 22
+        ;;
+esac
+"#
+    }
 
     #[test]
     fn builds_official_launcher_api_url() {
@@ -510,4 +596,25 @@ mod tests {
         assert_eq!(manifest.files.len(), 1);
         assert_eq!(manifest.files[0].path, "/BlueArchive.exe");
     }
+
+    #[test]
+    fn retries_transient_launcher_api_failures() {
+        let bin_dir = TempDir::new().unwrap();
+        let curl_path = bin_dir.path().join("curl");
+        write_shell_script(&curl_path, fake_flaky_launcher_curl_script());
+
+        let service = OfficialLauncherBootstrapService::with_curl_command(
+            "1.7.2",
+            curl_path.to_string_lossy().to_string(),
+        )
+        .with_retry_attempts(2);
+
+        let config = service.fetch_game_config().unwrap();
+
+        assert_eq!(config.game_latest_version, "1.70.436321");
+        assert_eq!(
+            fs::read_to_string(curl_path.with_extension("state")).unwrap(),
+            "2"
+        );
+    }
 }

infrastructure/src/official_pull.rs

@@ -185,14 +185,14 @@ mod tests {
     fn inventory() -> YostarJpDownloadInventory {
         YostarJpDownloadInventory::from_catalog_bytes(
             b"FullPatch_000.zip",
-            b"ExcelDB.db",
+            b"ExcelDB.db ExcelDB.db",
             b"JP_Airi.zip",
         )
     }
 
     fn platform_inventory() -> YostarJpPlatformDownloadInventory {
         YostarJpPlatformDownloadInventory::from_catalog_bytes(
-            b"ExcelDB.db",
+            b"ExcelDB.db ExcelDB.db",
             vec![
                 YostarJpPlatformCatalogInventory::from_catalog_bytes(
                     PatchPlatform::Windows,

infrastructure/tests/official_game_main_config_bootstrap.rs

@@ -315,7 +315,7 @@ elif [[ "$url" == "{addressables_root}/Android_PatchPack/catalog_Android.zip" ]]
 elif [[ "$url" == "{addressables_root}/Android_PatchPack/catalog_Android.hash" ]]; then
   emit_text "catalog-android-hash"
 elif [[ "$url" == "{addressables_root}/TableBundles/TableCatalog.bytes" ]]; then
-  emit_text "ExcelDB.db"
+  emit_text "ExcelDB.db ExcelDB.db"
 elif [[ "$url" == "{addressables_root}/TableBundles/TableCatalog.hash" ]]; then
   emit_text "{table_catalog_hash}"
 elif [[ "$url" == "{addressables_root}/Windows_PatchPack/BundlePackingInfo.bytes" ]]; then
@@ -361,7 +361,7 @@ fi
         server_info_url = TEST_SERVER_INFO_URL,
         connection_group = TEST_CONNECTION_GROUP,
         addressables_root = TEST_ADDRESSABLES_ROOT,
-        table_catalog_hash = xxhash32(b"ExcelDB.db"),
+        table_catalog_hash = xxhash32(b"ExcelDB.db ExcelDB.db"),
         windows_bundle_catalog_hash = xxhash32(b"FullPatch_000.zip"),
         windows_media_catalog_hash = xxhash32(b"JP_Airi_Win.zip"),
         android_bundle_catalog_hash = xxhash32(b"FullPatch_001.zip"),