feat: harden official sync scheduling and integrity

2026-07-22 06:35:16 +08:00 · 2026-07-11 01:06:02 +08:00
parent 2bdc55a1be
commit 264e78c440
11 changed files with 474 additions and 99 deletions
@@ -7,12 +7,16 @@ use serde::Serialize;
 use std::env;
 use std::path::PathBuf;
 use std::thread;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

 const EXIT_ERROR: i32 = 1;
 const EXIT_LOCKED: i32 = 75;
 const DEFAULT_WATCH_INTERVAL_SECONDS: u64 = 60 * 60;
 const DEFAULT_ERROR_RETRY_SECONDS: u64 = 60;
+const SECONDS_PER_DAY: u64 = 24 * 60 * 60;
+const BEIJING_UTC_OFFSET_SECONDS: u64 = 8 * 60 * 60;
+const DAILY_FORCED_REFRESH_LOCAL_SECONDS: [u64; 3] = [3 * 60 * 60, 16 * 60 * 60, 18 * 60 * 60];
+const DAILY_FORCED_REFRESH_LABEL: &str = "UTC+8 03:00, 16:00, 18:00";
 const STARTUP_BANNER: &str = r#"
 ============================================================
  ____  _             _             _     _           _____           _ _    _ _
@@ -120,25 +124,64 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
    }
    let service = OfficialUpdateService::new();
    let mut logger = ProgressLogger::new(options.progress);
+    let mut next_forced_refresh_at = next_forced_refresh_at_or_after(SystemTime::now());
+    let mut pending_scheduled_force = false;
+    logger.log_text(
+        "watch",
+        format!(
+            "daily forced refresh schedule: {DAILY_FORCED_REFRESH_LABEL}; next forced refresh in {}",
+            format_duration(duration_until(next_forced_refresh_at, SystemTime::now()))
+        ),
+    );
    loop {
+        let now = SystemTime::now();
+        if now >= next_forced_refresh_at {
+            pending_scheduled_force = true;
+            logger.log_text(
+                "watch",
+                format!("scheduled forced refresh triggered at {DAILY_FORCED_REFRESH_LABEL}"),
+            );
+            next_forced_refresh_at = next_forced_refresh_after(now);
+        }
+
+        let mut iteration_config = options.config.clone();
+        if pending_scheduled_force {
+            iteration_config.force = true;
+        }
+
        let mut sleep_for = options.interval;
-        logger.log_text("watch", "starting watch iteration");
-        match service.run_with_progress(&options.config, |event| logger.log(event)) {
+        logger.log_text(
+            "watch",
+            if pending_scheduled_force {
+                "starting watch iteration with scheduled force refresh"
+            } else {
+                "starting watch iteration"
+            },
+        );
+        match service.run_with_progress(&iteration_config, |event| logger.log(event)) {
            Ok(report) => {
+                if pending_scheduled_force {
+                    pending_scheduled_force = false;
+                }
                if should_print_status(report.update_status, options.quiet_up_to_date) {
                    println!("{}", serde_json::to_string_pretty(&report)?);
                }
+                sleep_for =
+                    sleep_for.min(duration_until(next_forced_refresh_at, SystemTime::now()));
                logger.log_text(
                    "watch",
                    format!(
-                        "iteration finished with status={}; next check in {}",
+                        "iteration finished with status={}; next check in {}; next scheduled force refresh in {}",
                        report.update_status.as_str(),
-                        format_duration(sleep_for)
+                        format_duration(sleep_for),
+                        format_duration(duration_until(next_forced_refresh_at, SystemTime::now()))
                    ),
                );
            }
            Err(error) => {
                sleep_for = options.error_retry_interval;
+                sleep_for =
+                    sleep_for.min(duration_until(next_forced_refresh_at, SystemTime::now()));
                logger.log_text(
                    "watch",
                    format!(
@@ -161,6 +204,62 @@ fn run_watch(options: CliOptions) -> anyhow::Result<()> {
    }
 }

+fn next_forced_refresh_at_or_after(now: SystemTime) -> SystemTime {
+    forced_refresh_time(now, ForcedRefreshBoundary::AtOrAfter)
+}
+
+fn next_forced_refresh_after(now: SystemTime) -> SystemTime {
+    forced_refresh_time(now, ForcedRefreshBoundary::After)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ForcedRefreshBoundary {
+    AtOrAfter,
+    After,
+}
+
+fn forced_refresh_time(now: SystemTime, boundary: ForcedRefreshBoundary) -> SystemTime {
+    let local_seconds = beijing_local_seconds_since_epoch(now);
+    let local_day = local_seconds / SECONDS_PER_DAY;
+    let local_second_of_day = local_seconds % SECONDS_PER_DAY;
+
+    for scheduled_second in DAILY_FORCED_REFRESH_LOCAL_SECONDS {
+        let matches_boundary = match boundary {
+            ForcedRefreshBoundary::AtOrAfter => scheduled_second >= local_second_of_day,
+            ForcedRefreshBoundary::After => scheduled_second > local_second_of_day,
+        };
+        if matches_boundary {
+            return system_time_from_beijing_local_seconds(
+                local_day
+                    .saturating_mul(SECONDS_PER_DAY)
+                    .saturating_add(scheduled_second),
+            );
+        }
+    }
+
+    system_time_from_beijing_local_seconds(
+        local_day
+            .saturating_add(1)
+            .saturating_mul(SECONDS_PER_DAY)
+            .saturating_add(DAILY_FORCED_REFRESH_LOCAL_SECONDS[0]),
+    )
+}
+
+fn beijing_local_seconds_since_epoch(time: SystemTime) -> u64 {
+    time.duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+        .saturating_add(BEIJING_UTC_OFFSET_SECONDS)
+}
+
+fn system_time_from_beijing_local_seconds(local_seconds: u64) -> SystemTime {
+    UNIX_EPOCH + Duration::from_secs(local_seconds.saturating_sub(BEIJING_UTC_OFFSET_SECONDS))
+}
+
+fn duration_until(deadline: SystemTime, now: SystemTime) -> Duration {
+    deadline.duration_since(now).unwrap_or_default()
+}
+
 #[derive(Debug, Clone)]
 struct ProgressLogger {
    enabled: bool,
@@ -365,6 +464,9 @@ fn print_usage(binary: &str) {
         [--curl curl] [--unzip unzip] [--dry-run] [--plan] [--force] [--audit-local|--no-audit-local] [--repair|--no-repair] \
         [--watch] [--interval 1h|60m|3600s] [--error-retry 60s] [--quiet-up-to-date|--no-quiet-up-to-date] [--progress|--no-progress] [--banner|--no-banner]"
    );
+    eprintln!(
+        "watch mode also runs scheduled force refreshes every day at {DAILY_FORCED_REFRESH_LABEL}"
+    );
 }

 fn parse_duration(value: &str) -> anyhow::Result<Duration> {
@@ -616,6 +718,35 @@ mod tests {
        assert_eq!(parse_duration("3600").unwrap(), Duration::from_secs(3600));
    }

+    #[test]
+    fn scheduled_forced_refresh_uses_beijing_local_times() {
+        let day = 20_000;
+
+        assert_eq!(
+            next_forced_refresh_at_or_after(beijing_time(day, 2, 30, 0)),
+            beijing_time(day, 3, 0, 0)
+        );
+        assert_eq!(
+            next_forced_refresh_at_or_after(beijing_time(day, 3, 0, 0)),
+            beijing_time(day, 3, 0, 0)
+        );
+        assert_eq!(
+            next_forced_refresh_after(beijing_time(day, 3, 0, 0)),
+            beijing_time(day, 16, 0, 0)
+        );
+        assert_eq!(
+            next_forced_refresh_after(beijing_time(day, 18, 0, 0)),
+            beijing_time(day + 1, 3, 0, 0)
+        );
+        assert_eq!(
+            duration_until(
+                next_forced_refresh_at_or_after(beijing_time(day, 17, 45, 0)),
+                beijing_time(day, 17, 45, 0)
+            ),
+            Duration::from_secs(15 * 60)
+        );
+    }
+
    #[test]
    fn formats_progress_durations() {
        assert_eq!(format_duration(Duration::from_millis(7)), "00:00.007");
@@ -642,4 +773,13 @@ mod tests {
        );
        assert_eq!(OfficialUpdateStatus::Downloaded.as_str(), "downloaded");
    }
+
+    fn beijing_time(day: u64, hour: u64, minute: u64, second: u64) -> SystemTime {
+        system_time_from_beijing_local_seconds(
+            day.saturating_mul(SECONDS_PER_DAY)
+                .saturating_add(hour.saturating_mul(60 * 60))
+                .saturating_add(minute.saturating_mul(60))
+                .saturating_add(second),
+        )
+    }
 }
@@ -276,6 +276,22 @@ impl OfficialLocalManifestAuditReport {
    }
 }

+/// Existing local state for an official pull plan.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OfficialLocalResourceState {
+    /// Number of expected URLs that already have download-manifest entries.
+    pub manifest_entry_count: usize,
+    /// Number of expected URL destinations that already exist on disk.
+    pub existing_file_count: usize,
+}
+
+impl OfficialLocalResourceState {
+    /// Returns true when this plan has any reusable or auditable local state.
+    pub fn has_any_resources(self) -> bool {
+        self.manifest_entry_count > 0 || self.existing_file_count > 0
+    }
+}
+
 /// Executes an official pull plan with the system `curl` command.
 #[derive(Debug, Clone)]
 pub struct OfficialResourcePullService {
@@ -357,6 +373,9 @@ impl OfficialResourcePullService {
        let urls = plan.all_urls()?;
        let total = urls.len();
        let mut items = Vec::new();
+        let mut verified_hashes = Vec::new();
+        let mut verified_hash_urls = HashSet::<String>::new();
+        let mut processed_urls = HashSet::<String>::new();
        for (offset, url) in urls.into_iter().enumerate() {
            if should_cancel() {
                return Err("official resource pull interrupted by shutdown request".to_string());
@@ -406,6 +425,14 @@ impl OfficialResourcePullService {
                result.bytes,
                result.transferred_bytes,
            ));
+            processed_urls.insert(url.clone());
+            self.verify_ready_official_hashes(
+                &official_hash_pairs,
+                &processed_urls,
+                &mut verified_hash_urls,
+                &mut verified_hashes,
+                &mut manifest,
+            )?;
            items.push(OfficialResourcePullItem {
                url,
                destination,
@@ -418,8 +445,7 @@ impl OfficialResourcePullService {
        if should_cancel() {
            return Err("official resource pull interrupted by shutdown request".to_string());
        }
-
-        let verified_hashes = self.verify_downloaded_official_hashes(&official_hash_pairs)?;
+        self.verify_all_official_hashes_are_complete(&official_hash_pairs, &verified_hash_urls)?;

        Ok(OfficialResourcePullReport {
            items,
@@ -447,6 +473,34 @@ impl OfficialResourcePullService {
        Ok(OfficialLocalManifestAuditReport { items })
    }

+    /// Returns whether the current output root already contains local state for
+    /// this plan. This is intentionally cheaper than a full audit and is used
+    /// to choose between audit-then-repair and first-time full pull flows.
+    pub fn local_resource_state(
+        &self,
+        plan: &OfficialResourcePullPlan,
+    ) -> Result<OfficialLocalResourceState, String> {
+        let manifest = self.read_download_manifest()?;
+        let mut manifest_entry_count = 0;
+        let mut existing_file_count = 0;
+
+        for url in plan.all_urls()? {
+            if manifest.entries.contains_key(&url) {
+                manifest_entry_count += 1;
+            }
+
+            let destination = self.destination_for_url(&url)?;
+            if file_len_if_exists(&destination)?.is_some() {
+                existing_file_count += 1;
+            }
+        }
+
+        Ok(OfficialLocalResourceState {
+            manifest_entry_count,
+            existing_file_count,
+        })
+    }
+
    /// Fetches an official URL into memory.
    ///
    /// This is intended for small discovery inputs such as `server-info`,
@@ -604,37 +658,103 @@ impl OfficialResourcePullService {
        Ok(())
    }

-    fn verify_downloaded_official_hashes(
+    fn verify_ready_official_hashes(
        &self,
        pairs: &[OfficialSeedHashPair],
-    ) -> Result<Vec<OfficialResourceHashVerification>, String> {
-        let mut verified = Vec::new();
-
+        processed_urls: &HashSet<String>,
+        verified_hash_urls: &mut HashSet<String>,
+        verified_hashes: &mut Vec<OfficialResourceHashVerification>,
+        manifest: &mut OfficialDownloadManifest,
+    ) -> Result<(), String> {
        for pair in pairs {
-            let data_path = self.destination_for_url(&pair.data_url)?;
-            let hash_path = self.destination_for_url(&pair.hash_url)?;
-            let data = fs::read(&data_path).map_err(|error| {
-                format!(
-                    "Failed to read official hash data file {}: {error}",
-                    data_path.display()
-                )
-            })?;
-            let hash = fs::read(&hash_path).map_err(|error| {
-                format!(
-                    "Failed to read official hash sidecar {}: {error}",
-                    hash_path.display()
-                )
-            })?;
+            if verified_hash_urls.contains(&pair.hash_url) {
+                continue;
+            }

-            verified.push(verify_official_seed_catalog_hash(
-                &pair.data_url,
-                &data,
-                &pair.hash_url,
-                &hash,
-            )?);
+            if !processed_urls.contains(&pair.data_url) || !processed_urls.contains(&pair.hash_url)
+            {
+                continue;
+            }
+
+            if !self.hash_pair_files_exist(pair)? {
+                continue;
+            }
+
+            let verification = match self.verify_official_seed_hash_pair_from_disk(pair) {
+                Ok(verification) => verification,
+                Err(error) => {
+                    if let Err(cleanup_error) =
+                        self.invalidate_download_manifest_hash_pair(manifest, pair)
+                    {
+                        return Err(format!(
+                            "{error}; additionally failed to invalidate local manifest entries for official hash pair: {cleanup_error}"
+                        ));
+                    }
+
+                    return Err(error);
+                }
+            };
+
+            verified_hashes.push(verification);
+            verified_hash_urls.insert(pair.hash_url.clone());
        }

-        Ok(verified)
+        Ok(())
+    }
+
+    fn hash_pair_files_exist(&self, pair: &OfficialSeedHashPair) -> Result<bool, String> {
+        let data_path = self.destination_for_url(&pair.data_url)?;
+        let hash_path = self.destination_for_url(&pair.hash_url)?;
+        Ok(file_len_if_exists(&data_path)?.is_some() && file_len_if_exists(&hash_path)?.is_some())
+    }
+
+    fn verify_official_seed_hash_pair_from_disk(
+        &self,
+        pair: &OfficialSeedHashPair,
+    ) -> Result<OfficialResourceHashVerification, String> {
+        let data_path = self.destination_for_url(&pair.data_url)?;
+        let hash_path = self.destination_for_url(&pair.hash_url)?;
+        let data = fs::read(&data_path).map_err(|error| {
+            format!(
+                "Failed to read official hash data file {}: {error}",
+                data_path.display()
+            )
+        })?;
+        let hash = fs::read(&hash_path).map_err(|error| {
+            format!(
+                "Failed to read official hash sidecar {}: {error}",
+                hash_path.display()
+            )
+        })?;
+
+        verify_official_seed_catalog_hash(&pair.data_url, &data, &pair.hash_url, &hash)
+    }
+
+    fn invalidate_download_manifest_hash_pair(
+        &self,
+        manifest: &mut OfficialDownloadManifest,
+        pair: &OfficialSeedHashPair,
+    ) -> Result<(), String> {
+        manifest.entries.remove(&pair.data_url);
+        manifest.entries.remove(&pair.hash_url);
+        self.write_download_manifest(manifest)
+    }
+
+    fn verify_all_official_hashes_are_complete(
+        &self,
+        pairs: &[OfficialSeedHashPair],
+        verified_hash_urls: &HashSet<String>,
+    ) -> Result<(), String> {
+        for pair in pairs {
+            if !verified_hash_urls.contains(&pair.hash_url) {
+                return Err(format!(
+                    "official hash pair was not verified for {} using {}",
+                    pair.data_url, pair.hash_url
+                ));
+            }
+        }
+
+        Ok(())
    }

    fn read_download_manifest(&self) -> Result<OfficialDownloadManifest, String> {
@@ -1607,6 +1727,43 @@ fi
            .all(|item| item.status == OfficialResourcePullStatus::Downloaded));
    }

+    #[test]
+    fn local_resource_state_reports_manifest_entries_and_existing_files() {
+        let out_dir = TempDir::new().unwrap();
+        let bin_dir = TempDir::new().unwrap();
+        let curl_path = bin_dir.path().join("curl");
+        write_fake_curl(&curl_path);
+
+        let service = OfficialResourcePullService::with_curl_command(out_dir.path(), &curl_path);
+        let plan = build_official_pull_plan(discovery_plan(), inventory());
+        let urls = plan.all_urls().unwrap();
+
+        let empty_state = service.local_resource_state(&plan).unwrap();
+        assert_eq!(empty_state.manifest_entry_count, 0);
+        assert_eq!(empty_state.existing_file_count, 0);
+        assert!(!empty_state.has_any_resources());
+
+        let first_destination = service.destination_for_url(&urls[0]).unwrap();
+        fs::create_dir_all(first_destination.parent().unwrap()).unwrap();
+        fs::write(&first_destination, b"local-only").unwrap();
+
+        let file_only_state = service.local_resource_state(&plan).unwrap();
+        assert_eq!(file_only_state.manifest_entry_count, 0);
+        assert_eq!(file_only_state.existing_file_count, 1);
+        assert!(file_only_state.has_any_resources());
+
+        let mut manifest = OfficialDownloadManifest::default();
+        service
+            .record_download_manifest_entry(&mut manifest, &urls[0], &first_destination)
+            .unwrap();
+        service.write_download_manifest(&manifest).unwrap();
+
+        let manifest_state = service.local_resource_state(&plan).unwrap();
+        assert_eq!(manifest_state.manifest_entry_count, 1);
+        assert_eq!(manifest_state.existing_file_count, 1);
+        assert!(manifest_state.has_any_resources());
+    }
+
    #[test]
    fn redownloads_existing_file_when_manifest_hash_mismatches() {
        let out_dir = TempDir::new().unwrap();
@@ -1677,6 +1834,17 @@ fi

        let error = service.pull(&plan).unwrap_err();
        assert!(error.contains("official hash mismatch"));
+
+        let manifest = service.read_download_manifest().unwrap();
+        assert!(manifest.entries.is_empty());
+
+        let audit = service.audit_local_manifest(&plan).unwrap();
+        assert!(!audit.is_clean());
+        assert_eq!(audit.repair_needed_count(), 2);
+        assert!(audit
+            .items
+            .iter()
+            .all(|item| { item.status == OfficialLocalManifestAuditStatus::MissingManifestEntry }));
    }

    #[test]
@@ -627,35 +627,39 @@ impl OfficialUpdateService {
                sync_plan.decision, config.force, remote_should_download
            ),
        ));
-        let pull_plan = if config.audit_local || remote_should_download || config.plan {
-            progress(OfficialUpdateProgress::new(
-                "plan",
-                "building official pull plan from seed catalogs",
-            ));
-            Some(build_pull_plan(
-                &server_info,
-                &connection_group,
-                &app_version,
-                platforms,
-                &fetcher,
-                &mut progress,
-                &mut should_cancel,
-            )?)
-        } else {
-            progress(OfficialUpdateProgress::new(
-                "plan",
-                "pull plan not needed for clean remote state and disabled plan output",
-            ));
-            None
-        };
-        if let Some(pull_plan) = pull_plan.as_ref() {
-            let url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len();
-            progress(OfficialUpdateProgress::new(
-                "plan",
-                format!("pull plan contains {url_count} official URLs"),
-            ));
-        }
-        let local_audit = if config.audit_local {
+        progress(OfficialUpdateProgress::new(
+            "plan",
+            "building official pull plan from latest seed catalogs",
+        ));
+        let pull_plan = build_pull_plan(
+            &server_info,
+            &connection_group,
+            &app_version,
+            platforms,
+            &fetcher,
+            &mut progress,
+            &mut should_cancel,
+        )?;
+        let url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len();
+        progress(OfficialUpdateProgress::new(
+            "plan",
+            format!("pull plan contains {url_count} official URLs"),
+        ));
+        let local_state = fetcher
+            .local_resource_state(&pull_plan)
+            .map_err(anyhow::Error::msg)?;
+        let has_local_resources = local_state.has_any_resources();
+        progress(OfficialUpdateProgress::new(
+            "local-state",
+            format!(
+                "manifest_entries={} existing_files={} has_local_resources={}",
+                local_state.manifest_entry_count,
+                local_state.existing_file_count,
+                has_local_resources
+            ),
+        ));
+        let should_audit_local = config.audit_local && has_local_resources;
+        let local_audit = if should_audit_local {
            progress(OfficialUpdateProgress::new(
                "audit",
                format!(
@@ -665,13 +669,15 @@ impl OfficialUpdateService {
            ));
            Some(
                fetcher
-                    .audit_local_manifest(
-                        pull_plan
-                            .as_ref()
-                            .ok_or_else(|| anyhow::anyhow!("local audit requires a pull plan"))?,
-                    )
+                    .audit_local_manifest(&pull_plan)
                    .map_err(anyhow::Error::msg)?,
            )
+        } else if config.audit_local {
+            progress(OfficialUpdateProgress::new(
+                "audit",
+                "no local resources found; skipping local audit before first full pull",
+            ));
+            None
        } else {
            progress(OfficialUpdateProgress::new(
                "audit",
@@ -692,7 +698,8 @@ impl OfficialUpdateService {
                .as_ref()
                .map(|audit| !audit.is_clean())
                .unwrap_or(false);
-        let should_download = remote_should_download || repair_needed;
+        let initial_pull_needed = !has_local_resources;
+        let should_download = remote_should_download || repair_needed || initial_pull_needed;
        progress(OfficialUpdateProgress::new(
            "audit",
            format!(
@@ -703,8 +710,8 @@ impl OfficialUpdateService {
        progress(OfficialUpdateProgress::new(
            "decision",
            format!(
-                "final should_download={} repair_needed={}",
-                should_download, repair_needed
+                "final should_download={} repair_needed={} initial_pull_needed={}",
+                should_download, repair_needed, initial_pull_needed
            ),
        ));
        check_shutdown_requested(&mut should_cancel)?;
@@ -760,13 +767,7 @@ impl OfficialUpdateService {

        if config.dry_run {
            if config.plan {
-                let urls = pull_plan
-                    .as_ref()
-                    .ok_or_else(|| {
-                        anyhow::anyhow!("dry-run plan requested but no pull plan exists")
-                    })?
-                    .all_urls()
-                    .map_err(anyhow::Error::msg)?;
+                let urls = pull_plan.all_urls().map_err(anyhow::Error::msg)?;
                report.download_url_count = Some(urls.len());
                report.download_urls = urls;
                progress(OfficialUpdateProgress::new(
@@ -791,9 +792,6 @@ impl OfficialUpdateService {
        }
        check_shutdown_requested(&mut should_cancel)?;

-        let pull_plan = pull_plan
-            .as_ref()
-            .ok_or_else(|| anyhow::anyhow!("download requested but no pull plan exists"))?;
        let download_url_count = pull_plan.all_urls().map_err(anyhow::Error::msg)?.len();
        progress(OfficialUpdateProgress::new(
            "download",
@@ -801,7 +799,7 @@ impl OfficialUpdateService {
        ));
        let pull_report = fetcher
            .pull_with_progress_and_cancellation(
-                pull_plan,
+                &pull_plan,
                |event| {
                    progress(progress_from_pull_event(event));
                },
@@ -824,7 +822,7 @@ impl OfficialUpdateService {
        ));
        check_shutdown_requested(&mut should_cancel)?;
        let final_audit = fetcher
-            .audit_local_manifest(pull_plan)
+            .audit_local_manifest(&pull_plan)
            .map_err(anyhow::Error::msg)?;
        progress(OfficialUpdateProgress::new(
            "snapshot",