feat(resource): 增加 CAS 诊断与索引过滤优化
bat-rust / Build and test Rust (push) Canceled after 0s
bat-rust / Build and test Go API (push) Canceled after 0s

补充 doctor cas 只读诊断、校验 CAS 对象分片布局,并将常用 ResourceRepository metadata 查询下推到 SQLite。

Refs G-011
This commit is contained in:
2026-09-01 00:41:41 +08:00
parent 4ed81f0030
commit fdd4075e7e
12 changed files with 467 additions and 22 deletions
+280 -3
View File
@@ -328,6 +328,10 @@ fn run() -> anyhow::Result<i32> {
let healthy = run_doctor_command(&options)?;
Ok(if healthy { 0 } else { EXIT_ERROR })
}
CliCommand::DoctorCas => {
let healthy = run_doctor_cas_command(&options)?;
Ok(if healthy { 0 } else { EXIT_ERROR })
}
CliCommand::CleanStable => {
run_clean_stable_command(&options)?;
Ok(0)
@@ -608,6 +612,7 @@ enum CliCommand {
UnityFsPatchStringField,
UnityFsPatchField,
Doctor,
DoctorCas,
Logs,
CleanStable,
}
@@ -5380,6 +5385,31 @@ impl HumanReport for DoctorReport {
}
}
impl HumanReport for DoctorCasReport {
fn print_human(&self) -> anyhow::Result<()> {
print_title(self.message);
print_field("命令", self.command);
print_field("状态", self.status);
print_field("健康", format_bool(self.healthy));
print_path_field("CAS 目录", &self.cas_root);
print_path_field("对象目录", &self.objects_dir);
print_path_field("元数据库", &self.metadata_db_path);
print_field("对象数", self.object_count);
print_field("总字节", self.total_size);
print_field("无效对象文件", self.invalid_object_count);
println!(" 检查:");
for check in &self.checks {
println!(
" [{}] {} - {}",
if check.ok { "OK" } else { "FAIL" },
check.name,
check.message
);
}
Ok(())
}
}
impl HumanReport for CleanStableReport {
fn print_human(&self) -> anyhow::Result<()> {
print_title(self.message);
@@ -5593,6 +5623,28 @@ struct DoctorReport {
checks: Vec<DoctorCheck>,
}
#[derive(Debug, Serialize)]
struct DoctorCasReport {
command: &'static str,
status: &'static str,
message: &'static str,
healthy: bool,
cas_root: PathBuf,
objects_dir: PathBuf,
metadata_db_path: PathBuf,
object_count: u64,
total_size: u64,
invalid_object_count: u64,
checks: Vec<DoctorCheck>,
}
#[derive(Debug, Default)]
struct CasObjectScanSummary {
object_count: u64,
total_size: u64,
invalid_object_count: u64,
}
fn build_doctor_report(
state_dir: &Path,
config: &OfficialUpdateConfig,
@@ -5773,6 +5825,77 @@ fn build_doctor_report(
})
}
fn build_doctor_cas_report(config: &OfficialUpdateConfig) -> anyhow::Result<DoctorCasReport> {
let cas_root = config
.import_cas_root
.clone()
.unwrap_or_else(|| config.output_root.join(".cas"));
let objects_dir = cas_root.join("objects");
let metadata_db_path = cas_root.join("metadata.sqlite");
let mut checks = vec![
required_dir_check("cas_root", &cas_root, "CAS 根目录可用"),
required_dir_check("cas_objects_dir", &objects_dir, "CAS 对象目录可用"),
required_file_check("cas_metadata_db", &metadata_db_path, "CAS 元数据库文件可用"),
];
let (object_count, total_size, invalid_object_count) = if objects_dir.is_dir() {
match scan_cas_object_directory(&objects_dir) {
Ok(summary) => {
let ok = summary.invalid_object_count == 0;
checks.push(DoctorCheck {
name: "cas_object_scan",
ok,
message: if ok {
format!(
"对象扫描完成:{} 个对象,{} 字节",
summary.object_count, summary.total_size
)
} else {
format!(
"对象扫描发现 {} 个无效文件:{} 个对象,{} 字节",
summary.invalid_object_count, summary.object_count, summary.total_size
)
},
});
(
summary.object_count,
summary.total_size,
summary.invalid_object_count,
)
}
Err(error) => {
checks.push(DoctorCheck {
name: "cas_object_scan",
ok: false,
message: format!("扫描 CAS 对象失败:{error}"),
});
(0, 0, 0)
}
}
} else {
(0, 0, 0)
};
let healthy = checks.iter().all(|check| check.ok);
Ok(DoctorCasReport {
command: "doctor cas",
status: if healthy { "ok" } else { "issues_found" },
message: if healthy {
"CAS 诊断通过"
} else {
"CAS 诊断发现问题"
},
healthy,
cas_root,
objects_dir,
metadata_db_path,
object_count,
total_size,
invalid_object_count,
checks,
})
}
fn run_doctor_command(options: &CliOptions) -> anyhow::Result<bool> {
let report = build_doctor_report(&options.state_dir, &options.config)?;
let healthy = report.healthy;
@@ -5780,6 +5903,13 @@ fn run_doctor_command(options: &CliOptions) -> anyhow::Result<bool> {
Ok(healthy)
}
fn run_doctor_cas_command(options: &CliOptions) -> anyhow::Result<bool> {
let report = build_doctor_cas_report(&options.config)?;
let healthy = report.healthy;
print_report(options.output_format, &report)?;
Ok(healthy)
}
fn path_check(name: &'static str, path: &Path, ready_message: &str) -> DoctorCheck {
let (ok, message) = if path.is_dir() {
(true, format!("{ready_message}{}", path.display()))
@@ -5802,6 +5932,116 @@ fn path_check(name: &'static str, path: &Path, ready_message: &str) -> DoctorChe
DoctorCheck { name, ok, message }
}
fn required_dir_check(name: &'static str, path: &Path, ready_message: &str) -> DoctorCheck {
if path.is_dir() {
DoctorCheck {
name,
ok: true,
message: format!("{ready_message}{}", path.display()),
}
} else if path.exists() {
DoctorCheck {
name,
ok: false,
message: format!("路径存在但不是目录:{}", path.display()),
}
} else {
DoctorCheck {
name,
ok: false,
message: format!("目录不存在:{}", path.display()),
}
}
}
fn required_file_check(name: &'static str, path: &Path, ready_message: &str) -> DoctorCheck {
if path.is_file() {
DoctorCheck {
name,
ok: true,
message: format!("{ready_message}{}", path.display()),
}
} else if path.exists() {
DoctorCheck {
name,
ok: false,
message: format!("路径存在但不是文件:{}", path.display()),
}
} else {
DoctorCheck {
name,
ok: false,
message: format!("文件不存在:{}", path.display()),
}
}
}
fn scan_cas_object_directory(objects_dir: &Path) -> anyhow::Result<CasObjectScanSummary> {
let mut summary = CasObjectScanSummary::default();
scan_cas_object_directory_recursive(objects_dir, objects_dir, &mut summary)?;
Ok(summary)
}
fn scan_cas_object_directory_recursive(
objects_dir: &Path,
path: &Path,
summary: &mut CasObjectScanSummary,
) -> anyhow::Result<()> {
for entry in fs::read_dir(path)? {
let entry = entry?;
let file_type = entry.file_type()?;
let entry_path = entry.path();
if file_type.is_dir() {
scan_cas_object_directory_recursive(objects_dir, &entry_path, summary)?;
continue;
}
if !file_type.is_file() {
continue;
}
let metadata = entry.metadata()?;
let file_name = entry.file_name();
let file_name = file_name.to_str().unwrap_or_default();
if is_cas_object_path(objects_dir, &entry_path, file_name) {
summary.object_count += 1;
summary.total_size += metadata.len();
} else {
summary.invalid_object_count += 1;
}
}
Ok(())
}
fn is_cas_object_path(objects_dir: &Path, object_path: &Path, file_name: &str) -> bool {
if !is_cas_object_file_name(file_name) {
return false;
}
let relative_path = match object_path.strip_prefix(objects_dir) {
Ok(path) => path,
Err(_) => return false,
};
let mut components = relative_path.components();
let Some(std::path::Component::Normal(prefix1)) = components.next() else {
return false;
};
let Some(std::path::Component::Normal(prefix2)) = components.next() else {
return false;
};
let Some(std::path::Component::Normal(name)) = components.next() else {
return false;
};
components.next().is_none()
&& prefix1.to_str() == Some(&file_name[0..2])
&& prefix2.to_str() == Some(&file_name[2..4])
&& name.to_str() == Some(file_name)
}
fn is_cas_object_file_name(file_name: &str) -> bool {
file_name.len() == 64 && file_name.chars().all(|ch| ch.is_ascii_hexdigit())
}
fn safety_check(name: &'static str, result: Result<(), String>, ok_message: &str) -> DoctorCheck {
match result {
Ok(()) => DoctorCheck {
@@ -7101,7 +7341,7 @@ fn parse_args_with_env(
raw_args: impl IntoIterator<Item = String>,
env_lookup: impl Fn(&str) -> Option<String>,
) -> anyhow::Result<CliOptions> {
let mut args = raw_args.into_iter();
let mut args = raw_args.into_iter().peekable();
let binary = args.next().unwrap_or_else(|| "bat".to_string());
let mut options = CliOptions::default();
// `BAT_*` 环境变量(含 .env 加载的)先作为默认值写入,不标记 explicit;
@@ -7196,7 +7436,12 @@ fn parse_args_with_env(
}
"doctor" => {
ensure_command_not_set(options.command, "doctor")?;
options.command = CliCommand::Doctor;
if args.peek().map(|value| value.as_str()) == Some("cas") {
args.next();
options.command = CliCommand::DoctorCas;
} else {
options.command = CliCommand::Doctor;
}
}
"logs" => {
ensure_command_not_set(options.command, "logs")?;
@@ -7845,7 +8090,25 @@ fn parse_args_with_env(
options.progress = false;
options.banner = false;
}
CliCommand::Doctor | CliCommand::CleanStable => {
CliCommand::Doctor => {
if options.sync_option_explicit {
return Err(anyhow::anyhow!(
"doctor 只接受 --output、--state-dir、--curl、--proxy 和 --unzip 等诊断参数"
));
}
options.progress = false;
options.banner = false;
}
CliCommand::DoctorCas => {
if doctor_cas_has_disallowed_sync_options(&options) {
return Err(anyhow::anyhow!(
"doctor cas 只接受 --output、--import-repository、--import-cas-root、--import-resource-db、--state-dir、--curl、--proxy 和 --unzip 等诊断参数"
));
}
options.progress = false;
options.banner = false;
}
CliCommand::CleanStable => {
if options.sync_option_explicit {
return Err(anyhow::anyhow!(
"doctor/clean-stable 只接受 --output、--state-dir、--curl、--proxy 和 --unzip 等诊断参数"
@@ -8282,6 +8545,19 @@ fn parse_args_with_env(
Ok(options)
}
fn doctor_cas_has_disallowed_sync_options(options: &CliOptions) -> bool {
let mut allowed = options.env_baseline_config.clone();
allowed.output_root = options.config.output_root.clone();
allowed.curl_command = options.config.curl_command.clone();
allowed.curl_proxy = options.config.curl_proxy.clone();
allowed.unzip_command = options.config.unzip_command.clone();
allowed.import_repository = options.config.import_repository;
allowed.import_cas_root = options.config.import_cas_root.clone();
allowed.import_resource_repository_path =
options.config.import_resource_repository_path.clone();
options.config != allowed
}
fn parse_resource_command(
args: &mut impl Iterator<Item = String>,
options: &mut CliOptions,
@@ -8623,6 +8899,7 @@ fn print_usage(binary: &str) {
eprintln!(" reload Ask daemon to rediscover metadata and force refresh");
eprintln!(" logs Show daemon log tail");
eprintln!(" doctor Run runtime diagnostics");
eprintln!(" doctor cas Inspect local CAS storage");
eprintln!(" clean-stable Remove .part/.tmp/stale lock, pid, and socket files");
eprintln!();
eprintln!("Examples:");
+74
View File
@@ -1440,6 +1440,22 @@ fn parses_management_and_resource_commands() {
let options = parse(&["bat", "resource", "status"]).unwrap();
assert_eq!(options.command, CliCommand::ResourceIndex);
let doctor_cas = parse(&["bat", "doctor", "cas", "--output", "/tmp/bat-resources"]).unwrap();
assert_eq!(doctor_cas.command, CliCommand::DoctorCas);
assert_eq!(
doctor_cas.config.output_root,
PathBuf::from("/tmp/bat-resources")
);
assert!(parse(&[
"bat",
"doctor",
"cas",
"--output",
"/tmp/bat-resources",
"--force"
])
.is_err());
let index = parse(&[
"bat",
"resource-index",
@@ -1601,6 +1617,64 @@ fn parses_management_and_resource_commands() {
.contains("--resource-type/--hash/--platform/--bundle-path"));
}
#[test]
fn doctor_cas_report_counts_objects_read_only() {
let temp = tempfile::TempDir::new().unwrap();
let output_root = temp.path().join("output");
let cas_root = output_root.join(".cas");
let objects_dir = cas_root.join("objects/01/23");
fs::create_dir_all(&objects_dir).unwrap();
let object_path =
objects_dir.join("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
fs::write(&object_path, b"payload").unwrap();
fs::write(cas_root.join("metadata.sqlite"), b"metadata").unwrap();
let report = build_doctor_cas_report(&OfficialUpdateConfig {
output_root,
..OfficialUpdateConfig::default()
})
.unwrap();
assert_eq!(report.command, "doctor cas");
assert!(report.healthy);
assert_eq!(report.object_count, 1);
assert_eq!(report.total_size, 7);
assert_eq!(report.invalid_object_count, 0);
assert!(report
.checks
.iter()
.any(|check| check.name == "cas_object_scan" && check.ok));
}
#[test]
fn doctor_cas_report_flags_invalid_object_layout() {
let temp = tempfile::TempDir::new().unwrap();
let output_root = temp.path().join("output");
let cas_root = output_root.join(".cas");
let objects_dir = cas_root.join("objects/ab/cd");
fs::create_dir_all(&objects_dir).unwrap();
fs::write(cas_root.join("metadata.sqlite"), b"metadata").unwrap();
fs::write(
objects_dir.join("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
b"payload",
)
.unwrap();
let report = build_doctor_cas_report(&OfficialUpdateConfig {
output_root,
..OfficialUpdateConfig::default()
})
.unwrap();
assert!(!report.healthy);
assert_eq!(report.object_count, 0);
assert_eq!(report.invalid_object_count, 1);
assert!(report
.checks
.iter()
.any(|check| check.name == "cas_object_scan" && !check.ok));
}
#[test]
fn parses_write_patch_commands_without_output_confusion() {
let patch = parse(&[
+92 -4
View File
@@ -186,6 +186,39 @@ impl SqliteResourceRepository {
)
.await?;
Self::execute_query(
&self.pool,
sqlx::query(
r#"
CREATE INDEX IF NOT EXISTS idx_resources_release_id
ON resources(json_extract(metadata_json, '$.official_release_id'))
"#,
),
)
.await?;
Self::execute_query(
&self.pool,
sqlx::query(
r#"
CREATE INDEX IF NOT EXISTS idx_resources_platform
ON resources(json_extract(metadata_json, '$.platform'))
"#,
),
)
.await?;
Self::execute_query(
&self.pool,
sqlx::query(
r#"
CREATE INDEX IF NOT EXISTS idx_resources_bundle_path
ON resources(json_extract(metadata_json, '$.bundle_path'))
"#,
),
)
.await?;
Ok(())
}
@@ -340,6 +373,51 @@ impl SqliteResourceRepository {
builder.push_bind(destination);
}
if let Some(release_id) = &query.official_release_id {
push_condition_prefix(builder, &mut has_where);
builder.push("json_extract(metadata_json, '$.official_release_id') = ");
builder.push_bind(release_id);
}
if let Some(platform) = &query.platform {
push_condition_prefix(builder, &mut has_where);
builder.push("json_extract(metadata_json, '$.platform') = ");
builder.push_bind(platform);
}
if let Some(bundle_path) = &query.bundle_path {
push_condition_prefix(builder, &mut has_where);
builder.push("json_extract(metadata_json, '$.bundle_path') = ");
builder.push_bind(bundle_path);
}
if let Some(archive_entry) = &query.archive_entry {
push_condition_prefix(builder, &mut has_where);
builder.push(
"EXISTS (SELECT 1 FROM json_each(metadata_json, '$.archive_entries') AS archive_entries WHERE archive_entries.value = ",
);
builder.push_bind(archive_entry);
builder.push(")");
}
if let Some(parse_status) = &query.parse_status {
push_condition_prefix(builder, &mut has_where);
builder.push(
"EXISTS (SELECT 1 FROM json_each(metadata_json, '$.parse_statuses') AS parse_statuses WHERE parse_statuses.value = ",
);
builder.push_bind(parse_status);
builder.push(")");
}
if let Some(text_unit_format) = &query.text_unit_format {
push_condition_prefix(builder, &mut has_where);
builder.push(
"EXISTS (SELECT 1 FROM json_each(metadata_json, '$.text_unit_formats') AS text_unit_formats WHERE text_unit_formats.value = ",
);
builder.push_bind(text_unit_format);
builder.push(")");
}
Ok(())
}
@@ -374,10 +452,6 @@ impl SqliteResourceRepository {
}
async fn count_resources(&self, query: &ResourceQuery) -> bat_core::Result<u64> {
if query.requires_resource_scan() {
return Ok(self.fetch_resources(query, None).await?.len() as u64);
}
let mut builder = QueryBuilder::<Sqlite>::new("SELECT COUNT(*) FROM resources");
Self::apply_filters(&mut builder, query)?;
@@ -801,6 +875,20 @@ mod tests {
#[tokio::test]
async fn sqlite_repository_persists_and_filters_resources() {
let (_temp_dir, repository) = sqlite_repository().await;
let indexes = sqlx::query_scalar::<_, String>(
"SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'resources'",
)
.fetch_all(&repository.pool)
.await
.unwrap();
assert!(indexes
.iter()
.any(|name| name == "idx_resources_release_id"));
assert!(indexes.iter().any(|name| name == "idx_resources_platform"));
assert!(indexes
.iter()
.any(|name| name == "idx_resources_bundle_path"));
let mut resource = resource(
"resource/sqlite-a",
"assets/model.bundle",