diff --git a/TODO.md b/TODO.md index 380399b..d73febc 100644 --- a/TODO.md +++ b/TODO.md @@ -2,7 +2,7 @@ > 基线：`experiment` > -> 当前已知基线提交：`37d49c9793d25288fd2fcc60b95c1ba55bdf7623` +> 当前已知基线提交：`045d598400f9435051bb9554e06e123d0d8a3704` > > 工程原则：不以“最小修复”为目标。所有修复与新功能优先考虑长期可维护性、可用性、安全性、明确契约、回归测试和后续扩展成本；避免无关重构，但一旦处理一个问题，应完整收口其工程边界。 @@ -168,7 +168,7 @@ transition、generation race、过期和失败失效回归，并验证现有 CDN # T02 — ResourceRepository 查询契约统一 **类型：** P2 -**状态：** Ready +**状态：** Done ## 问题 @@ -179,7 +179,7 @@ transition、generation race、过期和失败失效回归，并验证现有 CDN ```text * 不跨 / ** 可以跨 / -? 匹配一个字符 +? 匹配一个 Unicode scalar value ``` 当前 Rust matcher、SQLite LIKE 转换以及 `list()` / `count()` 的实际过滤语义并不完全一致。 @@ -225,12 +225,23 @@ list(query).len() != count(query) ## 关系 -**Hard Blocker：** +**历史 Hard Blocker（已满足）：** ```text T02 → T13 ``` +## 完成记录 + +已将 `ResourceQuery.path_pattern` 收口为 `bat-core` 共享的 Unicode scalar +matcher：`*` 与 `?` 不跨 `/`，`**` 支持跨 `/`，`**/` 可匹配根目录和任意深度目录， +`/`、大小写、`%`、`_`、`\` 均按字面量处理，空路径/空模式和连续通配符都有确定语义； +匹配使用有界动态规划，不再使用按字节递归回溯。InMemory 与 SQLite 共用同一最终判定， +SQLite 仅下推安全候选过滤，不再把 SQL `LIKE` 当作正式 glob 语义；包含路径或 metadata +后置过滤的 `count()` 会复用完整查询结果，保证与 `list()` 精确相等，无后置过滤时保留 +SQL COUNT 快速路径。已补齐 `*`、`**`、`?`、分隔符、Unicode、特殊字面量、无匹配、 +组合过滤、稳定 ID 顺序以及 InMemory/SQLite list/count parity 回归测试。 + --- # T03 — SQLite Schema Migration 体系化 @@ -828,7 +839,7 @@ T09/T10 ─→ T11 增加 AssetBundle/Patch 真实验证 # T13 — G-011 ResourceRepository 查询扩展 **类型：** Feature -**状态：** Blocked by T02 +**状态：** Ready ## 目标 @@ -844,9 +855,7 @@ T09/T10 ─→ T11 增加 AssetBundle/Patch 真实验证 ## 依赖 -**Hard Blocker：T02** - -## 完成记录 +前置任务 T02 已完成，查询 contract 已满足。 # T14 — G-012 Translation Memory 后续扩展 @@ -1025,7 +1034,7 @@ resource bootstrap │ │ │ ▼ ▼ ▼ -T02 ResourceRepository ──Hard──→ T13 G-011 +T02 ResourceRepository ──Hard──→ T13 G-011（历史前置关系，已满足） T03 SQLite Migration ──Hard──→ T04 TM Trust ──Hard──→ T14 G-012 │ @@ -1069,12 +1078,12 @@ Stable Resource / Release / AssetBundle Platform ## Ready ```text -T02 — ResourceRepository 查询契约统一 T05 — bat.sock 本地 IPC 安全与资源边界 T06 — AssetBundle / ZIP Parser Resource Budget T07 — Durable Atomic State Write T11 — Official Smoke / 长期运行证据 T14 — Translation Memory 后续扩展 +T13 — G-011 ResourceRepository 查询扩展 ``` T11 可以现在就开始采集，但 T01 后的数据才作为新的正式 distribution integrity evidence。 @@ -1089,8 +1098,6 @@ T10 ← T09 T12 ← 推荐 T06/T08 -T13 ← T02 - T15 ← 推荐 T03 T16 ← 推荐 T03 diff --git a/core/src/repositories/mod.rs b/core/src/repositories/mod.rs index dc6804f..2563956 100644 --- a/core/src/repositories/mod.rs +++ b/core/src/repositories/mod.rs @@ -10,6 +10,6 @@ pub mod translation_repository; pub use cas_repository::CasRepository; pub use glossary_repository::GlossaryRepository; -pub use resource_repository::ResourceRepository; +pub use resource_repository::{ResourcePathPattern, ResourceQuery, ResourceRepository}; pub use translation_memory_repository::TranslationMemoryRepository; pub use translation_repository::TranslationRepository; diff --git a/core/src/repositories/resource_repository.rs b/core/src/repositories/resource_repository.rs index e4d30c9..d5b3a68 100644 --- a/core/src/repositories/resource_repository.rs +++ b/core/src/repositories/resource_repository.rs @@ -35,6 +35,120 @@ use crate::domain::{Resource, ResourceType}; use async_trait::async_trait; +/// Compiled path pattern used by [`ResourceQuery::path_pattern`]. +/// +/// The matcher operates on Unicode scalar values rather than UTF-8 bytes: +/// +/// - `*` matches zero or more non-`/` scalar values. +/// - `**` matches zero or more scalar values, including `/`. +/// - `?` matches exactly one non-`/` scalar value. +/// - `/` and every other character are literals. +/// +/// There is no escape syntax. Matching is case-sensitive and uses a bounded +/// dynamic-programming table, so wildcard-heavy input cannot trigger +/// exponential backtracking. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResourcePathPattern { + tokens: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum ResourcePathToken { + Literal(char), + Single, + Star, + GlobStar, + RecursiveDirectory, +} + +impl ResourcePathPattern { + /// Compiles a resource path pattern. + /// + /// `**/` is treated as a recursive directory prefix so that patterns such + /// as `**/*.json` also match a root-level `a.json`. + pub fn new(pattern: &str) -> Self { + let chars = pattern.chars().collect::>(); + let mut tokens = Vec::with_capacity(chars.len()); + let mut index = 0; + + while index < chars.len() { + match chars[index] { + '*' if chars.get(index + 1) == Some(&'*') => { + index += 2; + if chars.get(index) == Some(&'/') { + tokens.push(ResourcePathToken::RecursiveDirectory); + index += 1; + } else { + tokens.push(ResourcePathToken::GlobStar); + } + } + '*' => { + tokens.push(ResourcePathToken::Star); + index += 1; + } + '?' => { + tokens.push(ResourcePathToken::Single); + index += 1; + } + literal => { + tokens.push(ResourcePathToken::Literal(literal)); + index += 1; + } + } + } + + Self { tokens } + } + + /// Returns whether `path` satisfies this pattern. + pub fn matches(&self, path: &str) -> bool { + let path = path.chars().collect::>(); + let token_count = self.tokens.len(); + let path_count = path.len(); + let mut table = vec![false; (token_count + 1) * (path_count + 1)]; + + let cell = |token: usize, value: usize| token * (path_count + 1) + value; + table[cell(token_count, path_count)] = true; + + for token in (0..token_count).rev() { + for value in (0..=path_count).rev() { + table[cell(token, value)] = match self.tokens[token] { + ResourcePathToken::Literal(literal) => { + value < path_count + && path[value] == literal + && table[cell(token + 1, value + 1)] + } + ResourcePathToken::Single => { + value < path_count + && path[value] != '/' + && table[cell(token + 1, value + 1)] + } + ResourcePathToken::Star => { + table[cell(token + 1, value)] + || (value < path_count + && path[value] != '/' + && table[cell(token, value + 1)]) + } + ResourcePathToken::GlobStar => { + table[cell(token + 1, value)] + || (value < path_count && table[cell(token, value + 1)]) + } + ResourcePathToken::RecursiveDirectory => { + if table[cell(token + 1, value)] { + true + } else { + (value..path_count) + .any(|index| path[index] == '/' && table[cell(token, index + 1)]) + } + } + }; + } + } + + table[cell(0, 0)] + } +} + /// 资源查询条件 /// /// 用于构建灵活的资源查询。支持按类型、Hash、路径、官方 release 和解析摘要过滤。 @@ -81,9 +195,12 @@ pub struct ResourceQuery { /// 如果为 `None`，不过滤路径。 /// /// 支持通配符： - /// - `*` 匹配任意字符（不包括 `/`） - /// - `**` 匹配任意字符（包括 `/`） - /// - `?` 匹配单个字符 + /// - `*` 匹配零个或多个 Unicode scalar value（不包括 `/`） + /// - `**` 匹配零个或多个 Unicode scalar value（包括 `/`） + /// - `?` 匹配恰好一个 Unicode scalar value（不包括 `/`） + /// - `/` 是字面量分隔符；`%`、`_`、`\` 没有特殊含义 + /// + /// 匹配区分大小写，不存在转义语法。空模式只匹配空路径。 /// /// # 示例 /// @@ -192,12 +309,13 @@ impl ResourceQuery { } } - /// 是否包含通用仓储需要读取完整 `Resource` 后才能判断的条件。 + /// 是否包含通用仓储需要读取完整 `Resource` 后才能最终判断的条件。 /// - /// 具体后端可以把这些 metadata 条件下推到自身索引；内存实现仍用完整 - /// `Resource` 过滤来保持 `list()` 与 `count()` 的语义一致。 + /// 具体后端可以把 metadata 条件下推到自身索引，但路径模式和这些条件都必须 + /// 经过同一最终过滤逻辑，以保持 `list()` 与 `count()` 的语义一致。 pub fn requires_resource_scan(&self) -> bool { - self.official_release_id.is_some() + self.path_pattern.is_some() + || self.official_release_id.is_some() || self.platform.is_some() || self.bundle_path.is_some() || self.archive_entry.is_some() @@ -355,7 +473,7 @@ pub trait ResourceRepository: Send + Sync { /// /// # 实现建议 /// - /// - 对于路径模式匹配，使用 `glob` crate + /// - 路径模式匹配使用 [`ResourcePathPattern`]，不要把 SQL `LIKE` 当作正式语义 /// - 考虑结果缓存以提高性能 /// - 对于超大结果集，返回迭代器（未来版本） async fn list(&self, query: ResourceQuery) -> crate::Result>; @@ -429,7 +547,8 @@ pub trait ResourceRepository: Send + Sync { /// /// # 性能 /// - /// 此方法应该比 `list()` 更快，因为不需要返回实际数据。 + /// 无路径模式或 metadata 后置过滤时可以使用索引/SQL COUNT 快速路径；存在 + /// 后置过滤时必须与 `list(query).len()` 完全一致。 /// /// # 示例 /// @@ -532,4 +651,46 @@ mod tests { assert_eq!(query2.resource_type, Some(ResourceType::Manifest)); } + + #[test] + fn resource_path_pattern_matches_unicode_and_separators() { + let cases = [ + ("", "", true), + ("", "a", false), + ("*", "", true), + ("*", "abc", true), + ("*", "a/b", false), + ("?", "你", true), + ("?", "あ", true), + ("?", "😀", true), + ("?", "/", false), + ("?", "ab", false), + ("assets/*.json", "assets/a.json", true), + ("assets/*.json", "assets/nested/a.json", false), + ("assets/**/*.json", "assets/a.json", true), + ("assets/**/*.json", "assets/nested/a.json", true), + ("assets/**/*.json", "assets/nested/deep/a.json", true), + ("**/*.json", "a.json", true), + ("**/*.json", "nested/a.json", true), + ("**/*.json", "nested/deep/a.json", true), + ("**/*.json", "a.txt", false), + ("A*", "abc", false), + ("A*", "Abc", true), + ("100%", "100%", true), + ("100%", "1000", false), + ("a_b", "a_b", true), + ("a_b", "axb", false), + (r"a\b", r"a\b", true), + (r"a\b", "a/b", false), + ("***", "nested/path", true), + ]; + + for (pattern, path, expected) in cases { + assert_eq!( + ResourcePathPattern::new(pattern).matches(path), + expected, + "pattern={pattern:?} path={path:?}" + ); + } + } } diff --git a/docs/reference/rpc-backend-api.md b/docs/reference/rpc-backend-api.md index 1c0fccc..e45d51c 100644 --- a/docs/reference/rpc-backend-api.md +++ b/docs/reference/rpc-backend-api.md @@ -119,6 +119,12 @@ SQLite `ResourceRepository`，索引不存在时返回 `ok=true` 且 属于 `parse.text_units` / `parse.errors` 的对象级查询。`limit` 范围是 `1..=1000`，非法参数返回 `BAT-ERR-700002`。 +`path_pattern` 使用 `ResourceRepository` 的统一匹配契约：`*` 匹配零个或多个 +不含 `/` 的 Unicode scalar value，`**` 可以跨 `/`，`?` 匹配恰好一个不含 `/` +的 Unicode scalar value；`**/` 可匹配根目录文件和任意深度目录。匹配区分大小写， +没有转义语法，`/`、`%`、`_` 和 `\` 都是字面量。SQLite 只做候选筛选，最终由 +同一 matcher 判定，带路径或 metadata 后置过滤时 `count()` 与 `list()` 保持一致。 + `resource.manifest` 的请求必须携带由 `release.attestation` 返回的 `release_id`、`expected_publication_identity`、`expected_manifest_identity` 和 `expected_verification_generation`。 diff --git a/infrastructure/src/resources.rs b/infrastructure/src/resources.rs index a405f38..af4430f 100644 --- a/infrastructure/src/resources.rs +++ b/infrastructure/src/resources.rs @@ -2,7 +2,9 @@ use async_trait::async_trait; use bat_core::domain::{Resource, ResourceEntry, ResourceMetadata, ResourceType}; -use bat_core::repositories::resource_repository::{ResourceQuery, ResourceRepository}; +use bat_core::repositories::resource_repository::{ + ResourcePathPattern, ResourceQuery, ResourceRepository, +}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteQueryResult}; use sqlx::{QueryBuilder, Sqlite, SqlitePool}; use std::collections::HashMap; @@ -56,9 +58,10 @@ impl ResourceRepository for InMemoryResourceRepository { async fn list(&self, query: ResourceQuery) -> bat_core::Result> { let resources = self.resources.read().await; + let path_pattern = query.path_pattern.as_deref().map(ResourcePathPattern::new); let resources = Self::sorted_resources(&resources) .into_iter() - .filter(|resource| query_matches(&query, resource)) + .filter(|resource| query_matches(&query, path_pattern.as_ref(), resource)) .collect(); Ok(resources) } @@ -359,14 +362,6 @@ impl SqliteResourceRepository { builder.push_bind(hash); } - if let Some(pattern) = &query.path_pattern { - push_condition_prefix(builder, &mut has_where); - builder.push("path LIKE "); - builder - .push_bind(glob_to_like(pattern)) - .push(" ESCAPE '\\'"); - } - if let Some(destination) = &query.destination { push_condition_prefix(builder, &mut has_where); builder.push("path = "); @@ -431,7 +426,7 @@ impl SqliteResourceRepository { ); Self::apply_filters(&mut builder, query)?; builder.push(" ORDER BY id"); - if let Some(limit) = limit { + if let Some(limit) = limit.filter(|_| !query.requires_resource_scan()) { builder.push(" LIMIT ").push_bind(limit as i64); } @@ -445,13 +440,22 @@ impl SqliteResourceRepository { .into_iter() .map(Self::resource_from_row) .collect::>>()?; - Ok(resources + let path_pattern = query.path_pattern.as_deref().map(ResourcePathPattern::new); + let mut resources = resources .into_iter() - .filter(|resource| query_matches(query, resource)) - .collect()) + .filter(|resource| query_matches(query, path_pattern.as_ref(), resource)) + .collect::>(); + if let Some(limit) = limit { + resources.truncate(limit); + } + Ok(resources) } async fn count_resources(&self, query: &ResourceQuery) -> bat_core::Result { + if query.requires_resource_scan() { + return Ok(self.fetch_resources(query, None).await?.len() as u64); + } + let mut builder = QueryBuilder::::new("SELECT COUNT(*) FROM resources"); Self::apply_filters(&mut builder, query)?; @@ -598,31 +602,11 @@ type ResourceRow = ( String, ); -fn glob_to_like(pattern: &str) -> String { - let mut escaped = String::new(); - let mut chars = pattern.chars().peekable(); - - while let Some(ch) = chars.next() { - match ch { - '*' => { - if matches!(chars.peek(), Some('*')) { - chars.next(); - } - escaped.push('%'); - } - '?' => escaped.push('_'), - '%' | '_' | '\\' => { - escaped.push('\\'); - escaped.push(ch); - } - other => escaped.push(other), - } - } - - escaped -} - -fn query_matches(query: &ResourceQuery, resource: &Resource) -> bool { +fn query_matches( + query: &ResourceQuery, + path_pattern: Option<&ResourcePathPattern>, + resource: &Resource, +) -> bool { if let Some(resource_type) = query.resource_type { if resource.entry.resource_type != resource_type { return false; @@ -635,8 +619,8 @@ fn query_matches(query: &ResourceQuery, resource: &Resource) -> bool { } } - if let Some(pattern) = &query.path_pattern { - if !wildcard_matches(pattern, &resource.entry.path) { + if let Some(pattern) = path_pattern { + if !pattern.matches(&resource.entry.path) { return false; } } @@ -701,28 +685,6 @@ fn query_matches(query: &ResourceQuery, resource: &Resource) -> bool { true } -fn wildcard_matches(pattern: &str, value: &str) -> bool { - wildcard_matches_bytes(pattern.as_bytes(), value.as_bytes()) -} - -fn wildcard_matches_bytes(pattern: &[u8], value: &[u8]) -> bool { - match (pattern.first(), value.first()) { - (None, None) => true, - (None, Some(_)) => false, - (Some(b'*'), _) => { - wildcard_matches_bytes(&pattern[1..], value) - || value - .first() - .is_some_and(|_| wildcard_matches_bytes(pattern, &value[1..])) - } - (Some(b'?'), Some(_)) => wildcard_matches_bytes(&pattern[1..], &value[1..]), - (Some(pattern_byte), Some(value_byte)) if pattern_byte == value_byte => { - wildcard_matches_bytes(&pattern[1..], &value[1..]) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; @@ -748,6 +710,90 @@ mod tests { } } + fn parity_resources() -> Vec { + let mut current = resource( + "resource/assets-a", + "assets/a.json", + "hash-assets-a", + ResourceType::TextAsset, + ); + current.metadata.official_release_id = Some("release-current".to_string()); + current.metadata.platform = Some("windows".to_string()); + + let mut unicode = resource( + "resource/assets-unicode", + "assets/你.json", + "hash-assets-unicode", + ResourceType::TextAsset, + ); + unicode.metadata.official_release_id = Some("release-current".to_string()); + unicode.metadata.platform = Some("windows".to_string()); + + let mut stale = resource( + "resource/assets-stale", + "assets/stale.json", + "hash-assets-stale", + ResourceType::TextAsset, + ); + stale.metadata.official_release_id = Some("release-old".to_string()); + stale.metadata.platform = Some("android".to_string()); + + vec![ + resource( + "resource/root-json", + "a.json", + "hash-root", + ResourceType::TextAsset, + ), + resource( + "resource/nested-json", + "nested/a.json", + "hash-nested", + ResourceType::TextAsset, + ), + current, + resource( + "resource/deep-json", + "assets/nested/deep/a.json", + "hash-assets-deep", + ResourceType::TextAsset, + ), + unicode, + resource( + "resource/assets-japanese", + "assets/あ.json", + "hash-assets-japanese", + ResourceType::TextAsset, + ), + resource( + "resource/assets-emoji", + "assets/😀.json", + "hash-assets-emoji", + ResourceType::TextAsset, + ), + stale, + resource( + "resource/assets-bundle", + "assets/a.bundle", + "hash-assets-bundle", + ResourceType::AssetBundle, + ), + resource( + "resource/special", + r"special/100%_\x", + "hash-special", + ResourceType::Media, + ), + ] + } + + fn resource_ids(resources: &[Resource]) -> Vec { + resources + .iter() + .map(|resource| resource.id.clone()) + .collect() + } + #[tokio::test] async fn add_find_update_and_delete_resource() { let repository = InMemoryResourceRepository::new(); @@ -864,6 +910,68 @@ mod tests { assert_eq!(repository.count(query).await.unwrap(), 1); } + #[tokio::test] + async fn in_memory_and_sqlite_share_path_and_count_contract() { + let memory = InMemoryResourceRepository::new(); + let (_temp_dir, sqlite) = sqlite_repository().await; + + for resource in parity_resources() { + memory.add(resource.clone()).await.unwrap(); + sqlite.add(resource).await.unwrap(); + } + + let queries = vec![ + ResourceQuery { + path_pattern: Some("**/*.json".to_string()), + ..ResourceQuery::all() + }, + ResourceQuery { + path_pattern: Some("assets/**/*.json".to_string()), + ..ResourceQuery::all() + }, + ResourceQuery { + path_pattern: Some("assets/?.json".to_string()), + ..ResourceQuery::all() + }, + ResourceQuery { + path_pattern: Some(r"special/100%_\x".to_string()), + ..ResourceQuery::all() + }, + ResourceQuery { + path_pattern: Some("does-not-exist/**".to_string()), + ..ResourceQuery::all() + }, + ResourceQuery { + resource_type: Some(ResourceType::TextAsset), + hash: Some("hash-assets-a".to_string()), + path_pattern: Some("assets/**/*.json".to_string()), + official_release_id: Some("release-current".to_string()), + platform: Some("windows".to_string()), + ..ResourceQuery::all() + }, + ]; + + for query in queries { + let memory_results = memory.list(query.clone()).await.unwrap(); + let sqlite_results = sqlite.list(query.clone()).await.unwrap(); + assert_eq!( + resource_ids(&memory_results), + resource_ids(&sqlite_results), + "list parity for query {query:?}" + ); + assert_eq!( + memory.count(query.clone()).await.unwrap(), + memory_results.len() as u64, + "in-memory count/list parity for query {query:?}" + ); + assert_eq!( + sqlite.count(query.clone()).await.unwrap(), + sqlite_results.len() as u64, + "SQLite count/list parity for query {query:?}" + ); + } + } + async fn sqlite_repository() -> (tempfile::TempDir, SqliteResourceRepository) { let temp_dir = tempfile::tempdir().unwrap(); let repository = SqliteResourceRepository::new(temp_dir.path().join("resources.sqlite"))