feat(sync): 接入解析缓存与汉化发布前置

补齐官方 release 解析缓存、TextUnit 明细索引、资源变更集、Crowdin handoff 预留、ResourceRepository 导入元数据和 localized release patch 前置链路。

同时开放文件级 patch.apply 与 UnityFS TextAsset/string/semantic field patch CLI/RPC 入口,并保留官方原版资源与汉化产物双目录发布状态。

验证:cargo test -p bat-assetbundle --locked;cargo clippy -p bat-assetbundle --all-targets --locked -- -D warnings;cargo test -p bat-infrastructure --locked。
This commit is contained in:
2026-07-31 00:38:45 +08:00
parent f4880a71bd
commit 2079c6a307
25 changed files with 17287 additions and 225 deletions
+585 -63
View File
@@ -10,9 +10,11 @@ use crate::path_security::{
ensure_path_within_root, ensure_safe_file_target, read_file_no_symlink, write_file_atomic,
STATE_FILE_MODE,
};
use bat_assetbundle::{Parser, UnityFsParser};
use bat_assetbundle::{
Parser, TextUnit, TextUnitExtractionError, TextUnitExtractor, UnityFsParser,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -20,7 +22,11 @@ use std::time::{SystemTime, UNIX_EPOCH};
/// Parse-cache file name stored under a published official resource root.
pub const OFFICIAL_PARSE_CACHE_FILE: &str = "official-parse-cache.json";
/// Current parse-cache schema version.
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 1;
pub const OFFICIAL_PARSE_CACHE_VERSION: u32 = 2;
/// TextUnit detail index file name stored under a published official resource root.
pub const OFFICIAL_TEXTUNIT_INDEX_FILE: &str = "official-textunit-index.json";
/// Current TextUnit detail-index schema version.
pub const OFFICIAL_TEXTUNIT_INDEX_VERSION: u32 = 1;
/// Configuration for one official resource parse-cache refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -55,6 +61,10 @@ pub struct OfficialParseReport {
pub cache_path: PathBuf,
/// Aggregate parse-cache summary.
pub summary: OfficialParseSummary,
/// TextUnit detail-index path written by the refresh.
pub textunit_index_path: PathBuf,
/// Aggregate TextUnit detail-index summary.
pub textunit_index_summary: OfficialTextUnitIndexSummary,
}
/// Aggregate counters for a parse-cache refresh.
@@ -78,6 +88,15 @@ pub struct OfficialParseSummary {
pub failed_count: usize,
/// Total TextAsset objects found in parsed Unity serialized files.
pub text_asset_count: usize,
/// Total TextUnit objects extracted from TextAsset payloads and TypeTree string fields.
#[serde(default)]
pub text_unit_count: usize,
/// Number of binary/invalid TextAsset payloads skipped by the TextUnit extractor.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Number of non-fatal TypeTree field extraction diagnostics.
#[serde(default)]
pub text_unit_error_count: usize,
}
/// Persistent parse cache for one official resource root.
@@ -127,10 +146,162 @@ pub struct OfficialParseCacheEntry {
pub text_assets: Vec<String>,
/// Non-fatal serialized-file parse diagnostic count.
pub serialized_parse_error_count: usize,
/// Number of translation-ready TextUnit entries extracted from this bundle.
#[serde(default)]
pub text_unit_count: usize,
/// Stable set of TextUnit payload formats such as json/csv/tsv/plain.
#[serde(default)]
pub text_unit_formats: Vec<String>,
/// Number of binary/invalid TextAsset payloads skipped by TextUnit extraction.
#[serde(default)]
pub skipped_binary_text_asset_count: usize,
/// Non-fatal TypeTree field extraction diagnostic count.
#[serde(default)]
pub text_unit_error_count: usize,
/// Human-readable error or skip reason.
pub error: Option<String>,
}
/// Persistent TextUnit detail index for one official resource root.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndex {
/// Index schema version.
#[serde(default = "default_textunit_index_version")]
pub version: u32,
/// Index generation time as Unix seconds.
pub generated_unix_seconds: u64,
/// Official resource root used to generate this index.
pub resource_root: PathBuf,
/// Aggregate counters for this index.
pub summary: OfficialTextUnitIndexSummary,
/// Extracted TextUnit details in deterministic order.
#[serde(default)]
pub units: Vec<OfficialTextUnitIndexUnit>,
/// Parse and extraction diagnostics in deterministic order.
#[serde(default)]
pub errors: Vec<OfficialTextUnitIndexError>,
}
/// Aggregate counters for a TextUnit detail index.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexSummary {
/// Number of TextUnit detail entries.
pub unit_count: usize,
/// Number of parse/extraction diagnostics.
pub error_count: usize,
/// Number of binary or invalid TextAsset payloads skipped by extraction.
pub skipped_binary_text_asset_count: usize,
}
/// One TextUnit detail entry.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexUnit {
/// Stable unit ID within the official release.
pub id: String,
/// Parse-cache entry key that produced this unit.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Unity editor version when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unity_version: Option<String>,
/// Original source text.
pub source_text: String,
/// Unity serialized file path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_offset: Option<usize>,
/// Number of bytes consumed by this field, including alignment padding.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_byte_size: Option<usize>,
/// TextUnit payload format such as json/csv/tsv/plain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
/// Extraction source kind such as TextAsset or TypeTreeField.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_source_kind: Option<String>,
/// TextAsset name when the unit came from a TextAsset payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset_name: Option<String>,
/// Stable extraction context copied from the parser.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub context: BTreeMap<String, String>,
}
/// One parse or extraction diagnostic in the TextUnit detail index.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OfficialTextUnitIndexError {
/// Stable diagnostic ID within the official release.
pub id: String,
/// Parse-cache entry key associated with this diagnostic.
pub parse_entry_key: String,
/// Official URL from the download manifest.
pub source_url: String,
/// Relative destination path under the official resource root.
pub destination: String,
/// Inner archive path when the source is a zip file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub archive_entry: Option<String>,
/// Source classification used by the parser.
pub source_kind: OfficialParseSourceKind,
/// Parse status associated with this diagnostic.
pub status: OfficialParseStatus,
/// Unity serialized file path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serialized_file: Option<String>,
/// Unity object path ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path_id: Option<i64>,
/// Unity class ID when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub class_id: Option<i32>,
/// TypeTree field path when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field_path: Option<String>,
/// Byte offset relative to the beginning of the Unity object payload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
/// Human-readable diagnostic.
pub error: String,
}
/// Query filters for TextUnit detail entries.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OfficialTextUnitQuery {
/// Filter by destination path.
pub destination: Option<String>,
/// Filter by destination glob pattern.
pub path_pattern: Option<String>,
/// Filter by archive entry.
pub archive_entry: Option<String>,
/// Filter by Unity object path ID.
pub path_id: Option<i64>,
/// Filter by Unity class ID.
pub class_id: Option<i32>,
/// Filter by field path.
pub field_path: Option<String>,
/// Filter by TextUnit format.
pub format: Option<String>,
}
/// Source kind for a parse-cache entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -187,21 +358,36 @@ impl OfficialParseCacheService {
)
})?;
let previous_cache = read_parse_cache_at(&config.resource_root)?;
let previous_textunit_index = read_textunit_index_at(&config.resource_root)?;
let mut summary = OfficialParseSummary {
manifest_entry_count: manifest.entries.len(),
..OfficialParseSummary::default()
};
let mut entries = BTreeMap::new();
let mut units = Vec::new();
let mut errors = Vec::new();
for manifest_entry in manifest.entries.values() {
let produced = process_manifest_entry(config, manifest_entry, previous_cache.as_ref());
for entry in produced {
summary.record_entry(&entry);
entries.insert(entry.key.clone(), entry);
let produced = process_manifest_entry(
config,
manifest_entry,
previous_cache.as_ref(),
previous_textunit_index.as_ref(),
);
for produced in produced {
summary.record_entry(&produced.entry);
units.extend(produced.units);
errors.extend(produced.errors);
entries.insert(produced.entry.key.clone(), produced.entry);
}
}
summary.cache_entry_count = entries.len();
let textunit_index_summary = OfficialTextUnitIndexSummary {
unit_count: units.len(),
error_count: errors.len(),
skipped_binary_text_asset_count: summary.skipped_binary_text_asset_count,
};
let cache = OfficialParseCache {
version: OFFICIAL_PARSE_CACHE_VERSION,
generated_unix_seconds: unix_seconds_now(),
@@ -209,15 +395,67 @@ impl OfficialParseCacheService {
entries,
};
write_parse_cache_at(&config.resource_root, &cache)?;
let textunit_index = OfficialTextUnitIndex {
version: OFFICIAL_TEXTUNIT_INDEX_VERSION,
generated_unix_seconds: cache.generated_unix_seconds,
resource_root: config.resource_root.clone(),
summary: textunit_index_summary.clone(),
units,
errors,
};
write_textunit_index_at(&config.resource_root, &textunit_index)?;
Ok(OfficialParseReport {
resource_root: config.resource_root.clone(),
cache_path: config.cache_path(),
summary,
textunit_index_path: config.resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE),
textunit_index_summary,
})
}
}
struct ParseProduced {
entry: OfficialParseCacheEntry,
units: Vec<OfficialTextUnitIndexUnit>,
errors: Vec<OfficialTextUnitIndexError>,
}
impl ParseProduced {
fn from_entry(entry: OfficialParseCacheEntry) -> Self {
let errors = error_from_status_entry(&entry).into_iter().collect();
Self {
entry,
units: Vec::new(),
errors,
}
}
fn from_cached(
mut entry: OfficialParseCacheEntry,
previous_index: Option<&OfficialTextUnitIndex>,
) -> Self {
entry.reused_from_previous_cache = true;
let units = previous_index
.into_iter()
.flat_map(|index| index.units.iter())
.filter(|unit| unit.parse_entry_key == entry.key)
.cloned()
.collect();
let errors = previous_index
.into_iter()
.flat_map(|index| index.errors.iter())
.filter(|error| error.parse_entry_key == entry.key)
.cloned()
.collect();
Self {
entry,
units,
errors,
}
}
}
impl OfficialParseSummary {
fn record_entry(&mut self, entry: &OfficialParseCacheEntry) {
match entry.source_kind {
@@ -234,6 +472,9 @@ impl OfficialParseSummary {
OfficialParseStatus::Parsed => {
self.parsed_bundle_count += 1;
self.text_asset_count += entry.text_asset_count;
self.text_unit_count += entry.text_unit_count;
self.skipped_binary_text_asset_count += entry.skipped_binary_text_asset_count;
self.text_unit_error_count += entry.text_unit_error_count;
}
OfficialParseStatus::SkippedUnsupported => {
self.unsupported_count += 1;
@@ -279,21 +520,81 @@ pub fn write_parse_cache_at(
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方解析缓存")
}
/// Reads the TextUnit detail index under a published official resource root.
pub fn read_textunit_index_at(
resource_root: &Path,
) -> Result<Option<OfficialTextUnitIndex>, String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
let Some(bytes) = read_file_no_symlink(&path, "官方 TextUnit 明细索引")? else {
return Ok(None);
};
let Ok(index) = serde_json::from_slice::<OfficialTextUnitIndex>(&bytes) else {
return Ok(None);
};
if index.version != OFFICIAL_TEXTUNIT_INDEX_VERSION {
return Ok(None);
}
Ok(Some(index))
}
/// Writes the TextUnit detail index under a published official resource root.
pub fn write_textunit_index_at(
resource_root: &Path,
index: &OfficialTextUnitIndex,
) -> Result<(), String> {
let path = resource_root.join(OFFICIAL_TEXTUNIT_INDEX_FILE);
ensure_path_within_root(resource_root, &path)?;
ensure_safe_file_target(resource_root, &path, "官方 TextUnit 明细索引")?;
let bytes = serde_json::to_vec_pretty(index)
.map_err(|error| format!("序列化官方 TextUnit 明细索引失败:{error}"))?;
write_file_atomic(&path, &bytes, STATE_FILE_MODE, "官方 TextUnit 明细索引")
}
/// Returns TextUnit detail entries matching a query.
pub fn query_textunit_index_units<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexUnit> {
index
.units
.iter()
.filter(|unit| textunit_unit_matches(unit, query))
.collect()
}
/// Returns parse/extraction diagnostics matching a query.
pub fn query_textunit_index_errors<'a>(
index: &'a OfficialTextUnitIndex,
query: &OfficialTextUnitQuery,
) -> Vec<&'a OfficialTextUnitIndexError> {
index
.errors
.iter()
.filter(|error| textunit_error_matches(error, query))
.collect()
}
fn process_manifest_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
) -> Vec<OfficialParseCacheEntry> {
previous_index: Option<&OfficialTextUnitIndex>,
) -> Vec<ParseProduced> {
let fingerprint = fingerprint_for(manifest_entry);
if looks_like_zip_source(manifest_entry) {
return process_zip_entry(config, manifest_entry, previous_cache, fingerprint);
return process_zip_entry(
config,
manifest_entry,
previous_cache,
previous_index,
fingerprint,
);
}
if looks_like_direct_bundle_source(manifest_entry) {
let key = direct_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true;
return vec![cached];
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
return vec![parse_direct_bundle(
config,
@@ -304,61 +605,58 @@ fn process_manifest_entry(
}
let key = unsupported_key(&manifest_entry.url);
if let Some(mut cached) = reusable_entry(previous_cache, &key, &fingerprint) {
cached.reused_from_previous_cache = true;
return vec![cached];
if let Some(cached) = reusable_entry(previous_cache, &key, &fingerprint) {
return vec![ParseProduced::from_cached(cached, previous_index)];
}
vec![unsupported_entry(
vec![ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
key,
"非 UnityFS 候选资源",
)]
))]
}
fn process_zip_entry(
config: &OfficialParseConfig,
manifest_entry: &OfficialDownloadManifestEntry,
previous_cache: Option<&OfficialParseCache>,
previous_index: Option<&OfficialTextUnitIndex>,
fingerprint: OfficialParseSourceFingerprint,
) -> Vec<OfficialParseCacheEntry> {
) -> Vec<ParseProduced> {
let cached_entries = reusable_archive_entries(previous_cache, manifest_entry, &fingerprint);
if !cached_entries.is_empty() {
return cached_entries
.into_iter()
.map(|mut entry| {
entry.reused_from_previous_cache = true;
entry
})
.map(|entry| ParseProduced::from_cached(entry, previous_index))
.collect();
}
let archive_path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return vec![failed_entry(
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
)]
))]
}
};
let archive_entries = match list_zip_entries(&config.unzip_command, &archive_path) {
Ok(entries) => entries,
Err(error) => {
return vec![failed_entry(
return vec![ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::ZipEntry,
fingerprint,
zip_list_key(&manifest_entry.url),
error,
)]
))]
}
};
@@ -369,14 +667,14 @@ fn process_zip_entry(
{
Ok(bytes) => bytes,
Err(error) => {
produced.push(failed_entry(
produced.push(ParseProduced::from_entry(failed_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint.clone(),
key,
error,
));
)));
continue;
}
};
@@ -390,14 +688,14 @@ fn process_zip_entry(
}
if produced.is_empty() {
produced.push(unsupported_entry(
produced.push(ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::Unsupported,
fingerprint,
zip_list_key(&manifest_entry.url),
"ZIP 内没有可检查文件条目",
));
)));
}
produced
}
@@ -407,42 +705,42 @@ fn parse_direct_bundle(
manifest_entry: &OfficialDownloadManifestEntry,
key: String,
fingerprint: OfficialParseSourceFingerprint,
) -> OfficialParseCacheEntry {
) -> ParseProduced {
let path = match resource_path_for(&config.resource_root, manifest_entry) {
Ok(path) => path,
Err(error) => {
return failed_entry(
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
)
))
}
};
let bytes = match read_resource_file(&path) {
Ok(bytes) => bytes,
Err(error) => {
return failed_entry(
return ParseProduced::from_entry(failed_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
error,
)
))
}
};
if !UnityFsParser::has_unityfs_signature(&bytes) {
return unsupported_entry(
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
None,
OfficialParseSourceKind::DirectBundle,
fingerprint,
key,
"文件不是 UnityFS bundle",
);
));
}
parsed_bundle_entry(
manifest_entry,
@@ -460,16 +758,16 @@ fn parse_zip_inner_file(
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> OfficialParseCacheEntry {
) -> ParseProduced {
if !UnityFsParser::has_unityfs_signature(bytes) {
return unsupported_entry(
return ParseProduced::from_entry(unsupported_entry(
manifest_entry,
Some(archive_entry),
OfficialParseSourceKind::ZipEntry,
fingerprint,
key,
"ZIP 条目不是 UnityFS bundle",
);
));
}
parsed_bundle_entry(
manifest_entry,
@@ -488,38 +786,254 @@ fn parsed_bundle_entry(
fingerprint: OfficialParseSourceFingerprint,
key: String,
bytes: &[u8],
) -> OfficialParseCacheEntry {
) -> ParseProduced {
let parser = UnityFsParser::new();
match parser.parse(bytes) {
Ok(parsed) => OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some(parsed.unity_version),
file_count: parsed.files.len(),
serialized_file_count: parsed.serialized_files.len(),
text_asset_count: parsed.text_assets.len(),
text_assets: parsed
.text_assets
Ok(parsed) => {
let text_units = TextUnitExtractor::new().extract_bundle_with_context(
&parsed,
Some(&manifest_entry.destination),
archive_entry.as_deref(),
);
let text_unit_formats = text_units
.units
.iter()
.map(|asset| asset.name.clone())
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
error: None,
},
Err(error) => failed_entry(
.filter_map(|unit| unit.context.get("format").cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let entry = OfficialParseCacheEntry {
key,
source_url: manifest_entry.url.clone(),
destination: manifest_entry.destination.clone(),
archive_entry,
source_kind,
fingerprint,
status: OfficialParseStatus::Parsed,
reused_from_previous_cache: false,
unity_version: Some(parsed.unity_version),
file_count: parsed.files.len(),
serialized_file_count: parsed.serialized_files.len(),
text_asset_count: parsed.text_assets.len(),
text_assets: parsed
.text_assets
.iter()
.map(|asset| asset.name.clone())
.collect(),
serialized_parse_error_count: parsed.serialized_parse_errors.len(),
text_unit_count: text_units.units.len(),
text_unit_formats,
skipped_binary_text_asset_count: text_units.skipped_binary_text_assets,
text_unit_error_count: text_units.errors.len(),
error: None,
};
let units = indexed_text_units_for_entry(&entry, &text_units.units);
let errors = indexed_extraction_errors_for_entry(&entry, &text_units.errors);
ParseProduced {
entry,
units,
errors,
}
}
Err(error) => ParseProduced::from_entry(failed_entry(
manifest_entry,
archive_entry,
source_kind,
fingerprint,
key,
error.to_string(),
),
)),
}
}
fn indexed_text_units_for_entry(
entry: &OfficialParseCacheEntry,
units: &[TextUnit],
) -> Vec<OfficialTextUnitIndexUnit> {
units
.iter()
.enumerate()
.map(|(index, unit)| OfficialTextUnitIndexUnit {
id: format!("{}#unit:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
unity_version: entry.unity_version.clone(),
source_text: unit.source_text.clone(),
serialized_file: unit.serialized_file.clone(),
path_id: unit.path_id,
class_id: unit.class_id,
field_path: unit.field_path.clone(),
field_offset: unit.field_offset,
field_byte_size: unit.field_byte_size,
format: unit.context.get("format").cloned(),
text_source_kind: unit.context.get("source_kind").cloned(),
asset_name: unit.context.get("asset_name").cloned(),
context: unit.context.clone(),
})
.collect()
}
fn indexed_extraction_errors_for_entry(
entry: &OfficialParseCacheEntry,
errors: &[TextUnitExtractionError],
) -> Vec<OfficialTextUnitIndexError> {
errors
.iter()
.enumerate()
.map(|(index, error)| OfficialTextUnitIndexError {
id: format!("{}#extract-error:{index}", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: error.serialized_file.clone(),
path_id: error.path_id,
class_id: error.class_id,
field_path: error.field_path.clone(),
offset: error.offset,
error: error.error.clone(),
})
.collect()
}
fn error_from_status_entry(entry: &OfficialParseCacheEntry) -> Option<OfficialTextUnitIndexError> {
let error = entry.error.as_ref()?;
Some(OfficialTextUnitIndexError {
id: format!("{}#status", entry.key),
parse_entry_key: entry.key.clone(),
source_url: entry.source_url.clone(),
destination: entry.destination.clone(),
archive_entry: entry.archive_entry.clone(),
source_kind: entry.source_kind,
status: entry.status,
serialized_file: None,
path_id: None,
class_id: None,
field_path: None,
offset: None,
error: error.clone(),
})
}
fn textunit_unit_matches(unit: &OfficialTextUnitIndexUnit, query: &OfficialTextUnitQuery) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &unit.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &unit.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| unit.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| unit.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| unit.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| unit.field_path.as_ref() != Some(field_path))
{
return false;
}
if query
.format
.as_ref()
.is_some_and(|format| unit.format.as_ref() != Some(format))
{
return false;
}
true
}
fn textunit_error_matches(
error: &OfficialTextUnitIndexError,
query: &OfficialTextUnitQuery,
) -> bool {
if query
.destination
.as_ref()
.is_some_and(|destination| &error.destination != destination)
{
return false;
}
if query
.path_pattern
.as_ref()
.is_some_and(|pattern| !glob_matches(pattern, &error.destination))
{
return false;
}
if query
.archive_entry
.as_ref()
.is_some_and(|archive_entry| error.archive_entry.as_ref() != Some(archive_entry))
{
return false;
}
if query
.path_id
.is_some_and(|path_id| error.path_id != Some(path_id))
{
return false;
}
if query
.class_id
.is_some_and(|class_id| error.class_id != Some(class_id))
{
return false;
}
if query
.field_path
.as_ref()
.is_some_and(|field_path| error.field_path.as_ref() != Some(field_path))
{
return false;
}
true
}
fn glob_matches(pattern: &str, value: &str) -> bool {
glob_matches_bytes(pattern.as_bytes(), value.as_bytes())
}
fn glob_matches_bytes(pattern: &[u8], value: &[u8]) -> bool {
match pattern.split_first() {
None => value.is_empty(),
Some((&b'*', rest)) => {
glob_matches_bytes(rest, value)
|| (!value.is_empty() && glob_matches_bytes(pattern, &value[1..]))
}
Some((&b'?', rest)) => !value.is_empty() && glob_matches_bytes(rest, &value[1..]),
Some((&literal, rest)) => value
.split_first()
.is_some_and(|(&head, tail)| head == literal && glob_matches_bytes(rest, tail)),
}
}
@@ -585,6 +1099,10 @@ fn status_entry(
text_asset_count: 0,
text_assets: Vec::new(),
serialized_parse_error_count: 0,
text_unit_count: 0,
text_unit_formats: Vec::new(),
skipped_binary_text_asset_count: 0,
text_unit_error_count: 0,
error: Some(reason),
}
}
@@ -735,6 +1253,10 @@ fn default_parse_cache_version() -> u32 {
OFFICIAL_PARSE_CACHE_VERSION
}
fn default_textunit_index_version() -> u32 {
OFFICIAL_TEXTUNIT_INDEX_VERSION
}
fn unix_seconds_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)