feat: implement production cas v1

This commit is contained in:
2026-06-28 12:39:17 +08:00
parent dd53e3054e
commit b202d7b231
33 changed files with 1082 additions and 447 deletions
+2 -1
View File
@@ -7,11 +7,12 @@ license.workspace = true
[dependencies]
bat-core = { path = "../core" }
bat-cas-engine = { path = "../crates/bat-cas-engine" }
anyhow.workspace = true
thiserror.workspace = true
serde.workspace = true
blake3.workspace = true
tokio.workspace = true
async-trait.workspace = true
[dev-dependencies]
tokio = { workspace = true, features = ["test-util", "macros"] }
+5 -1
View File
@@ -1 +1,5 @@
//! CAS 存储实现占位
//! CAS 仓储适配器。
pub mod filesystem;
pub use filesystem::FileSystemCasRepository;
+130 -131
View File
@@ -1,152 +1,134 @@
//! CAS 文件系统实现
//! 文件系统 CAS 仓储适配层。
//!
//! 基于文件系统的内容寻址存储实现
//! 这里不实现 CAS 核心算法,只将 `bat-cas-engine` 适配到
//! `bat-core::repositories::CasRepository`。
use async_trait::async_trait;
use bat_cas_engine::hash::Hash;
use bat_cas_engine::repository as engine_repository;
use bat_core::repositories::cas_repository::{CasRepository, ObjectId};
use blake3::Hasher;
use std::path::PathBuf;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use tokio::sync::OnceCell;
/// 文件系统 CAS Repository
///
/// 使用文件系统存储对象,目录结构:
/// ```text
/// cas_root/
/// ├── objects/
/// │ ├── ab/
/// │ │ ├── cd/
/// │ │ │ └── abcd1234...
/// └── refs.db (SQLite)
/// ```
/// 文件系统 CAS Repository 适配器。
pub struct FileSystemCasRepository {
/// CAS 根目录
root: PathBuf,
inner: OnceCell<engine_repository::FileSystemCasRepository>,
}
impl FileSystemCasRepository {
/// 创建新的文件系统 CAS Repository
///
/// # 参数
///
/// - `root`: CAS 根目录
///
/// # 示例
///
/// ```rust,ignore
/// let repo = FileSystemCasRepository::new("/path/to/cas");
/// ```
/// 创建新的文件系统 CAS Repository 适配器。
pub fn new(root: impl Into<PathBuf>) -> Self {
Self {
root: root.into(),
inner: OnceCell::new(),
}
}
/// 计算对象 Hash
fn compute_hash(data: &[u8]) -> ObjectId {
let mut hasher = Hasher::new();
hasher.update(data);
hasher.finalize().to_hex().to_string()
}
/// 获取对象路径
fn object_path(&self, id: &ObjectId) -> PathBuf {
let prefix1 = &id[0..2];
let prefix2 = &id[2..4];
self.root
.join("objects")
.join(prefix1)
.join(prefix2)
.join(id)
}
/// 初始化存储
/// 初始化底层 CAS 引擎。
pub async fn init(&self) -> bat_core::Result<()> {
fs::create_dir_all(self.root.join("objects"))
self.engine().await.map(|_| ())
}
async fn engine(&self) -> bat_core::Result<&engine_repository::FileSystemCasRepository> {
self.inner
.get_or_try_init(|| async {
engine_repository::FileSystemCasRepository::new(&self.root)
.await
.map_err(Self::map_error)
})
.await
.map_err(bat_core::Error::Io)?;
Ok(())
}
fn parse_object_id(id: &ObjectId) -> bat_core::Result<Hash> {
id.parse::<Hash>()
.map_err(|error| bat_core::Error::InvalidArgument(error.to_string()))
}
fn map_error(error: bat_cas_engine::CasError) -> bat_core::Error {
match error {
bat_cas_engine::CasError::Io(error) => bat_core::Error::Io(error),
bat_cas_engine::CasError::ObjectNotFound(id) => bat_core::Error::NotFound(id),
bat_cas_engine::CasError::HashMismatch { expected, actual } => {
bat_core::Error::InvalidArgument(format!(
"Hash mismatch: expected {}, got {}",
expected, actual
))
}
bat_cas_engine::CasError::InvalidHash(hash) => {
bat_core::Error::InvalidArgument(format!("Invalid hash: {}", hash))
}
bat_cas_engine::CasError::ReferenceUnderflow(hash) => bat_core::Error::InvalidArgument(
format!("Reference count is already zero: {}", hash),
),
bat_cas_engine::CasError::Database(message) => {
bat_core::Error::Other(anyhow::anyhow!("CAS metadata error: {}", message))
}
bat_cas_engine::CasError::Other(error) => bat_core::Error::Other(error),
}
}
}
#[async_trait]
impl CasRepository for FileSystemCasRepository {
async fn store(&self, data: &[u8]) -> bat_core::Result<ObjectId> {
let hash = Self::compute_hash(data);
let path = self.object_path(&hash);
// 如果对象已存在,直接返回
if path.exists() {
return Ok(hash);
}
// 创建父目录
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.await
.map_err(bat_core::Error::Io)?;
}
// 写入对象
let mut file = fs::File::create(&path)
let hash = self
.engine()
.await?
.store(data)
.await
.map_err(bat_core::Error::Io)?;
file.write_all(data)
.await
.map_err(bat_core::Error::Io)?;
file.sync_all()
.await
.map_err(bat_core::Error::Io)?;
Ok(hash)
.map_err(Self::map_error)?;
Ok(hash.to_string())
}
async fn get(&self, id: &ObjectId) -> bat_core::Result<Vec<u8>> {
let path = self.object_path(id);
if !path.exists() {
return Err(bat_core::Error::NotFound(format!("Object not found: {}", id)));
}
let data = fs::read(&path)
let hash = Self::parse_object_id(id)?;
self.engine()
.await?
.get(&hash)
.await
.map_err(bat_core::Error::Io)?;
// 验证 Hash
let computed_hash = Self::compute_hash(&data);
if &computed_hash != id {
return Err(bat_core::Error::InvalidArgument(format!(
"Hash mismatch: expected {}, got {}",
id, computed_hash
)));
}
Ok(data)
.map_err(Self::map_error)
}
async fn exists(&self, id: &ObjectId) -> bool {
self.object_path(id).exists()
let Ok(hash) = Self::parse_object_id(id) else {
return false;
};
let Ok(engine) = self.engine().await else {
return false;
};
engine.exists(&hash).await
}
async fn add_reference(&self, _id: &ObjectId) -> bat_core::Result<u64> {
// TODO: Phase 1 Week 3 - 实现引用计数
Ok(1)
async fn add_reference(&self, id: &ObjectId) -> bat_core::Result<u64> {
let hash = Self::parse_object_id(id)?;
self.engine()
.await?
.add_reference(&hash)
.await
.map_err(Self::map_error)
}
async fn remove_reference(&self, _id: &ObjectId) -> bat_core::Result<u64> {
// TODO: Phase 1 Week 3 - 实现引用计数
Ok(0)
async fn remove_reference(&self, id: &ObjectId) -> bat_core::Result<u64> {
let hash = Self::parse_object_id(id)?;
self.engine()
.await?
.remove_reference(&hash)
.await
.map_err(Self::map_error)
}
async fn get_reference_count(&self, _id: &ObjectId) -> bat_core::Result<u64> {
// TODO: Phase 1 Week 3 - 实现引用计数
Ok(1)
async fn get_reference_count(&self, id: &ObjectId) -> bat_core::Result<u64> {
let hash = Self::parse_object_id(id)?;
self.engine()
.await?
.get_reference_count(&hash)
.await
.map_err(Self::map_error)
}
async fn gc(&self) -> bat_core::Result<u64> {
// TODO: Phase 1 Week 3 - 实现垃圾回收
Ok(0)
self.engine().await?.gc().await.map_err(Self::map_error)
}
}
@@ -156,7 +138,7 @@ mod tests {
use tempfile::TempDir;
#[tokio::test]
async fn test_store_and_get() {
async fn store_get_and_reference_count() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
repo.init().await.unwrap();
@@ -166,39 +148,56 @@ mod tests {
let retrieved = repo.get(&id).await.unwrap();
assert_eq!(data, retrieved.as_slice());
assert_eq!(repo.get_reference_count(&id).await.unwrap(), 1);
}
#[tokio::test]
async fn test_exists() {
async fn duplicate_store_increments_reference_count() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
repo.init().await.unwrap();
let data = b"Test data";
let id = repo.store(data).await.unwrap();
assert!(repo.exists(&id).await);
assert!(!repo.exists(&"nonexistent".to_string()).await);
}
#[tokio::test]
async fn test_deduplication() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
repo.init().await.unwrap();
let data = b"Duplicate data";
let id1 = repo.store(data).await.unwrap();
let id2 = repo.store(data).await.unwrap();
let id1 = repo.store(b"Duplicate data").await.unwrap();
let id2 = repo.store(b"Duplicate data").await.unwrap();
assert_eq!(id1, id2);
assert_eq!(repo.get_reference_count(&id1).await.unwrap(), 2);
}
#[test]
fn test_compute_hash() {
let data = b"test";
let hash = FileSystemCasRepository::compute_hash(data);
assert!(!hash.is_empty());
assert_eq!(hash.len(), 64); // BLAKE3 produces 32 bytes = 64 hex chars
#[tokio::test]
async fn gc_removes_zero_reference_objects() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
repo.init().await.unwrap();
let id = repo.store(b"collect me").await.unwrap();
assert_eq!(repo.remove_reference(&id).await.unwrap(), 0);
assert_eq!(repo.gc().await.unwrap(), 1);
assert!(!repo.exists(&id).await);
assert!(matches!(
repo.get(&id).await,
Err(bat_core::Error::NotFound(_))
));
}
#[tokio::test]
async fn invalid_object_id_is_rejected() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
repo.init().await.unwrap();
let error = repo.get(&"not-a-hash".to_string()).await.unwrap_err();
assert!(matches!(error, bat_core::Error::InvalidArgument(_)));
}
#[tokio::test]
async fn store_returns_blake3_hex_object_id() {
let temp_dir = TempDir::new().unwrap();
let repo = FileSystemCasRepository::new(temp_dir.path());
let hash = repo.store(b"test").await.unwrap();
assert_eq!(hash.len(), 64);
}
}
+2
View File
@@ -12,5 +12,7 @@
pub mod cas;
pub use cas::FileSystemCasRepository;
/// Infrastructure 版本号
pub const VERSION: &str = env!("CARGO_PKG_VERSION");