mirror of
https://github.com/Yuyi-Oak/BlueArchiveToolkit.git
synced 2026-07-22 04:06:44 +08:00
feat: implement production cas v1
This commit is contained in:
@@ -22,6 +22,14 @@ pub enum CasError {
|
||||
#[error("Object not found: {0}")]
|
||||
ObjectNotFound(String),
|
||||
|
||||
/// Hash 格式无效
|
||||
#[error("Invalid hash: {0}")]
|
||||
InvalidHash(String),
|
||||
|
||||
/// 引用计数下溢
|
||||
#[error("Reference count is already zero: {0}")]
|
||||
ReferenceUnderflow(String),
|
||||
|
||||
/// 数据库错误
|
||||
#[error("Database error: {0}")]
|
||||
Database(String),
|
||||
@@ -33,3 +41,9 @@ pub enum CasError {
|
||||
|
||||
/// CAS Engine Result 类型
|
||||
pub type Result<T> = std::result::Result<T, CasError>;
|
||||
|
||||
impl From<sqlx::Error> for CasError {
|
||||
fn from(error: sqlx::Error) -> Self {
|
||||
Self::Database(error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,14 +29,11 @@ impl Hash {
|
||||
|
||||
/// 从十六进制字符串解析
|
||||
pub fn from_hex(s: &str) -> Result<Self, crate::error::CasError> {
|
||||
let bytes = hex::decode(s).map_err(|_| {
|
||||
crate::error::CasError::Other(anyhow::anyhow!("Invalid hex string"))
|
||||
})?;
|
||||
let bytes =
|
||||
hex::decode(s).map_err(|_| crate::error::CasError::InvalidHash(s.to_string()))?;
|
||||
|
||||
if bytes.len() != 32 {
|
||||
return Err(crate::error::CasError::Other(anyhow::anyhow!(
|
||||
"Hash must be 32 bytes"
|
||||
)));
|
||||
return Err(crate::error::CasError::InvalidHash(s.to_string()));
|
||||
}
|
||||
|
||||
let mut hash_bytes = [0u8; 32];
|
||||
|
||||
@@ -13,9 +13,10 @@
|
||||
#![warn(clippy::all)]
|
||||
|
||||
pub mod error;
|
||||
pub mod storage;
|
||||
pub mod hash;
|
||||
pub mod refcount;
|
||||
pub mod repository;
|
||||
pub mod storage;
|
||||
|
||||
pub use error::{CasError, Result};
|
||||
|
||||
|
||||
@@ -1,4 +1,276 @@
|
||||
//! 引用计数模块占位
|
||||
//! CAS 引用计数和对象元数据。
|
||||
|
||||
/// 引用计数管理(待实现)
|
||||
pub struct RefCounter;
|
||||
use crate::error::{CasError, Result};
|
||||
use crate::hash::Hash;
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
|
||||
use sqlx::SqlitePool;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// CAS 对象元数据。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObjectMetadata {
|
||||
/// 对象 Hash。
|
||||
pub hash: Hash,
|
||||
/// 对象大小,单位字节。
|
||||
pub size: u64,
|
||||
/// 当前引用计数。
|
||||
pub ref_count: u64,
|
||||
/// 创建时间,Unix 秒。
|
||||
pub created_at: i64,
|
||||
/// 更新时间,Unix 秒。
|
||||
pub updated_at: i64,
|
||||
/// 引用计数首次归零时间,Unix 秒。
|
||||
pub zero_ref_at: Option<i64>,
|
||||
}
|
||||
|
||||
/// SQLite 引用计数存储。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SqliteRefCounter {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteRefCounter {
|
||||
/// 打开或创建 SQLite 元数据库。
|
||||
pub async fn new(path: impl AsRef<Path>) -> Result<Self> {
|
||||
if let Some(parent) = path.as_ref().parent() {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
let options =
|
||||
SqliteConnectOptions::from_str(&format!("sqlite://{}", path.as_ref().display()))
|
||||
.map_err(|error| CasError::Database(error.to_string()))?
|
||||
.create_if_missing(true);
|
||||
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await?;
|
||||
|
||||
let counter = Self { pool };
|
||||
counter.init_schema().await?;
|
||||
Ok(counter)
|
||||
}
|
||||
|
||||
async fn init_schema(&self) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS cas_objects (
|
||||
hash TEXT PRIMARY KEY NOT NULL,
|
||||
size INTEGER NOT NULL CHECK(size >= 0),
|
||||
ref_count INTEGER NOT NULL CHECK(ref_count >= 0),
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
zero_ref_at INTEGER
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE INDEX IF NOT EXISTS idx_cas_objects_ref_count
|
||||
ON cas_objects(ref_count)
|
||||
"#,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn now() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs() as i64)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// 注册对象元数据。如果对象已存在,仅更新大小和更新时间。
|
||||
pub async fn ensure_object(&self, hash: &Hash, size: u64) -> Result<()> {
|
||||
let now = Self::now();
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO cas_objects(hash, size, ref_count, created_at, updated_at, zero_ref_at)
|
||||
VALUES(?1, ?2, 0, ?3, ?3, ?3)
|
||||
ON CONFLICT(hash) DO UPDATE SET
|
||||
size = excluded.size,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(hash.to_string())
|
||||
.bind(size as i64)
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 增加对象引用计数。
|
||||
pub async fn add_reference(&self, hash: &Hash) -> Result<u64> {
|
||||
let now = Self::now();
|
||||
let updated: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
UPDATE cas_objects
|
||||
SET ref_count = ref_count + 1,
|
||||
updated_at = ?1,
|
||||
zero_ref_at = NULL
|
||||
WHERE hash = ?2
|
||||
RETURNING ref_count
|
||||
"#,
|
||||
)
|
||||
.bind(now)
|
||||
.bind(hash.to_string())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
updated
|
||||
.map(|count| count as u64)
|
||||
.ok_or_else(|| CasError::ObjectNotFound(hash.to_string()))
|
||||
}
|
||||
|
||||
/// 减少对象引用计数。
|
||||
pub async fn remove_reference(&self, hash: &Hash) -> Result<u64> {
|
||||
let now = Self::now();
|
||||
let updated: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
UPDATE cas_objects
|
||||
SET ref_count = ref_count - 1,
|
||||
updated_at = ?1,
|
||||
zero_ref_at = CASE WHEN ref_count = 1 THEN ?1 ELSE zero_ref_at END
|
||||
WHERE hash = ?2 AND ref_count > 0
|
||||
RETURNING ref_count
|
||||
"#,
|
||||
)
|
||||
.bind(now)
|
||||
.bind(hash.to_string())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
if let Some(count) = updated {
|
||||
return Ok(count as u64);
|
||||
}
|
||||
|
||||
if self.metadata(hash).await?.is_some() {
|
||||
Err(CasError::ReferenceUnderflow(hash.to_string()))
|
||||
} else {
|
||||
Err(CasError::ObjectNotFound(hash.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取对象引用计数。
|
||||
pub async fn get_reference_count(&self, hash: &Hash) -> Result<u64> {
|
||||
let count: Option<i64> =
|
||||
sqlx::query_scalar("SELECT ref_count FROM cas_objects WHERE hash = ?1")
|
||||
.bind(hash.to_string())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
count
|
||||
.map(|count| count as u64)
|
||||
.ok_or_else(|| CasError::ObjectNotFound(hash.to_string()))
|
||||
}
|
||||
|
||||
/// 获取对象元数据。
|
||||
pub async fn metadata(&self, hash: &Hash) -> Result<Option<ObjectMetadata>> {
|
||||
let row: Option<(String, i64, i64, i64, i64, Option<i64>)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT hash, size, ref_count, created_at, updated_at, zero_ref_at
|
||||
FROM cas_objects
|
||||
WHERE hash = ?1
|
||||
"#,
|
||||
)
|
||||
.bind(hash.to_string())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
row.map(
|
||||
|(hash, size, ref_count, created_at, updated_at, zero_ref_at)| {
|
||||
Ok(ObjectMetadata {
|
||||
hash: hash.parse()?,
|
||||
size: size as u64,
|
||||
ref_count: ref_count as u64,
|
||||
created_at,
|
||||
updated_at,
|
||||
zero_ref_at,
|
||||
})
|
||||
},
|
||||
)
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// 列出引用计数为 0 的对象。
|
||||
pub async fn zero_ref_objects(&self) -> Result<Vec<Hash>> {
|
||||
let hashes: Vec<String> =
|
||||
sqlx::query_scalar("SELECT hash FROM cas_objects WHERE ref_count = 0")
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
hashes
|
||||
.into_iter()
|
||||
.map(|hash| hash.parse())
|
||||
.collect::<Result<Vec<_>>>()
|
||||
}
|
||||
|
||||
/// 删除对象元数据。
|
||||
pub async fn delete_metadata(&self, hash: &Hash) -> Result<bool> {
|
||||
let result = sqlx::query("DELETE FROM cas_objects WHERE hash = ?1")
|
||||
.bind(hash.to_string())
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::hash::compute_hash;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reference_count_lifecycle() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let counter = SqliteRefCounter::new(temp_dir.path().join("metadata.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let hash = compute_hash(b"tracked object");
|
||||
|
||||
counter.ensure_object(&hash, 14).await.unwrap();
|
||||
assert_eq!(counter.add_reference(&hash).await.unwrap(), 1);
|
||||
assert_eq!(counter.add_reference(&hash).await.unwrap(), 2);
|
||||
assert_eq!(counter.remove_reference(&hash).await.unwrap(), 1);
|
||||
assert_eq!(counter.remove_reference(&hash).await.unwrap(), 0);
|
||||
assert_eq!(counter.get_reference_count(&hash).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_reference_objects_are_listed() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let counter = SqliteRefCounter::new(temp_dir.path().join("metadata.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let hash = compute_hash(b"candidate");
|
||||
|
||||
counter.ensure_object(&hash, 9).await.unwrap();
|
||||
counter.add_reference(&hash).await.unwrap();
|
||||
counter.remove_reference(&hash).await.unwrap();
|
||||
|
||||
assert_eq!(counter.zero_ref_objects().await.unwrap(), vec![hash]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_zero_reference_returns_underflow() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let counter = SqliteRefCounter::new(temp_dir.path().join("metadata.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let hash = compute_hash(b"underflow");
|
||||
|
||||
counter.ensure_object(&hash, 9).await.unwrap();
|
||||
|
||||
let error = counter.remove_reference(&hash).await.unwrap_err();
|
||||
assert!(matches!(error, CasError::ReferenceUnderflow(_)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
//! CAS V1 repository,组合对象存储与引用计数元数据。
|
||||
|
||||
use crate::error::{CasError, Result};
|
||||
use crate::hash::{compute_hash, Hash};
|
||||
use crate::refcount::SqliteRefCounter;
|
||||
use crate::storage::{FileSystemStorage, Storage, StorageStats};
|
||||
use std::path::Path;
|
||||
|
||||
/// 文件系统 CAS repository。
|
||||
///
|
||||
/// 该类型是 CAS V1 的核心实现。对象文件由 `FileSystemStorage` 管理,
|
||||
/// 引用计数和对象元数据由 `SqliteRefCounter` 管理。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileSystemCasRepository {
|
||||
storage: FileSystemStorage,
|
||||
ref_counter: SqliteRefCounter,
|
||||
}
|
||||
|
||||
impl FileSystemCasRepository {
|
||||
/// 打开或初始化 CAS repository。
|
||||
pub async fn new(root: impl AsRef<Path>) -> Result<Self> {
|
||||
let root = root.as_ref();
|
||||
let storage = FileSystemStorage::new(root).await?;
|
||||
let ref_counter = SqliteRefCounter::new(root.join("metadata.sqlite")).await?;
|
||||
|
||||
Ok(Self {
|
||||
storage,
|
||||
ref_counter,
|
||||
})
|
||||
}
|
||||
|
||||
/// 返回底层文件存储。
|
||||
pub fn storage(&self) -> &FileSystemStorage {
|
||||
&self.storage
|
||||
}
|
||||
|
||||
/// 存储对象并增加引用计数。
|
||||
pub async fn store(&self, data: &[u8]) -> Result<Hash> {
|
||||
let hash = compute_hash(data);
|
||||
let existed = self.storage.exists(&hash).await;
|
||||
let stored_hash = self.storage.put(data).await?;
|
||||
|
||||
if let Err(error) = self
|
||||
.ref_counter
|
||||
.ensure_object(&stored_hash, data.len() as u64)
|
||||
.await
|
||||
{
|
||||
if !existed {
|
||||
let _ = self.storage.delete(&stored_hash).await;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
match self.ref_counter.add_reference(&stored_hash).await {
|
||||
Ok(count) => {
|
||||
debug_assert!(count > 0);
|
||||
Ok(stored_hash)
|
||||
}
|
||||
Err(error) => {
|
||||
if !existed {
|
||||
let _ = self.storage.delete(&stored_hash).await;
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取对象并验证 Hash。
|
||||
pub async fn get(&self, hash: &Hash) -> Result<Vec<u8>> {
|
||||
self.storage.get(hash).await
|
||||
}
|
||||
|
||||
/// 检查对象是否存在。
|
||||
pub async fn exists(&self, hash: &Hash) -> bool {
|
||||
self.storage.exists(hash).await
|
||||
}
|
||||
|
||||
/// 增加引用计数。
|
||||
pub async fn add_reference(&self, hash: &Hash) -> Result<u64> {
|
||||
if !self.storage.exists(hash).await {
|
||||
return Err(CasError::ObjectNotFound(hash.to_string()));
|
||||
}
|
||||
|
||||
let size = self.storage.size(hash).await?;
|
||||
self.ref_counter.ensure_object(hash, size).await?;
|
||||
self.ref_counter.add_reference(hash).await
|
||||
}
|
||||
|
||||
/// 减少引用计数。
|
||||
pub async fn remove_reference(&self, hash: &Hash) -> Result<u64> {
|
||||
self.ref_counter.remove_reference(hash).await
|
||||
}
|
||||
|
||||
/// 获取引用计数。
|
||||
pub async fn get_reference_count(&self, hash: &Hash) -> Result<u64> {
|
||||
self.ref_counter.get_reference_count(hash).await
|
||||
}
|
||||
|
||||
/// 返回当前 GC 候选对象。
|
||||
pub async fn gc_candidates(&self) -> Result<Vec<Hash>> {
|
||||
self.ref_counter.zero_ref_objects().await
|
||||
}
|
||||
|
||||
/// 删除引用计数为 0 的对象。
|
||||
pub async fn gc(&self) -> Result<u64> {
|
||||
let candidates = self.gc_candidates().await?;
|
||||
let mut deleted = 0u64;
|
||||
|
||||
for hash in candidates {
|
||||
let ref_count = self.ref_counter.get_reference_count(&hash).await?;
|
||||
if ref_count != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
match self.storage.delete(&hash).await {
|
||||
Ok(()) => {
|
||||
deleted += 1;
|
||||
}
|
||||
Err(CasError::ObjectNotFound(_)) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
|
||||
self.ref_counter.delete_metadata(&hash).await?;
|
||||
}
|
||||
|
||||
Ok(deleted)
|
||||
}
|
||||
|
||||
/// 获取存储统计信息。
|
||||
pub async fn stats(&self) -> Result<StorageStats> {
|
||||
self.storage.stats().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_increments_reference_count() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = FileSystemCasRepository::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let hash = repo.store(b"same data").await.unwrap();
|
||||
let second = repo.store(b"same data").await.unwrap();
|
||||
|
||||
assert_eq!(hash, second);
|
||||
assert_eq!(repo.get_reference_count(&hash).await.unwrap(), 2);
|
||||
assert_eq!(repo.stats().await.unwrap().object_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gc_deletes_only_zero_reference_objects() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = FileSystemCasRepository::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let keep = repo.store(b"keep").await.unwrap();
|
||||
let delete = repo.store(b"delete").await.unwrap();
|
||||
|
||||
repo.remove_reference(&delete).await.unwrap();
|
||||
|
||||
assert_eq!(repo.gc().await.unwrap(), 1);
|
||||
assert!(repo.exists(&keep).await);
|
||||
assert!(!repo.exists(&delete).await);
|
||||
assert!(matches!(
|
||||
repo.get_reference_count(&delete).await,
|
||||
Err(CasError::ObjectNotFound(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_store_is_safe_and_counts_references() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = Arc::new(FileSystemCasRepository::new(temp_dir.path()).await.unwrap());
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for _ in 0..32 {
|
||||
let repo = Arc::clone(&repo);
|
||||
tasks.push(tokio::spawn(async move { repo.store(b"shared").await }));
|
||||
}
|
||||
|
||||
let mut hashes = Vec::new();
|
||||
for task in tasks {
|
||||
hashes.push(task.await.unwrap().unwrap());
|
||||
}
|
||||
|
||||
let first = hashes[0];
|
||||
assert!(hashes.iter().all(|hash| *hash == first));
|
||||
assert_eq!(repo.get_reference_count(&first).await.unwrap(), 32);
|
||||
assert_eq!(repo.stats().await.unwrap().object_count, 1);
|
||||
assert_eq!(repo.get(&first).await.unwrap(), b"shared");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupted_object_is_detected_through_repository() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = FileSystemCasRepository::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let hash = repo.store(b"valid").await.unwrap();
|
||||
tokio::fs::write(repo.storage().object_path(&hash), b"invalid")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
repo.get(&hash).await,
|
||||
Err(CasError::HashMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_reference_requires_existing_object() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = FileSystemCasRepository::new(temp_dir.path()).await.unwrap();
|
||||
let missing = compute_hash(b"missing");
|
||||
|
||||
assert!(matches!(
|
||||
repo.add_reference(&missing).await,
|
||||
Err(CasError::ObjectNotFound(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remove_reference_does_not_go_below_zero() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let repo = FileSystemCasRepository::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let hash = repo.store(b"underflow").await.unwrap();
|
||||
assert_eq!(repo.remove_reference(&hash).await.unwrap(), 0);
|
||||
|
||||
assert!(matches!(
|
||||
repo.remove_reference(&hash).await,
|
||||
Err(CasError::ReferenceUnderflow(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -1,177 +1,177 @@
|
||||
//! CAS 存储接口定义
|
||||
//! CAS 对象存储接口与文件系统实现。
|
||||
//!
|
||||
//! 提供统一的 Content Addressable Storage 抽象接口
|
||||
//! 本模块只负责不可变对象文件的写入、读取和完整性校验。引用计数和
|
||||
//! GC 元数据由 `refcount` 与 `repository` 模块负责。
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::hash::Hash;
|
||||
use crate::error::{CasError, Result};
|
||||
use crate::hash::{compute_hash, Hash};
|
||||
use async_trait::async_trait;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
/// CAS 存储接口
|
||||
///
|
||||
/// 所有 CAS 存储实现必须实现此 trait
|
||||
/// CAS 对象存储接口。
|
||||
#[async_trait]
|
||||
pub trait Storage: Send + Sync {
|
||||
/// 存储对象
|
||||
///
|
||||
/// # 参数
|
||||
/// - `data`: 要存储的数据
|
||||
///
|
||||
/// # 返回
|
||||
/// - 成功:返回对象的 Hash
|
||||
/// - 失败:返回错误
|
||||
/// 存储对象并返回内容 Hash。
|
||||
async fn put(&self, data: &[u8]) -> Result<Hash>;
|
||||
|
||||
/// 获取对象
|
||||
///
|
||||
/// # 参数
|
||||
/// - `hash`: 对象的 Hash
|
||||
///
|
||||
/// # 返回
|
||||
/// - 成功:返回对象数据
|
||||
/// - 失败:如果对象不存在或读取失败,返回错误
|
||||
/// 读取对象并校验内容 Hash。
|
||||
async fn get(&self, hash: &Hash) -> Result<Vec<u8>>;
|
||||
|
||||
/// 检查对象是否存在
|
||||
///
|
||||
/// # 参数
|
||||
/// - `hash`: 对象的 Hash
|
||||
///
|
||||
/// # 返回
|
||||
/// - 存在返回 true,否则返回 false
|
||||
/// 检查对象文件是否存在。
|
||||
async fn exists(&self, hash: &Hash) -> bool;
|
||||
|
||||
/// 删除对象
|
||||
///
|
||||
/// # 参数
|
||||
/// - `hash`: 对象的 Hash
|
||||
///
|
||||
/// # 返回
|
||||
/// - 成功:返回 Ok(())
|
||||
/// - 失败:返回错误
|
||||
/// 删除对象文件。
|
||||
async fn delete(&self, hash: &Hash) -> Result<()>;
|
||||
|
||||
/// 获取对象大小
|
||||
///
|
||||
/// # 参数
|
||||
/// - `hash`: 对象的 Hash
|
||||
///
|
||||
/// # 返回
|
||||
/// - 成功:返回对象大小(字节)
|
||||
/// - 失败:返回错误
|
||||
/// 获取对象大小。
|
||||
async fn size(&self, hash: &Hash) -> Result<u64>;
|
||||
|
||||
/// 列出所有对象
|
||||
///
|
||||
/// # 返回
|
||||
/// - 所有对象的 Hash 列表
|
||||
/// 列出所有可解析的对象 Hash。
|
||||
async fn list(&self) -> Result<Vec<Hash>>;
|
||||
|
||||
/// 获取存储统计信息
|
||||
///
|
||||
/// # 返回
|
||||
/// - 存储统计信息
|
||||
/// 获取存储统计信息。
|
||||
async fn stats(&self) -> Result<StorageStats>;
|
||||
}
|
||||
|
||||
/// 存储统计信息
|
||||
/// 存储统计信息。
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct StorageStats {
|
||||
/// 对象总数
|
||||
/// 对象总数。
|
||||
pub object_count: u64,
|
||||
|
||||
/// 总存储大小(字节)
|
||||
/// 总存储大小,单位字节。
|
||||
pub total_size: u64,
|
||||
|
||||
/// 存储路径
|
||||
/// 存储根路径。
|
||||
pub storage_path: String,
|
||||
}
|
||||
|
||||
/// 文件系统 CAS 存储实现
|
||||
/// 文件系统 CAS 对象存储。
|
||||
///
|
||||
/// 对象路径使用两级分片:
|
||||
///
|
||||
/// ```text
|
||||
/// objects/ab/cd/abcdef...
|
||||
/// ```
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileSystemStorage {
|
||||
/// 存储根目录
|
||||
root: std::path::PathBuf,
|
||||
root: PathBuf,
|
||||
objects_dir: PathBuf,
|
||||
tmp_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl FileSystemStorage {
|
||||
/// 创建新的文件系统存储
|
||||
///
|
||||
/// # 参数
|
||||
/// - `root`: 存储根目录
|
||||
///
|
||||
/// # 返回
|
||||
/// - 成功:返回存储实例
|
||||
/// - 失败:返回错误
|
||||
/// 创建文件系统存储并初始化目录。
|
||||
pub async fn new(root: impl AsRef<Path>) -> Result<Self> {
|
||||
let root = root.as_ref().to_path_buf();
|
||||
|
||||
// 创建根目录
|
||||
tokio::fs::create_dir_all(&root).await.map_err(|e| {
|
||||
crate::error::CasError::Io(e)
|
||||
})?;
|
||||
|
||||
// 创建 objects 目录
|
||||
let objects_dir = root.join("objects");
|
||||
tokio::fs::create_dir_all(&objects_dir).await.map_err(|e| {
|
||||
crate::error::CasError::Io(e)
|
||||
})?;
|
||||
let tmp_dir = root.join("tmp");
|
||||
|
||||
Ok(Self { root })
|
||||
tokio::fs::create_dir_all(&objects_dir).await?;
|
||||
tokio::fs::create_dir_all(&tmp_dir).await?;
|
||||
|
||||
Ok(Self {
|
||||
root,
|
||||
objects_dir,
|
||||
tmp_dir,
|
||||
})
|
||||
}
|
||||
|
||||
/// 获取对象文件路径
|
||||
///
|
||||
/// 使用 Hash 的前两个字符作为子目录,避免单目录文件过多
|
||||
///
|
||||
/// 例如:hash = "abcdef1234..." -> objects/ab/cdef1234...
|
||||
fn object_path(&self, hash: &Hash) -> std::path::PathBuf {
|
||||
/// 返回存储根路径。
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
/// 返回对象文件路径。
|
||||
pub fn object_path(&self, hash: &Hash) -> PathBuf {
|
||||
let hash_str = hash.to_string();
|
||||
let (prefix, suffix) = hash_str.split_at(2);
|
||||
self.root.join("objects").join(prefix).join(suffix)
|
||||
let prefix1 = &hash_str[0..2];
|
||||
let prefix2 = &hash_str[2..4];
|
||||
self.objects_dir.join(prefix1).join(prefix2).join(hash_str)
|
||||
}
|
||||
|
||||
fn temp_path(&self, hash: &Hash) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_nanos())
|
||||
.unwrap_or_default();
|
||||
self.tmp_dir
|
||||
.join(format!("{}.{}.{}.tmp", hash, std::process::id(), nonce))
|
||||
}
|
||||
|
||||
fn sync_directory(path: &Path) -> Result<()> {
|
||||
let directory = std::fs::File::open(path)?;
|
||||
directory.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_temp_file(&self, path: &Path, data: &[u8]) -> Result<()> {
|
||||
let mut file = tokio::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path)
|
||||
.await?;
|
||||
file.write_all(data).await?;
|
||||
file.sync_all().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Storage for FileSystemStorage {
|
||||
async fn put(&self, data: &[u8]) -> Result<Hash> {
|
||||
// 计算 Hash
|
||||
let hash = crate::hash::compute_hash(data);
|
||||
|
||||
// 检查是否已存在
|
||||
if self.exists(&hash).await {
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
// 获取文件路径
|
||||
let hash = compute_hash(data);
|
||||
let path = self.object_path(&hash);
|
||||
|
||||
// 创建父目录
|
||||
if let Some(parent) = path.parent() {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
if self.exists(&hash).await {
|
||||
match self.get(&hash).await {
|
||||
Ok(_) => return Ok(hash),
|
||||
Err(CasError::HashMismatch { .. }) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
// 写入文件
|
||||
tokio::fs::write(&path, data).await?;
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| CasError::Other(anyhow::anyhow!("Object path has no parent")))?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
tokio::fs::create_dir_all(&self.tmp_dir).await?;
|
||||
|
||||
Ok(hash)
|
||||
let tmp_path = self.temp_path(&hash);
|
||||
self.write_temp_file(&tmp_path, data).await?;
|
||||
|
||||
match tokio::fs::rename(&tmp_path, &path).await {
|
||||
Ok(()) => {
|
||||
Self::sync_directory(parent)?;
|
||||
Ok(hash)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = tokio::fs::remove_file(&tmp_path).await;
|
||||
self.get(&hash).await?;
|
||||
Ok(hash)
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = tokio::fs::remove_file(&tmp_path).await;
|
||||
Err(CasError::Io(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get(&self, hash: &Hash) -> Result<Vec<u8>> {
|
||||
let path = self.object_path(hash);
|
||||
|
||||
// 读取文件
|
||||
let data = tokio::fs::read(&path).await.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::error::CasError::ObjectNotFound(hash.to_string())
|
||||
let data = tokio::fs::read(&path).await.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
CasError::ObjectNotFound(hash.to_string())
|
||||
} else {
|
||||
crate::error::CasError::Io(e)
|
||||
CasError::Io(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
// 验证 Hash
|
||||
let computed_hash = crate::hash::compute_hash(&data);
|
||||
let computed_hash = compute_hash(&data);
|
||||
if computed_hash != *hash {
|
||||
return Err(crate::error::CasError::HashMismatch {
|
||||
return Err(CasError::HashMismatch {
|
||||
expected: hash.to_string(),
|
||||
actual: computed_hash.to_string(),
|
||||
});
|
||||
@@ -181,52 +181,57 @@ impl Storage for FileSystemStorage {
|
||||
}
|
||||
|
||||
async fn exists(&self, hash: &Hash) -> bool {
|
||||
let path = self.object_path(hash);
|
||||
tokio::fs::try_exists(&path).await.unwrap_or(false)
|
||||
tokio::fs::try_exists(self.object_path(hash))
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn delete(&self, hash: &Hash) -> Result<()> {
|
||||
let path = self.object_path(hash);
|
||||
tokio::fs::remove_file(&path).await.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::error::CasError::ObjectNotFound(hash.to_string())
|
||||
tokio::fs::remove_file(&path).await.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
CasError::ObjectNotFound(hash.to_string())
|
||||
} else {
|
||||
crate::error::CasError::Io(e)
|
||||
CasError::Io(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn size(&self, hash: &Hash) -> Result<u64> {
|
||||
let path = self.object_path(hash);
|
||||
let metadata = tokio::fs::metadata(&path).await.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::error::CasError::ObjectNotFound(hash.to_string())
|
||||
} else {
|
||||
crate::error::CasError::Io(e)
|
||||
}
|
||||
})?;
|
||||
let metadata = tokio::fs::metadata(self.object_path(hash))
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
CasError::ObjectNotFound(hash.to_string())
|
||||
} else {
|
||||
CasError::Io(error)
|
||||
}
|
||||
})?;
|
||||
Ok(metadata.len())
|
||||
}
|
||||
|
||||
async fn list(&self) -> Result<Vec<Hash>> {
|
||||
let objects_dir = self.root.join("objects");
|
||||
let mut hashes = Vec::new();
|
||||
let mut first_level = tokio::fs::read_dir(&self.objects_dir).await?;
|
||||
|
||||
// 遍历所有子目录
|
||||
let mut read_dir = tokio::fs::read_dir(&objects_dir).await?;
|
||||
while let Some(entry) = read_dir.next_entry().await? {
|
||||
let prefix_path = entry.path();
|
||||
if !prefix_path.is_dir() {
|
||||
while let Some(first_entry) = first_level.next_entry().await? {
|
||||
if !first_entry.file_type().await?.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 遍历子目录中的文件
|
||||
let mut sub_read_dir = tokio::fs::read_dir(&prefix_path).await?;
|
||||
while let Some(file_entry) = sub_read_dir.next_entry().await? {
|
||||
if let Some(file_name) = file_entry.file_name().to_str() {
|
||||
if let Some(prefix) = prefix_path.file_name().and_then(|p| p.to_str()) {
|
||||
let hash_str = format!("{}{}", prefix, file_name);
|
||||
if let Ok(hash) = hash_str.parse() {
|
||||
let mut second_level = tokio::fs::read_dir(first_entry.path()).await?;
|
||||
while let Some(second_entry) = second_level.next_entry().await? {
|
||||
if !second_entry.file_type().await?.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut files = tokio::fs::read_dir(second_entry.path()).await?;
|
||||
while let Some(file_entry) = files.next_entry().await? {
|
||||
if !file_entry.file_type().await?.is_file() {
|
||||
continue;
|
||||
}
|
||||
if let Some(file_name) = file_entry.file_name().to_str() {
|
||||
if let Ok(hash) = file_name.parse() {
|
||||
hashes.push(hash);
|
||||
}
|
||||
}
|
||||
@@ -234,6 +239,7 @@ impl Storage for FileSystemStorage {
|
||||
}
|
||||
}
|
||||
|
||||
hashes.sort_by_key(|hash: &Hash| hash.to_string());
|
||||
Ok(hashes)
|
||||
}
|
||||
|
||||
@@ -242,9 +248,7 @@ impl Storage for FileSystemStorage {
|
||||
let mut total_size = 0u64;
|
||||
|
||||
for hash in &hashes {
|
||||
if let Ok(size) = self.size(hash).await {
|
||||
total_size += size;
|
||||
}
|
||||
total_size += self.size(hash).await?;
|
||||
}
|
||||
|
||||
Ok(StorageStats {
|
||||
@@ -260,7 +264,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_and_get() {
|
||||
async fn put_and_get_roundtrip() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let storage = FileSystemStorage::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
@@ -269,10 +273,11 @@ mod tests {
|
||||
|
||||
let retrieved = storage.get(&hash).await.unwrap();
|
||||
assert_eq!(data, retrieved.as_slice());
|
||||
assert!(storage.object_path(&hash).exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deduplication() {
|
||||
async fn put_is_deduplicated() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let storage = FileSystemStorage::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
@@ -281,19 +286,62 @@ mod tests {
|
||||
let hash2 = storage.put(data).await.unwrap();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
let stats = storage.stats().await.unwrap();
|
||||
assert_eq!(stats.object_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_exists() {
|
||||
async fn exists_and_delete() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let storage = FileSystemStorage::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let data = b"Test data";
|
||||
let hash = storage.put(data).await.unwrap();
|
||||
let hash = storage.put(b"Test data").await.unwrap();
|
||||
|
||||
assert!(storage.exists(&hash).await);
|
||||
|
||||
storage.delete(&hash).await.unwrap();
|
||||
assert!(!storage.exists(&hash).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupted_object_is_rejected() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let storage = FileSystemStorage::new(temp_dir.path()).await.unwrap();
|
||||
|
||||
let hash = storage.put(b"valid data").await.unwrap();
|
||||
tokio::fs::write(storage.object_path(&hash), b"corrupted")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = storage.get(&hash).await.unwrap_err();
|
||||
assert!(matches!(error, CasError::HashMismatch { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_reports_path_errors() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("not-a-directory");
|
||||
tokio::fs::write(&file_path, b"file").await.unwrap();
|
||||
|
||||
let result = FileSystemStorage::new(file_path.join("cas")).await;
|
||||
assert!(matches!(result, Err(CasError::Io(_))));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn put_reports_permission_errors_when_enforced_by_platform() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let storage = FileSystemStorage::new(temp_dir.path()).await.unwrap();
|
||||
let objects_dir = temp_dir.path().join("objects");
|
||||
let original_permissions = std::fs::metadata(&objects_dir).unwrap().permissions();
|
||||
|
||||
std::fs::set_permissions(&objects_dir, std::fs::Permissions::from_mode(0o500)).unwrap();
|
||||
let result = storage.put(b"permission test").await;
|
||||
std::fs::set_permissions(&objects_dir, original_permissions).unwrap();
|
||||
|
||||
if let Err(CasError::Io(error)) = result {
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user