NotAShelf/pinakes
1
0
Fork 0
Code Issues 6 Pull requests Projects Releases Packages Wiki Activity Actions

pinakes-core: update remaining modules and tests

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I9e0ff5ea33a5cf697473423e88f167ce6a6a6964
This commit is contained in:
raf 2026-03-08 00:42:29 +03:00
commit 3d9f8933d2
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
44 changed files with 1207 additions and 578 deletions
Download patch file Download diff file Expand all files Collapse all files

90
crates/pinakes-core/src/managed_storage.rs
View file

@ -28,7 +28,8 @@ pub struct ManagedStorageService {
impl ManagedStorageService {
/// Create a new managed storage service.
pub fn new(
#[must_use]
pub const fn new(
root_dir: PathBuf,
max_upload_size: u64,
verify_on_read: bool,
@ -41,6 +42,10 @@ impl ManagedStorageService {
}
/// Initialize the storage directory structure.
///
/// # Errors
///
/// Returns [`PinakesError`] if the directory cannot be created.
pub async fn init(&self) -> Result<()> {
fs::create_dir_all(&self.root_dir).await?;
info!(path = %self.root_dir.display(), "initialized managed storage");
@ -50,6 +55,7 @@ impl ManagedStorageService {
/// Get the storage path for a content hash.
///
/// Layout: `<root>/<hash[0:2]>/<hash[2:4]>/<full_hash>`
#[must_use]
pub fn path(&self, hash: &ContentHash) -> PathBuf {
let h = &hash.0;
if h.len() >= 4 {
@ -61,7 +67,8 @@ impl ManagedStorageService {
}
/// Check if a blob exists in storage.
pub async fn exists(&self, hash: &ContentHash) -> bool {
#[must_use]
pub fn exists(&self, hash: &ContentHash) -> bool {
self.path(hash).exists()
}
@ -70,6 +77,11 @@ impl ManagedStorageService {
/// Returns the content hash and file size.
/// If the file already exists with the same hash, returns early
/// (deduplication).
///
/// # Errors
///
/// Returns [`PinakesError`] if the file cannot be stored or exceeds the size
/// limit.
pub async fn store_stream<R: AsyncRead + Unpin>(
&self,
mut reader: R,
@ -119,14 +131,13 @@ impl ManagedStorageService {
debug!(hash = %hash, "blob already exists, deduplicating");
let _ = fs::remove_file(&temp_path).await;
return Ok((hash, total_size));
} else {
warn!(
hash = %hash,
expected = total_size,
actual = existing_meta.len(),
"size mismatch for existing blob, replacing"
);
}
warn!(
hash = %hash,
expected = total_size,
actual = existing_meta.len(),
"size mismatch for existing blob, replacing"
);
}
// Move temp file to final location
@ -140,6 +151,10 @@ impl ManagedStorageService {
}
/// Store a file from a path.
///
/// # Errors
///
/// Returns [`PinakesError`] if the file cannot be opened or stored.
pub async fn store_file(&self, path: &Path) -> Result<(ContentHash, u64)> {
let file = fs::File::open(path).await?;
let reader = BufReader::new(file);
@ -147,6 +162,11 @@ impl ManagedStorageService {
}
/// Store bytes directly.
///
/// # Errors
///
/// Returns [`PinakesError`] if the data cannot be stored or exceeds the size
/// limit.
pub async fn store_bytes(&self, data: &[u8]) -> Result<(ContentHash, u64)> {
use std::io::Cursor;
let cursor = Cursor::new(data);
@ -154,6 +174,10 @@ impl ManagedStorageService {
}
/// Open a blob for reading.
///
/// # Errors
///
/// Returns [`PinakesError`] if the blob does not exist or cannot be opened.
pub async fn open(&self, hash: &ContentHash) -> Result<fs::File> {
let path = self.path(hash);
if !path.exists() {
@ -168,6 +192,11 @@ impl ManagedStorageService {
}
/// Read a blob entirely into memory.
///
/// # Errors
///
/// Returns [`PinakesError`] if the blob does not exist, cannot be read, or
/// fails integrity check.
pub async fn read(&self, hash: &ContentHash) -> Result<Vec<u8>> {
let path = self.path(hash);
if !path.exists() {
@ -180,8 +209,7 @@ impl ManagedStorageService {
let computed = blake3::hash(&data);
if computed.to_hex().to_string() != hash.0 {
return Err(PinakesError::StorageIntegrity(format!(
"hash mismatch for blob {}",
hash
"hash mismatch for blob {hash}"
)));
}
}
@ -190,6 +218,11 @@ impl ManagedStorageService {
}
/// Verify the integrity of a stored blob.
///
/// # Errors
///
/// Returns [`PinakesError`] if the blob cannot be read or has a hash
/// mismatch.
pub async fn verify(&self, hash: &ContentHash) -> Result<bool> {
let path = self.path(hash);
if !path.exists() {
@ -217,8 +250,7 @@ impl ManagedStorageService {
"blob integrity check failed"
);
return Err(PinakesError::StorageIntegrity(format!(
"hash mismatch: expected {}, computed {}",
hash, computed
"hash mismatch: expected {hash}, computed {computed}"
)));
}
@ -227,6 +259,10 @@ impl ManagedStorageService {
}
/// Delete a blob from storage.
///
/// # Errors
///
/// Returns [`PinakesError`] if the blob cannot be removed.
pub async fn delete(&self, hash: &ContentHash) -> Result<()> {
let path = self.path(hash);
if path.exists() {
@ -245,6 +281,11 @@ impl ManagedStorageService {
}
/// Get the size of a stored blob.
///
/// # Errors
///
/// Returns [`PinakesError`] if the blob does not exist or metadata cannot be
/// read.
pub async fn size(&self, hash: &ContentHash) -> Result<u64> {
let path = self.path(hash);
if !path.exists() {
@ -255,18 +296,23 @@ impl ManagedStorageService {
}
/// List all blob hashes in storage.
///
/// # Errors
///
/// Returns [`PinakesError`] if the storage directory cannot be read.
pub async fn list_all(&self) -> Result<Vec<ContentHash>> {
let mut hashes = Vec::new();
let mut entries = fs::read_dir(&self.root_dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.is_dir() && path.file_name().map(|n| n.len()) == Some(2) {
if path.is_dir() && path.file_name().map(std::ffi::OsStr::len) == Some(2)
{
let mut sub_entries = fs::read_dir(&path).await?;
while let Some(sub_entry) = sub_entries.next_entry().await? {
let sub_path = sub_entry.path();
if sub_path.is_dir()
&& sub_path.file_name().map(|n| n.len()) == Some(2)
&& sub_path.file_name().map(std::ffi::OsStr::len) == Some(2)
{
let mut file_entries = fs::read_dir(&sub_path).await?;
while let Some(file_entry) = file_entries.next_entry().await? {
@ -287,6 +333,10 @@ impl ManagedStorageService {
}
/// Calculate total storage used by all blobs.
///
/// # Errors
///
/// Returns [`StorageError`] if listing blobs or querying sizes fails.
pub async fn total_size(&self) -> Result<u64> {
let hashes = self.list_all().await?;
let mut total = 0u64;
@ -299,6 +349,10 @@ impl ManagedStorageService {
}
/// Clean up any orphaned temp files.
///
/// # Errors
///
/// Returns [`PinakesError`] if the temp directory cannot be read.
pub async fn cleanup_temp(&self) -> Result<u64> {
let temp_dir = self.root_dir.join("temp");
if !temp_dir.exists() {
@ -349,7 +403,7 @@ mod tests {
let (hash, size) = service.store_bytes(data).await.unwrap();
assert_eq!(size, data.len() as u64);
assert!(service.exists(&hash).await);
assert!(service.exists(&hash));
let retrieved = service.read(&hash).await.unwrap();
assert_eq!(retrieved, data);
@ -405,9 +459,9 @@ mod tests {
let data = b"delete me";
let (hash, _) = service.store_bytes(data).await.unwrap();
assert!(service.exists(&hash).await);
assert!(service.exists(&hash));
service.delete(&hash).await.unwrap();
assert!(!service.exists(&hash).await);
assert!(!service.exists(&hash));
}
}