treewide: better cross-device sync capabilities; in-database storage
Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: Id99798df6f7e4470caae8a193c2654aa6a6a6964
This commit is contained in:
parent
5521488a93
commit
f34c78b238
41 changed files with 8806 additions and 138 deletions
|
|
@ -104,6 +104,12 @@ pub struct Config {
|
|||
pub analytics: AnalyticsConfig,
|
||||
#[serde(default)]
|
||||
pub photos: PhotoConfig,
|
||||
#[serde(default)]
|
||||
pub managed_storage: ManagedStorageConfig,
|
||||
#[serde(default)]
|
||||
pub sync: SyncConfig,
|
||||
#[serde(default)]
|
||||
pub sharing: SharingConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
|
@ -560,6 +566,180 @@ impl Default for PhotoConfig {
|
|||
}
|
||||
}
|
||||
|
||||
// ===== Managed Storage Configuration =====
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ManagedStorageConfig {
|
||||
/// Enable managed storage for file uploads
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Directory where managed files are stored
|
||||
#[serde(default = "default_managed_storage_dir")]
|
||||
pub storage_dir: PathBuf,
|
||||
/// Maximum upload size in bytes (default: 10GB)
|
||||
#[serde(default = "default_max_upload_size")]
|
||||
pub max_upload_size: u64,
|
||||
/// Allowed MIME types for uploads (empty = allow all)
|
||||
#[serde(default)]
|
||||
pub allowed_mime_types: Vec<String>,
|
||||
/// Automatically clean up orphaned blobs
|
||||
#[serde(default = "default_true")]
|
||||
pub auto_cleanup: bool,
|
||||
/// Verify file integrity on read
|
||||
#[serde(default)]
|
||||
pub verify_on_read: bool,
|
||||
}
|
||||
|
||||
fn default_managed_storage_dir() -> PathBuf {
|
||||
Config::default_data_dir().join("managed")
|
||||
}
|
||||
|
||||
fn default_max_upload_size() -> u64 {
|
||||
10 * 1024 * 1024 * 1024 // 10GB
|
||||
}
|
||||
|
||||
impl Default for ManagedStorageConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
storage_dir: default_managed_storage_dir(),
|
||||
max_upload_size: default_max_upload_size(),
|
||||
allowed_mime_types: vec![],
|
||||
auto_cleanup: true,
|
||||
verify_on_read: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Sync Configuration =====
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ConflictResolution {
|
||||
ServerWins,
|
||||
ClientWins,
|
||||
KeepBoth,
|
||||
Manual,
|
||||
}
|
||||
|
||||
impl Default for ConflictResolution {
|
||||
fn default() -> Self {
|
||||
Self::KeepBoth
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncConfig {
|
||||
/// Enable cross-device sync functionality
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Default conflict resolution strategy
|
||||
#[serde(default)]
|
||||
pub default_conflict_resolution: ConflictResolution,
|
||||
/// Maximum file size for sync in MB
|
||||
#[serde(default = "default_max_sync_file_size")]
|
||||
pub max_file_size_mb: u64,
|
||||
/// Chunk size for chunked uploads in KB
|
||||
#[serde(default = "default_chunk_size")]
|
||||
pub chunk_size_kb: u64,
|
||||
/// Upload session timeout in hours
|
||||
#[serde(default = "default_upload_timeout")]
|
||||
pub upload_timeout_hours: u64,
|
||||
/// Maximum concurrent uploads per device
|
||||
#[serde(default = "default_max_concurrent_uploads")]
|
||||
pub max_concurrent_uploads: usize,
|
||||
/// Sync log retention in days
|
||||
#[serde(default = "default_sync_log_retention")]
|
||||
pub sync_log_retention_days: u64,
|
||||
}
|
||||
|
||||
fn default_max_sync_file_size() -> u64 {
|
||||
4096 // 4GB
|
||||
}
|
||||
|
||||
fn default_chunk_size() -> u64 {
|
||||
4096 // 4MB
|
||||
}
|
||||
|
||||
fn default_upload_timeout() -> u64 {
|
||||
24 // 24 hours
|
||||
}
|
||||
|
||||
fn default_max_concurrent_uploads() -> usize {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_sync_log_retention() -> u64 {
|
||||
90 // 90 days
|
||||
}
|
||||
|
||||
impl Default for SyncConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
default_conflict_resolution: ConflictResolution::default(),
|
||||
max_file_size_mb: default_max_sync_file_size(),
|
||||
chunk_size_kb: default_chunk_size(),
|
||||
upload_timeout_hours: default_upload_timeout(),
|
||||
max_concurrent_uploads: default_max_concurrent_uploads(),
|
||||
sync_log_retention_days: default_sync_log_retention(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Sharing Configuration =====
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SharingConfig {
|
||||
/// Enable sharing functionality
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
/// Allow creating public share links
|
||||
#[serde(default = "default_true")]
|
||||
pub allow_public_links: bool,
|
||||
/// Require password for public share links
|
||||
#[serde(default)]
|
||||
pub require_public_link_password: bool,
|
||||
/// Maximum expiry time for public links in hours (0 = unlimited)
|
||||
#[serde(default)]
|
||||
pub max_public_link_expiry_hours: u64,
|
||||
/// Allow users to reshare content shared with them
|
||||
#[serde(default = "default_true")]
|
||||
pub allow_reshare: bool,
|
||||
/// Enable share notifications
|
||||
#[serde(default = "default_true")]
|
||||
pub notifications_enabled: bool,
|
||||
/// Notification retention in days
|
||||
#[serde(default = "default_notification_retention")]
|
||||
pub notification_retention_days: u64,
|
||||
/// Share activity log retention in days
|
||||
#[serde(default = "default_activity_retention")]
|
||||
pub activity_retention_days: u64,
|
||||
}
|
||||
|
||||
fn default_notification_retention() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_activity_retention() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
impl Default for SharingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
allow_public_links: true,
|
||||
require_public_link_password: false,
|
||||
max_public_link_expiry_hours: 0,
|
||||
allow_reshare: true,
|
||||
notifications_enabled: true,
|
||||
notification_retention_days: default_notification_retention(),
|
||||
activity_retention_days: default_activity_retention(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Storage Configuration =====
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
|
@ -929,6 +1109,9 @@ impl Default for Config {
|
|||
cloud: CloudConfig::default(),
|
||||
analytics: AnalyticsConfig::default(),
|
||||
photos: PhotoConfig::default(),
|
||||
managed_storage: ManagedStorageConfig::default(),
|
||||
sync: SyncConfig::default(),
|
||||
sharing: SharingConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,54 @@ pub enum PinakesError {
|
|||
|
||||
#[error("external API error: {0}")]
|
||||
External(String),
|
||||
|
||||
// Managed Storage errors
|
||||
#[error("managed storage not enabled")]
|
||||
ManagedStorageDisabled,
|
||||
|
||||
#[error("upload too large: {0} bytes exceeds limit")]
|
||||
UploadTooLarge(u64),
|
||||
|
||||
#[error("blob not found: {0}")]
|
||||
BlobNotFound(String),
|
||||
|
||||
#[error("storage integrity error: {0}")]
|
||||
StorageIntegrity(String),
|
||||
|
||||
// Sync errors
|
||||
#[error("sync not enabled")]
|
||||
SyncDisabled,
|
||||
|
||||
#[error("device not found: {0}")]
|
||||
DeviceNotFound(String),
|
||||
|
||||
#[error("sync conflict: {0}")]
|
||||
SyncConflict(String),
|
||||
|
||||
#[error("upload session expired: {0}")]
|
||||
UploadSessionExpired(String),
|
||||
|
||||
#[error("upload session not found: {0}")]
|
||||
UploadSessionNotFound(String),
|
||||
|
||||
#[error("chunk out of order: expected {expected}, got {actual}")]
|
||||
ChunkOutOfOrder { expected: u64, actual: u64 },
|
||||
|
||||
// Sharing errors
|
||||
#[error("share not found: {0}")]
|
||||
ShareNotFound(String),
|
||||
|
||||
#[error("share expired: {0}")]
|
||||
ShareExpired(String),
|
||||
|
||||
#[error("share password required")]
|
||||
SharePasswordRequired,
|
||||
|
||||
#[error("share password invalid")]
|
||||
SharePasswordInvalid,
|
||||
|
||||
#[error("insufficient share permissions")]
|
||||
InsufficientSharePermissions,
|
||||
}
|
||||
|
||||
impl From<rusqlite::Error> for PinakesError {
|
||||
|
|
|
|||
|
|
@ -195,6 +195,12 @@ pub async fn import_file_with_options(
|
|||
rating: extracted.rating,
|
||||
perceptual_hash,
|
||||
|
||||
// Managed storage fields - external files use defaults
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ pub mod hash;
|
|||
pub mod import;
|
||||
pub mod integrity;
|
||||
pub mod jobs;
|
||||
pub mod managed_storage;
|
||||
pub mod media_type;
|
||||
pub mod metadata;
|
||||
pub mod model;
|
||||
|
|
@ -22,10 +23,13 @@ pub mod plugin;
|
|||
pub mod scan;
|
||||
pub mod scheduler;
|
||||
pub mod search;
|
||||
pub mod sharing;
|
||||
pub mod social;
|
||||
pub mod storage;
|
||||
pub mod subtitles;
|
||||
pub mod sync;
|
||||
pub mod tags;
|
||||
pub mod thumbnail;
|
||||
pub mod transcode;
|
||||
pub mod upload;
|
||||
pub mod users;
|
||||
|
|
|
|||
396
crates/pinakes-core/src/managed_storage.rs
Normal file
396
crates/pinakes-core/src/managed_storage.rs
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
//! Content-addressable managed storage service.
|
||||
//!
|
||||
//! Provides server-side file storage with:
|
||||
//! - BLAKE3 content hashing for deduplication
|
||||
//! - Hierarchical storage layout: `<root>/<hash[0:2]>/<hash[2:4]>/<full_hash>`
|
||||
//! - Integrity verification on read (optional)
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::error::{PinakesError, Result};
|
||||
use crate::model::ContentHash;
|
||||
|
||||
/// Content-addressable storage service for managed files.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManagedStorageService {
|
||||
root_dir: PathBuf,
|
||||
max_upload_size: u64,
|
||||
verify_on_read: bool,
|
||||
}
|
||||
|
||||
impl ManagedStorageService {
|
||||
/// Create a new managed storage service.
|
||||
pub fn new(root_dir: PathBuf, max_upload_size: u64, verify_on_read: bool) -> Self {
|
||||
Self {
|
||||
root_dir,
|
||||
max_upload_size,
|
||||
verify_on_read,
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the storage directory structure.
|
||||
pub async fn init(&self) -> Result<()> {
|
||||
fs::create_dir_all(&self.root_dir).await?;
|
||||
info!(path = %self.root_dir.display(), "initialized managed storage");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the storage path for a content hash.
|
||||
///
|
||||
/// Layout: `<root>/<hash[0:2]>/<hash[2:4]>/<full_hash>`
|
||||
pub fn path(&self, hash: &ContentHash) -> PathBuf {
|
||||
let h = &hash.0;
|
||||
if h.len() >= 4 {
|
||||
self.root_dir.join(&h[0..2]).join(&h[2..4]).join(h)
|
||||
} else {
|
||||
// Fallback for short hashes (shouldn't happen with BLAKE3)
|
||||
self.root_dir.join(h)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a blob exists in storage.
|
||||
pub async fn exists(&self, hash: &ContentHash) -> bool {
|
||||
self.path(hash).exists()
|
||||
}
|
||||
|
||||
/// Store a file from an async reader, computing the hash as we go.
|
||||
///
|
||||
/// Returns the content hash and file size.
|
||||
/// If the file already exists with the same hash, returns early (deduplication).
|
||||
pub async fn store_stream<R: AsyncRead + Unpin>(
|
||||
&self,
|
||||
mut reader: R,
|
||||
) -> Result<(ContentHash, u64)> {
|
||||
// First, stream to a temp file while computing the hash
|
||||
let temp_dir = self.root_dir.join("temp");
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
|
||||
let temp_id = uuid::Uuid::now_v7();
|
||||
let temp_path = temp_dir.join(temp_id.to_string());
|
||||
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let mut temp_file = fs::File::create(&temp_path).await?;
|
||||
let mut total_size = 0u64;
|
||||
|
||||
let mut buf = vec![0u8; 64 * 1024]; // 64KB buffer
|
||||
loop {
|
||||
let n = reader.read(&mut buf).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
total_size += n as u64;
|
||||
if total_size > self.max_upload_size {
|
||||
// Clean up temp file
|
||||
drop(temp_file);
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
return Err(PinakesError::UploadTooLarge(total_size));
|
||||
}
|
||||
|
||||
hasher.update(&buf[..n]);
|
||||
temp_file.write_all(&buf[..n]).await?;
|
||||
}
|
||||
|
||||
temp_file.flush().await?;
|
||||
temp_file.sync_all().await?;
|
||||
drop(temp_file);
|
||||
|
||||
let hash = ContentHash::new(hasher.finalize().to_hex().to_string());
|
||||
let final_path = self.path(&hash);
|
||||
|
||||
// Check if file already exists (deduplication)
|
||||
if final_path.exists() {
|
||||
// Verify size matches
|
||||
let existing_meta = fs::metadata(&final_path).await?;
|
||||
if existing_meta.len() == total_size {
|
||||
debug!(hash = %hash, "blob already exists, deduplicating");
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
return Ok((hash, total_size));
|
||||
} else {
|
||||
warn!(
|
||||
hash = %hash,
|
||||
expected = total_size,
|
||||
actual = existing_meta.len(),
|
||||
"size mismatch for existing blob, replacing"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Move temp file to final location
|
||||
if let Some(parent) = final_path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
fs::rename(&temp_path, &final_path).await?;
|
||||
|
||||
info!(hash = %hash, size = total_size, "stored new blob");
|
||||
Ok((hash, total_size))
|
||||
}
|
||||
|
||||
/// Store a file from a path.
|
||||
pub async fn store_file(&self, path: &Path) -> Result<(ContentHash, u64)> {
|
||||
let file = fs::File::open(path).await?;
|
||||
let reader = BufReader::new(file);
|
||||
self.store_stream(reader).await
|
||||
}
|
||||
|
||||
/// Store bytes directly.
|
||||
pub async fn store_bytes(&self, data: &[u8]) -> Result<(ContentHash, u64)> {
|
||||
use std::io::Cursor;
|
||||
let cursor = Cursor::new(data);
|
||||
self.store_stream(cursor).await
|
||||
}
|
||||
|
||||
/// Open a blob for reading.
|
||||
pub async fn open(&self, hash: &ContentHash) -> Result<fs::File> {
|
||||
let path = self.path(hash);
|
||||
if !path.exists() {
|
||||
return Err(PinakesError::BlobNotFound(hash.0.clone()));
|
||||
}
|
||||
|
||||
if self.verify_on_read {
|
||||
self.verify(hash).await?;
|
||||
}
|
||||
|
||||
fs::File::open(&path).await.map_err(|e| PinakesError::Io(e))
|
||||
}
|
||||
|
||||
/// Read a blob entirely into memory.
|
||||
pub async fn read(&self, hash: &ContentHash) -> Result<Vec<u8>> {
|
||||
let path = self.path(hash);
|
||||
if !path.exists() {
|
||||
return Err(PinakesError::BlobNotFound(hash.0.clone()));
|
||||
}
|
||||
|
||||
let data = fs::read(&path).await?;
|
||||
|
||||
if self.verify_on_read {
|
||||
let computed = blake3::hash(&data);
|
||||
if computed.to_hex().to_string() != hash.0 {
|
||||
return Err(PinakesError::StorageIntegrity(format!(
|
||||
"hash mismatch for blob {}",
|
||||
hash
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Verify the integrity of a stored blob.
|
||||
pub async fn verify(&self, hash: &ContentHash) -> Result<bool> {
|
||||
let path = self.path(hash);
|
||||
if !path.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let file = fs::File::open(&path).await?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let n = reader.read(&mut buf).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
|
||||
let computed = hasher.finalize().to_hex().to_string();
|
||||
if computed != hash.0 {
|
||||
warn!(
|
||||
expected = %hash,
|
||||
computed = %computed,
|
||||
"blob integrity check failed"
|
||||
);
|
||||
return Err(PinakesError::StorageIntegrity(format!(
|
||||
"hash mismatch: expected {}, computed {}",
|
||||
hash, computed
|
||||
)));
|
||||
}
|
||||
|
||||
debug!(hash = %hash, "blob integrity verified");
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Delete a blob from storage.
|
||||
pub async fn delete(&self, hash: &ContentHash) -> Result<()> {
|
||||
let path = self.path(hash);
|
||||
if path.exists() {
|
||||
fs::remove_file(&path).await?;
|
||||
info!(hash = %hash, "deleted blob");
|
||||
|
||||
// Try to remove empty parent directories
|
||||
if let Some(parent) = path.parent() {
|
||||
let _ = fs::remove_dir(parent).await;
|
||||
if let Some(grandparent) = parent.parent() {
|
||||
let _ = fs::remove_dir(grandparent).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the size of a stored blob.
|
||||
pub async fn size(&self, hash: &ContentHash) -> Result<u64> {
|
||||
let path = self.path(hash);
|
||||
if !path.exists() {
|
||||
return Err(PinakesError::BlobNotFound(hash.0.clone()));
|
||||
}
|
||||
let meta = fs::metadata(&path).await?;
|
||||
Ok(meta.len())
|
||||
}
|
||||
|
||||
/// List all blob hashes in storage.
|
||||
pub async fn list_all(&self) -> Result<Vec<ContentHash>> {
|
||||
let mut hashes = Vec::new();
|
||||
|
||||
let mut entries = fs::read_dir(&self.root_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.is_dir() && path.file_name().map(|n| n.len()) == Some(2) {
|
||||
let mut sub_entries = fs::read_dir(&path).await?;
|
||||
while let Some(sub_entry) = sub_entries.next_entry().await? {
|
||||
let sub_path = sub_entry.path();
|
||||
if sub_path.is_dir() && sub_path.file_name().map(|n| n.len()) == Some(2) {
|
||||
let mut file_entries = fs::read_dir(&sub_path).await?;
|
||||
while let Some(file_entry) = file_entries.next_entry().await? {
|
||||
let file_path = file_entry.path();
|
||||
if file_path.is_file() {
|
||||
if let Some(name) = file_path.file_name() {
|
||||
hashes
|
||||
.push(ContentHash::new(name.to_string_lossy().to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hashes)
|
||||
}
|
||||
|
||||
/// Calculate total storage used by all blobs.
|
||||
pub async fn total_size(&self) -> Result<u64> {
|
||||
let hashes = self.list_all().await?;
|
||||
let mut total = 0u64;
|
||||
for hash in hashes {
|
||||
if let Ok(size) = self.size(&hash).await {
|
||||
total += size;
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Clean up any orphaned temp files.
|
||||
pub async fn cleanup_temp(&self) -> Result<u64> {
|
||||
let temp_dir = self.root_dir.join("temp");
|
||||
if !temp_dir.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut entries = fs::read_dir(&temp_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.is_file() {
|
||||
// Check if temp file is old (> 1 hour)
|
||||
if let Ok(meta) = fs::metadata(&path).await {
|
||||
if let Ok(modified) = meta.modified() {
|
||||
let age = std::time::SystemTime::now()
|
||||
.duration_since(modified)
|
||||
.unwrap_or_default();
|
||||
if age.as_secs() > 3600 {
|
||||
let _ = fs::remove_file(&path).await;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
info!(count, "cleaned up orphaned temp files");
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_store_and_retrieve() {
|
||||
let dir = tempdir().unwrap();
|
||||
let service = ManagedStorageService::new(dir.path().to_path_buf(), 1024 * 1024, false);
|
||||
service.init().await.unwrap();
|
||||
|
||||
let data = b"hello, world!";
|
||||
let (hash, size) = service.store_bytes(data).await.unwrap();
|
||||
|
||||
assert_eq!(size, data.len() as u64);
|
||||
assert!(service.exists(&hash).await);
|
||||
|
||||
let retrieved = service.read(&hash).await.unwrap();
|
||||
assert_eq!(retrieved, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deduplication() {
|
||||
let dir = tempdir().unwrap();
|
||||
let service = ManagedStorageService::new(dir.path().to_path_buf(), 1024 * 1024, false);
|
||||
service.init().await.unwrap();
|
||||
|
||||
let data = b"duplicate content";
|
||||
let (hash1, _) = service.store_bytes(data).await.unwrap();
|
||||
let (hash2, _) = service.store_bytes(data).await.unwrap();
|
||||
|
||||
assert_eq!(hash1.0, hash2.0);
|
||||
assert_eq!(service.list_all().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_verify_integrity() {
|
||||
let dir = tempdir().unwrap();
|
||||
let service = ManagedStorageService::new(dir.path().to_path_buf(), 1024 * 1024, true);
|
||||
service.init().await.unwrap();
|
||||
|
||||
let data = b"verify me";
|
||||
let (hash, _) = service.store_bytes(data).await.unwrap();
|
||||
|
||||
assert!(service.verify(&hash).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_upload_too_large() {
|
||||
let dir = tempdir().unwrap();
|
||||
let service = ManagedStorageService::new(dir.path().to_path_buf(), 100, false);
|
||||
service.init().await.unwrap();
|
||||
|
||||
let data = vec![0u8; 200];
|
||||
let result = service.store_bytes(&data).await;
|
||||
|
||||
assert!(matches!(result, Err(PinakesError::UploadTooLarge(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delete() {
|
||||
let dir = tempdir().unwrap();
|
||||
let service = ManagedStorageService::new(dir.path().to_path_buf(), 1024 * 1024, false);
|
||||
service.init().await.unwrap();
|
||||
|
||||
let data = b"delete me";
|
||||
let (hash, _) = service.store_bytes(data).await.unwrap();
|
||||
assert!(service.exists(&hash).await);
|
||||
|
||||
service.delete(&hash).await.unwrap();
|
||||
assert!(!service.exists(&hash).await);
|
||||
}
|
||||
}
|
||||
|
|
@ -44,6 +44,71 @@ impl fmt::Display for ContentHash {
|
|||
}
|
||||
}
|
||||
|
||||
// ===== Managed Storage Types =====
|
||||
|
||||
/// Storage mode for media items
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum StorageMode {
|
||||
/// File exists on disk, referenced by path
|
||||
#[default]
|
||||
External,
|
||||
/// File is stored in managed content-addressable storage
|
||||
Managed,
|
||||
}
|
||||
|
||||
impl fmt::Display for StorageMode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::External => write!(f, "external"),
|
||||
Self::Managed => write!(f, "managed"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for StorageMode {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"external" => Ok(Self::External),
|
||||
"managed" => Ok(Self::Managed),
|
||||
_ => Err(format!("unknown storage mode: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A blob stored in managed storage (content-addressable)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ManagedBlob {
|
||||
pub content_hash: ContentHash,
|
||||
pub file_size: u64,
|
||||
pub mime_type: String,
|
||||
pub reference_count: u32,
|
||||
pub stored_at: DateTime<Utc>,
|
||||
pub last_verified: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Result of uploading a file to managed storage
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UploadResult {
|
||||
pub media_id: MediaId,
|
||||
pub content_hash: ContentHash,
|
||||
pub was_duplicate: bool,
|
||||
pub file_size: u64,
|
||||
}
|
||||
|
||||
/// Statistics about managed storage
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ManagedStorageStats {
|
||||
pub total_blobs: u64,
|
||||
pub total_size_bytes: u64,
|
||||
pub unique_size_bytes: u64,
|
||||
pub deduplication_ratio: f64,
|
||||
pub managed_media_count: u64,
|
||||
pub orphaned_blobs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MediaItem {
|
||||
pub id: MediaId,
|
||||
|
|
@ -73,6 +138,17 @@ pub struct MediaItem {
|
|||
pub rating: Option<i32>,
|
||||
pub perceptual_hash: Option<String>,
|
||||
|
||||
// Managed storage fields
|
||||
/// How the file is stored (external on disk or managed in content-addressable storage)
|
||||
#[serde(default)]
|
||||
pub storage_mode: StorageMode,
|
||||
/// Original filename for uploaded files (preserved separately from file_name)
|
||||
pub original_filename: Option<String>,
|
||||
/// When the file was uploaded to managed storage
|
||||
pub uploaded_at: Option<DateTime<Utc>>,
|
||||
/// Storage key for looking up the blob (usually same as content_hash)
|
||||
pub storage_key: Option<String>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
|
|
|||
434
crates/pinakes-core/src/sharing.rs
Normal file
434
crates/pinakes-core/src/sharing.rs
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
//! Enhanced sharing system.
|
||||
//!
|
||||
//! Provides comprehensive sharing capabilities:
|
||||
//! - Public link sharing with optional password/expiry
|
||||
//! - User-to-user sharing with granular permissions
|
||||
//! - Collection/tag sharing with inheritance
|
||||
//! - Activity logging and notifications
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::model::MediaId;
|
||||
use crate::users::UserId;
|
||||
|
||||
/// Unique identifier for a share.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ShareId(pub Uuid);
|
||||
|
||||
impl ShareId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ShareId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ShareId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// What is being shared.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ShareTarget {
|
||||
Media { media_id: MediaId },
|
||||
Collection { collection_id: Uuid },
|
||||
Tag { tag_id: Uuid },
|
||||
SavedSearch { search_id: Uuid },
|
||||
}
|
||||
|
||||
impl ShareTarget {
|
||||
pub fn target_type(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Media { .. } => "media",
|
||||
Self::Collection { .. } => "collection",
|
||||
Self::Tag { .. } => "tag",
|
||||
Self::SavedSearch { .. } => "saved_search",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn target_id(&self) -> Uuid {
|
||||
match self {
|
||||
Self::Media { media_id } => media_id.0,
|
||||
Self::Collection { collection_id } => *collection_id,
|
||||
Self::Tag { tag_id } => *tag_id,
|
||||
Self::SavedSearch { search_id } => *search_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Who the share is with.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ShareRecipient {
|
||||
/// Public link accessible to anyone with the token
|
||||
PublicLink {
|
||||
token: String,
|
||||
password_hash: Option<String>,
|
||||
},
|
||||
/// Shared with a specific user
|
||||
User { user_id: UserId },
|
||||
/// Shared with a group
|
||||
Group { group_id: Uuid },
|
||||
/// Shared with a federated user on another server
|
||||
Federated {
|
||||
user_handle: String,
|
||||
server_url: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ShareRecipient {
|
||||
pub fn recipient_type(&self) -> &'static str {
|
||||
match self {
|
||||
Self::PublicLink { .. } => "public_link",
|
||||
Self::User { .. } => "user",
|
||||
Self::Group { .. } => "group",
|
||||
Self::Federated { .. } => "federated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Permissions granted by a share.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SharePermissions {
|
||||
/// Can view the content
|
||||
pub can_view: bool,
|
||||
/// Can download the content
|
||||
pub can_download: bool,
|
||||
/// Can edit the content/metadata
|
||||
pub can_edit: bool,
|
||||
/// Can delete the content
|
||||
pub can_delete: bool,
|
||||
/// Can reshare with others
|
||||
pub can_reshare: bool,
|
||||
/// Can add new items (for collections)
|
||||
pub can_add: bool,
|
||||
}
|
||||
|
||||
impl SharePermissions {
|
||||
/// View-only permissions
|
||||
pub fn view_only() -> Self {
|
||||
Self {
|
||||
can_view: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Download permissions (includes view)
|
||||
pub fn download() -> Self {
|
||||
Self {
|
||||
can_view: true,
|
||||
can_download: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Edit permissions (includes view and download)
|
||||
pub fn edit() -> Self {
|
||||
Self {
|
||||
can_view: true,
|
||||
can_download: true,
|
||||
can_edit: true,
|
||||
can_add: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Full permissions
|
||||
pub fn full() -> Self {
|
||||
Self {
|
||||
can_view: true,
|
||||
can_download: true,
|
||||
can_edit: true,
|
||||
can_delete: true,
|
||||
can_reshare: true,
|
||||
can_add: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge permissions (takes the most permissive of each)
|
||||
pub fn merge(&self, other: &Self) -> Self {
|
||||
Self {
|
||||
can_view: self.can_view || other.can_view,
|
||||
can_download: self.can_download || other.can_download,
|
||||
can_edit: self.can_edit || other.can_edit,
|
||||
can_delete: self.can_delete || other.can_delete,
|
||||
can_reshare: self.can_reshare || other.can_reshare,
|
||||
can_add: self.can_add || other.can_add,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A share record.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Share {
|
||||
pub id: ShareId,
|
||||
pub target: ShareTarget,
|
||||
pub owner_id: UserId,
|
||||
pub recipient: ShareRecipient,
|
||||
pub permissions: SharePermissions,
|
||||
pub note: Option<String>,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub access_count: u64,
|
||||
pub last_accessed: Option<DateTime<Utc>>,
|
||||
/// Whether children (media in collection, etc.) inherit this share
|
||||
pub inherit_to_children: bool,
|
||||
/// Parent share if this was created via reshare
|
||||
pub parent_share_id: Option<ShareId>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl Share {
|
||||
/// Create a new public link share.
|
||||
pub fn new_public_link(
|
||||
owner_id: UserId,
|
||||
target: ShareTarget,
|
||||
token: String,
|
||||
permissions: SharePermissions,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: ShareId::new(),
|
||||
target,
|
||||
owner_id,
|
||||
recipient: ShareRecipient::PublicLink {
|
||||
token,
|
||||
password_hash: None,
|
||||
},
|
||||
permissions,
|
||||
note: None,
|
||||
expires_at: None,
|
||||
access_count: 0,
|
||||
last_accessed: None,
|
||||
inherit_to_children: true,
|
||||
parent_share_id: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new user share.
|
||||
pub fn new_user_share(
|
||||
owner_id: UserId,
|
||||
target: ShareTarget,
|
||||
recipient_user_id: UserId,
|
||||
permissions: SharePermissions,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: ShareId::new(),
|
||||
target,
|
||||
owner_id,
|
||||
recipient: ShareRecipient::User {
|
||||
user_id: recipient_user_id,
|
||||
},
|
||||
permissions,
|
||||
note: None,
|
||||
expires_at: None,
|
||||
access_count: 0,
|
||||
last_accessed: None,
|
||||
inherit_to_children: true,
|
||||
parent_share_id: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the share has expired.
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.expires_at.map(|exp| exp < Utc::now()).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Check if this is a public link share.
|
||||
pub fn is_public(&self) -> bool {
|
||||
matches!(self.recipient, ShareRecipient::PublicLink { .. })
|
||||
}
|
||||
|
||||
/// Get the public token if this is a public link share.
|
||||
pub fn public_token(&self) -> Option<&str> {
|
||||
match &self.recipient {
|
||||
ShareRecipient::PublicLink { token, .. } => Some(token),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Types of share activity actions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ShareActivityAction {
|
||||
Created,
|
||||
Updated,
|
||||
Accessed,
|
||||
Downloaded,
|
||||
Revoked,
|
||||
Expired,
|
||||
PasswordFailed,
|
||||
}
|
||||
|
||||
impl fmt::Display for ShareActivityAction {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Created => write!(f, "created"),
|
||||
Self::Updated => write!(f, "updated"),
|
||||
Self::Accessed => write!(f, "accessed"),
|
||||
Self::Downloaded => write!(f, "downloaded"),
|
||||
Self::Revoked => write!(f, "revoked"),
|
||||
Self::Expired => write!(f, "expired"),
|
||||
Self::PasswordFailed => write!(f, "password_failed"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for ShareActivityAction {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"created" => Ok(Self::Created),
|
||||
"updated" => Ok(Self::Updated),
|
||||
"accessed" => Ok(Self::Accessed),
|
||||
"downloaded" => Ok(Self::Downloaded),
|
||||
"revoked" => Ok(Self::Revoked),
|
||||
"expired" => Ok(Self::Expired),
|
||||
"password_failed" => Ok(Self::PasswordFailed),
|
||||
_ => Err(format!("unknown share activity action: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Activity log entry for a share.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ShareActivity {
|
||||
pub id: Uuid,
|
||||
pub share_id: ShareId,
|
||||
pub actor_id: Option<UserId>,
|
||||
pub actor_ip: Option<String>,
|
||||
pub action: ShareActivityAction,
|
||||
pub details: Option<String>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl ShareActivity {
|
||||
pub fn new(share_id: ShareId, action: ShareActivityAction) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
share_id,
|
||||
actor_id: None,
|
||||
actor_ip: None,
|
||||
action,
|
||||
details: None,
|
||||
timestamp: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_actor(mut self, actor_id: UserId) -> Self {
|
||||
self.actor_id = Some(actor_id);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_ip(mut self, ip: &str) -> Self {
|
||||
self.actor_ip = Some(ip.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_details(mut self, details: &str) -> Self {
|
||||
self.details = Some(details.to_string());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Types of share notifications.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ShareNotificationType {
|
||||
NewShare,
|
||||
ShareUpdated,
|
||||
ShareRevoked,
|
||||
ShareExpiring,
|
||||
ShareAccessed,
|
||||
}
|
||||
|
||||
impl fmt::Display for ShareNotificationType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::NewShare => write!(f, "new_share"),
|
||||
Self::ShareUpdated => write!(f, "share_updated"),
|
||||
Self::ShareRevoked => write!(f, "share_revoked"),
|
||||
Self::ShareExpiring => write!(f, "share_expiring"),
|
||||
Self::ShareAccessed => write!(f, "share_accessed"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for ShareNotificationType {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"new_share" => Ok(Self::NewShare),
|
||||
"share_updated" => Ok(Self::ShareUpdated),
|
||||
"share_revoked" => Ok(Self::ShareRevoked),
|
||||
"share_expiring" => Ok(Self::ShareExpiring),
|
||||
"share_accessed" => Ok(Self::ShareAccessed),
|
||||
_ => Err(format!("unknown share notification type: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A notification about a share.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ShareNotification {
|
||||
pub id: Uuid,
|
||||
pub user_id: UserId,
|
||||
pub share_id: ShareId,
|
||||
pub notification_type: ShareNotificationType,
|
||||
pub is_read: bool,
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl ShareNotification {
|
||||
pub fn new(
|
||||
user_id: UserId,
|
||||
share_id: ShareId,
|
||||
notification_type: ShareNotificationType,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
user_id,
|
||||
share_id,
|
||||
notification_type,
|
||||
is_read: false,
|
||||
created_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a random share token using UUID.
|
||||
pub fn generate_share_token() -> String {
|
||||
// Use UUIDv4 for random tokens - simple string representation
|
||||
Uuid::new_v4().simple().to_string()
|
||||
}
|
||||
|
||||
/// Hash a share password.
|
||||
pub fn hash_share_password(password: &str) -> String {
|
||||
// Use BLAKE3 for password hashing (in production, use Argon2)
|
||||
blake3::hash(password.as_bytes()).to_hex().to_string()
|
||||
}
|
||||
|
||||
/// Verify a share password.
|
||||
pub fn verify_share_password(password: &str, hash: &str) -> bool {
|
||||
let computed = hash_share_password(password);
|
||||
computed == hash
|
||||
}
|
||||
|
|
@ -511,6 +511,236 @@ pub trait StorageBackend: Send + Sync + 'static {
|
|||
language: Option<&str>,
|
||||
pagination: &Pagination,
|
||||
) -> Result<Vec<MediaItem>>;
|
||||
|
||||
// ===== Managed Storage =====
|
||||
|
||||
/// Insert a media item that uses managed storage
|
||||
async fn insert_managed_media(&self, item: &MediaItem) -> Result<()>;
|
||||
|
||||
/// Get or create a managed blob record (for deduplication tracking)
|
||||
async fn get_or_create_blob(
|
||||
&self,
|
||||
hash: &ContentHash,
|
||||
size: u64,
|
||||
mime_type: &str,
|
||||
) -> Result<ManagedBlob>;
|
||||
|
||||
/// Get a managed blob by its content hash
|
||||
async fn get_blob(&self, hash: &ContentHash) -> Result<Option<ManagedBlob>>;
|
||||
|
||||
/// Increment the reference count for a blob
|
||||
async fn increment_blob_ref(&self, hash: &ContentHash) -> Result<()>;
|
||||
|
||||
/// Decrement the reference count for a blob. Returns true if blob should be deleted.
|
||||
async fn decrement_blob_ref(&self, hash: &ContentHash) -> Result<bool>;
|
||||
|
||||
/// Update the last_verified timestamp for a blob
|
||||
async fn update_blob_verified(&self, hash: &ContentHash) -> Result<()>;
|
||||
|
||||
/// List orphaned blobs (reference_count = 0)
|
||||
async fn list_orphaned_blobs(&self) -> Result<Vec<ManagedBlob>>;
|
||||
|
||||
/// Delete a blob record
|
||||
async fn delete_blob(&self, hash: &ContentHash) -> Result<()>;
|
||||
|
||||
/// Get managed storage statistics
|
||||
async fn managed_storage_stats(&self) -> Result<ManagedStorageStats>;
|
||||
|
||||
// ===== Sync Devices =====
|
||||
|
||||
/// Register a new sync device
|
||||
async fn register_device(
|
||||
&self,
|
||||
device: &crate::sync::SyncDevice,
|
||||
token_hash: &str,
|
||||
) -> Result<crate::sync::SyncDevice>;
|
||||
|
||||
/// Get a sync device by ID
|
||||
async fn get_device(&self, id: crate::sync::DeviceId) -> Result<crate::sync::SyncDevice>;
|
||||
|
||||
/// Get a sync device by its token hash
|
||||
async fn get_device_by_token(
|
||||
&self,
|
||||
token_hash: &str,
|
||||
) -> Result<Option<crate::sync::SyncDevice>>;
|
||||
|
||||
/// List all devices for a user
|
||||
async fn list_user_devices(&self, user_id: UserId) -> Result<Vec<crate::sync::SyncDevice>>;
|
||||
|
||||
/// Update a sync device
|
||||
async fn update_device(&self, device: &crate::sync::SyncDevice) -> Result<()>;
|
||||
|
||||
/// Delete a sync device
|
||||
async fn delete_device(&self, id: crate::sync::DeviceId) -> Result<()>;
|
||||
|
||||
/// Update the last_seen_at timestamp for a device
|
||||
async fn touch_device(&self, id: crate::sync::DeviceId) -> Result<()>;
|
||||
|
||||
// ===== Sync Log =====
|
||||
|
||||
/// Record a change in the sync log
|
||||
async fn record_sync_change(&self, change: &crate::sync::SyncLogEntry) -> Result<()>;
|
||||
|
||||
/// Get changes since a cursor position
|
||||
async fn get_changes_since(
|
||||
&self,
|
||||
cursor: i64,
|
||||
limit: u64,
|
||||
) -> Result<Vec<crate::sync::SyncLogEntry>>;
|
||||
|
||||
/// Get the current sync cursor (highest sequence number)
|
||||
async fn get_current_sync_cursor(&self) -> Result<i64>;
|
||||
|
||||
/// Clean up old sync log entries
|
||||
async fn cleanup_old_sync_log(&self, before: DateTime<Utc>) -> Result<u64>;
|
||||
|
||||
// ===== Device Sync State =====
|
||||
|
||||
/// Get sync state for a device and path
|
||||
async fn get_device_sync_state(
|
||||
&self,
|
||||
device_id: crate::sync::DeviceId,
|
||||
path: &str,
|
||||
) -> Result<Option<crate::sync::DeviceSyncState>>;
|
||||
|
||||
/// Insert or update device sync state
|
||||
async fn upsert_device_sync_state(&self, state: &crate::sync::DeviceSyncState) -> Result<()>;
|
||||
|
||||
/// List all pending sync items for a device
|
||||
async fn list_pending_sync(
|
||||
&self,
|
||||
device_id: crate::sync::DeviceId,
|
||||
) -> Result<Vec<crate::sync::DeviceSyncState>>;
|
||||
|
||||
// ===== Upload Sessions (Chunked Uploads) =====
|
||||
|
||||
/// Create a new upload session
|
||||
async fn create_upload_session(&self, session: &crate::sync::UploadSession) -> Result<()>;
|
||||
|
||||
/// Get an upload session by ID
|
||||
async fn get_upload_session(&self, id: Uuid) -> Result<crate::sync::UploadSession>;
|
||||
|
||||
/// Update an upload session
|
||||
async fn update_upload_session(&self, session: &crate::sync::UploadSession) -> Result<()>;
|
||||
|
||||
/// Record a received chunk
|
||||
async fn record_chunk(&self, upload_id: Uuid, chunk: &crate::sync::ChunkInfo) -> Result<()>;
|
||||
|
||||
/// Get all chunks for an upload
|
||||
async fn get_upload_chunks(&self, upload_id: Uuid) -> Result<Vec<crate::sync::ChunkInfo>>;
|
||||
|
||||
/// Clean up expired upload sessions
|
||||
async fn cleanup_expired_uploads(&self) -> Result<u64>;
|
||||
|
||||
// ===== Sync Conflicts =====
|
||||
|
||||
/// Record a sync conflict
|
||||
async fn record_conflict(&self, conflict: &crate::sync::SyncConflict) -> Result<()>;
|
||||
|
||||
/// Get unresolved conflicts for a device
|
||||
async fn get_unresolved_conflicts(
|
||||
&self,
|
||||
device_id: crate::sync::DeviceId,
|
||||
) -> Result<Vec<crate::sync::SyncConflict>>;
|
||||
|
||||
/// Resolve a conflict
|
||||
async fn resolve_conflict(
|
||||
&self,
|
||||
id: Uuid,
|
||||
resolution: crate::config::ConflictResolution,
|
||||
) -> Result<()>;
|
||||
|
||||
// ===== Enhanced Sharing =====
|
||||
|
||||
/// Create a new share
|
||||
async fn create_share(&self, share: &crate::sharing::Share) -> Result<crate::sharing::Share>;
|
||||
|
||||
/// Get a share by ID
|
||||
async fn get_share(&self, id: crate::sharing::ShareId) -> Result<crate::sharing::Share>;
|
||||
|
||||
/// Get a share by its public token
|
||||
async fn get_share_by_token(&self, token: &str) -> Result<crate::sharing::Share>;
|
||||
|
||||
/// List shares created by a user
|
||||
async fn list_shares_by_owner(
|
||||
&self,
|
||||
owner_id: UserId,
|
||||
pagination: &Pagination,
|
||||
) -> Result<Vec<crate::sharing::Share>>;
|
||||
|
||||
/// List shares received by a user
|
||||
async fn list_shares_for_user(
|
||||
&self,
|
||||
user_id: UserId,
|
||||
pagination: &Pagination,
|
||||
) -> Result<Vec<crate::sharing::Share>>;
|
||||
|
||||
/// List all shares for a specific target
|
||||
async fn list_shares_for_target(
|
||||
&self,
|
||||
target: &crate::sharing::ShareTarget,
|
||||
) -> Result<Vec<crate::sharing::Share>>;
|
||||
|
||||
/// Update a share
|
||||
async fn update_share(&self, share: &crate::sharing::Share) -> Result<crate::sharing::Share>;
|
||||
|
||||
/// Delete a share
|
||||
async fn delete_share(&self, id: crate::sharing::ShareId) -> Result<()>;
|
||||
|
||||
/// Record that a share was accessed
|
||||
async fn record_share_access(&self, id: crate::sharing::ShareId) -> Result<()>;
|
||||
|
||||
/// Check share access for a user and target
|
||||
async fn check_share_access(
|
||||
&self,
|
||||
user_id: Option<UserId>,
|
||||
target: &crate::sharing::ShareTarget,
|
||||
) -> Result<Option<crate::sharing::SharePermissions>>;
|
||||
|
||||
/// Get effective permissions for a media item (considering inheritance)
|
||||
async fn get_effective_share_permissions(
|
||||
&self,
|
||||
user_id: Option<UserId>,
|
||||
media_id: MediaId,
|
||||
) -> Result<Option<crate::sharing::SharePermissions>>;
|
||||
|
||||
/// Batch delete shares
|
||||
async fn batch_delete_shares(&self, ids: &[crate::sharing::ShareId]) -> Result<u64>;
|
||||
|
||||
/// Clean up expired shares
|
||||
async fn cleanup_expired_shares(&self) -> Result<u64>;
|
||||
|
||||
// ===== Share Activity =====
|
||||
|
||||
/// Record share activity
|
||||
async fn record_share_activity(&self, activity: &crate::sharing::ShareActivity) -> Result<()>;
|
||||
|
||||
/// Get activity for a share
|
||||
async fn get_share_activity(
|
||||
&self,
|
||||
share_id: crate::sharing::ShareId,
|
||||
pagination: &Pagination,
|
||||
) -> Result<Vec<crate::sharing::ShareActivity>>;
|
||||
|
||||
// ===== Share Notifications =====
|
||||
|
||||
/// Create a share notification
|
||||
async fn create_share_notification(
|
||||
&self,
|
||||
notification: &crate::sharing::ShareNotification,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Get unread notifications for a user
|
||||
async fn get_unread_notifications(
|
||||
&self,
|
||||
user_id: UserId,
|
||||
) -> Result<Vec<crate::sharing::ShareNotification>>;
|
||||
|
||||
/// Mark a notification as read
|
||||
async fn mark_notification_read(&self, id: Uuid) -> Result<()>;
|
||||
|
||||
/// Mark all notifications as read for a user
|
||||
async fn mark_all_notifications_read(&self, user_id: UserId) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Comprehensive library statistics.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
297
crates/pinakes-core/src/sync/chunked.rs
Normal file
297
crates/pinakes-core/src/sync/chunked.rs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
//! Chunked upload handling for large file sync.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
use tracing::{debug, info};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::{PinakesError, Result};
|
||||
|
||||
use super::{ChunkInfo, UploadSession};
|
||||
|
||||
/// Manager for chunked uploads.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChunkedUploadManager {
|
||||
temp_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl ChunkedUploadManager {
|
||||
/// Create a new chunked upload manager.
|
||||
pub fn new(temp_dir: PathBuf) -> Self {
|
||||
Self { temp_dir }
|
||||
}
|
||||
|
||||
/// Initialize the temp directory.
|
||||
pub async fn init(&self) -> Result<()> {
|
||||
fs::create_dir_all(&self.temp_dir).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the temp file path for an upload session.
|
||||
pub fn temp_path(&self, session_id: Uuid) -> PathBuf {
|
||||
self.temp_dir.join(format!("{}.upload", session_id))
|
||||
}
|
||||
|
||||
/// Create the temp file for a new upload session.
|
||||
pub async fn create_temp_file(&self, session: &UploadSession) -> Result<()> {
|
||||
let path = self.temp_path(session.id);
|
||||
|
||||
// Create a sparse file of the expected size
|
||||
let file = fs::File::create(&path).await?;
|
||||
file.set_len(session.expected_size).await?;
|
||||
|
||||
debug!(
|
||||
session_id = %session.id,
|
||||
size = session.expected_size,
|
||||
"created temp file for upload"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write a chunk to the temp file.
|
||||
pub async fn write_chunk(
|
||||
&self,
|
||||
session: &UploadSession,
|
||||
chunk_index: u64,
|
||||
data: &[u8],
|
||||
) -> Result<ChunkInfo> {
|
||||
let path = self.temp_path(session.id);
|
||||
|
||||
if !path.exists() {
|
||||
return Err(PinakesError::UploadSessionNotFound(session.id.to_string()));
|
||||
}
|
||||
|
||||
// Calculate offset
|
||||
let offset = chunk_index * session.chunk_size;
|
||||
|
||||
// Validate chunk
|
||||
if offset >= session.expected_size {
|
||||
return Err(PinakesError::ChunkOutOfOrder {
|
||||
expected: session.chunk_count - 1,
|
||||
actual: chunk_index,
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate expected chunk size
|
||||
let expected_size = if chunk_index == session.chunk_count - 1 {
|
||||
// Last chunk may be smaller
|
||||
session.expected_size - offset
|
||||
} else {
|
||||
session.chunk_size
|
||||
};
|
||||
|
||||
if data.len() as u64 != expected_size {
|
||||
return Err(PinakesError::InvalidData(format!(
|
||||
"chunk {} has wrong size: expected {}, got {}",
|
||||
chunk_index,
|
||||
expected_size,
|
||||
data.len()
|
||||
)));
|
||||
}
|
||||
|
||||
// Write chunk to file at offset
|
||||
let mut file = fs::OpenOptions::new().write(true).open(&path).await?;
|
||||
|
||||
file.seek(std::io::SeekFrom::Start(offset)).await?;
|
||||
file.write_all(data).await?;
|
||||
file.flush().await?;
|
||||
|
||||
// Compute chunk hash
|
||||
let hash = blake3::hash(data).to_hex().to_string();
|
||||
|
||||
debug!(
|
||||
session_id = %session.id,
|
||||
chunk_index,
|
||||
offset,
|
||||
size = data.len(),
|
||||
"wrote chunk"
|
||||
);
|
||||
|
||||
Ok(ChunkInfo {
|
||||
upload_id: session.id,
|
||||
chunk_index,
|
||||
offset,
|
||||
size: data.len() as u64,
|
||||
hash,
|
||||
received_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Verify and finalize the upload.
|
||||
///
|
||||
/// Checks that:
|
||||
/// 1. All chunks are received
|
||||
/// 2. File size matches expected
|
||||
/// 3. Content hash matches expected
|
||||
pub async fn finalize(
|
||||
&self,
|
||||
session: &UploadSession,
|
||||
received_chunks: &[ChunkInfo],
|
||||
) -> Result<PathBuf> {
|
||||
let path = self.temp_path(session.id);
|
||||
|
||||
// Check all chunks received
|
||||
if received_chunks.len() as u64 != session.chunk_count {
|
||||
return Err(PinakesError::InvalidData(format!(
|
||||
"missing chunks: expected {}, got {}",
|
||||
session.chunk_count,
|
||||
received_chunks.len()
|
||||
)));
|
||||
}
|
||||
|
||||
// Verify chunk indices
|
||||
let mut indices: Vec<u64> = received_chunks.iter().map(|c| c.chunk_index).collect();
|
||||
indices.sort();
|
||||
for (i, idx) in indices.iter().enumerate() {
|
||||
if *idx != i as u64 {
|
||||
return Err(PinakesError::InvalidData(format!(
|
||||
"chunk {} missing or out of order",
|
||||
i
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// Verify file size
|
||||
let metadata = fs::metadata(&path).await?;
|
||||
if metadata.len() != session.expected_size {
|
||||
return Err(PinakesError::InvalidData(format!(
|
||||
"file size mismatch: expected {}, got {}",
|
||||
session.expected_size,
|
||||
metadata.len()
|
||||
)));
|
||||
}
|
||||
|
||||
// Verify content hash
|
||||
let computed_hash = compute_file_hash(&path).await?;
|
||||
if computed_hash != session.expected_hash.0 {
|
||||
return Err(PinakesError::StorageIntegrity(format!(
|
||||
"hash mismatch: expected {}, computed {}",
|
||||
session.expected_hash, computed_hash
|
||||
)));
|
||||
}
|
||||
|
||||
info!(
|
||||
session_id = %session.id,
|
||||
hash = %session.expected_hash,
|
||||
size = session.expected_size,
|
||||
"finalized chunked upload"
|
||||
);
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Cancel an upload and clean up temp file.
|
||||
pub async fn cancel(&self, session_id: Uuid) -> Result<()> {
|
||||
let path = self.temp_path(session_id);
|
||||
if path.exists() {
|
||||
fs::remove_file(&path).await?;
|
||||
debug!(session_id = %session_id, "cancelled upload, removed temp file");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clean up expired temp files.
|
||||
pub async fn cleanup_expired(&self, max_age_hours: u64) -> Result<u64> {
|
||||
let mut count = 0u64;
|
||||
let max_age = std::time::Duration::from_secs(max_age_hours * 3600);
|
||||
|
||||
let mut entries = fs::read_dir(&self.temp_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.extension().map(|e| e == "upload").unwrap_or(false) {
|
||||
if let Ok(metadata) = fs::metadata(&path).await {
|
||||
if let Ok(modified) = metadata.modified() {
|
||||
let age = std::time::SystemTime::now()
|
||||
.duration_since(modified)
|
||||
.unwrap_or_default();
|
||||
if age > max_age {
|
||||
let _ = fs::remove_file(&path).await;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
info!(count, "cleaned up expired upload temp files");
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the BLAKE3 hash of a file.
|
||||
async fn compute_file_hash(path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path).await?;
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let n = file.read(&mut buf).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::model::ContentHash;
|
||||
use crate::sync::UploadStatus;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chunked_upload() {
|
||||
let dir = tempdir().unwrap();
|
||||
let manager = ChunkedUploadManager::new(dir.path().to_path_buf());
|
||||
manager.init().await.unwrap();
|
||||
|
||||
// Create test data
|
||||
let data = b"Hello, World! This is test data for chunked upload.";
|
||||
let hash = blake3::hash(data).to_hex().to_string();
|
||||
let chunk_size = 20u64;
|
||||
|
||||
let session = UploadSession {
|
||||
id: Uuid::now_v7(),
|
||||
device_id: super::super::DeviceId::new(),
|
||||
target_path: "/test/file.txt".to_string(),
|
||||
expected_hash: ContentHash::new(hash.clone()),
|
||||
expected_size: data.len() as u64,
|
||||
chunk_size,
|
||||
chunk_count: (data.len() as u64 + chunk_size - 1) / chunk_size,
|
||||
status: UploadStatus::InProgress,
|
||||
created_at: Utc::now(),
|
||||
expires_at: Utc::now() + chrono::Duration::hours(24),
|
||||
last_activity: Utc::now(),
|
||||
};
|
||||
|
||||
manager.create_temp_file(&session).await.unwrap();
|
||||
|
||||
// Write chunks
|
||||
let mut chunks = Vec::new();
|
||||
for i in 0..session.chunk_count {
|
||||
let start = (i * chunk_size) as usize;
|
||||
let end = ((i + 1) * chunk_size).min(data.len() as u64) as usize;
|
||||
let chunk_data = &data[start..end];
|
||||
|
||||
let chunk = manager.write_chunk(&session, i, chunk_data).await.unwrap();
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
// Finalize
|
||||
let final_path = manager.finalize(&session, &chunks).await.unwrap();
|
||||
assert!(final_path.exists());
|
||||
|
||||
// Verify content
|
||||
let content = fs::read(&final_path).await.unwrap();
|
||||
assert_eq!(&content[..], data);
|
||||
}
|
||||
}
|
||||
144
crates/pinakes-core/src/sync/conflict.rs
Normal file
144
crates/pinakes-core/src/sync/conflict.rs
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
//! Conflict detection and resolution for sync.
|
||||
|
||||
use crate::config::ConflictResolution;
|
||||
|
||||
use super::DeviceSyncState;
|
||||
|
||||
/// Detect if there's a conflict between local and server state.
|
||||
pub fn detect_conflict(state: &DeviceSyncState) -> Option<ConflictInfo> {
|
||||
// If either side has no hash, no conflict possible
|
||||
let local_hash = state.local_hash.as_ref()?;
|
||||
let server_hash = state.server_hash.as_ref()?;
|
||||
|
||||
// Same hash = no conflict
|
||||
if local_hash == server_hash {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Both have different hashes = conflict
|
||||
Some(ConflictInfo {
|
||||
path: state.path.clone(),
|
||||
local_hash: local_hash.clone(),
|
||||
server_hash: server_hash.clone(),
|
||||
local_mtime: state.local_mtime,
|
||||
server_mtime: state.server_mtime,
|
||||
})
|
||||
}
|
||||
|
||||
/// Information about a detected conflict.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConflictInfo {
|
||||
pub path: String,
|
||||
pub local_hash: String,
|
||||
pub server_hash: String,
|
||||
pub local_mtime: Option<i64>,
|
||||
pub server_mtime: Option<i64>,
|
||||
}
|
||||
|
||||
/// Result of resolving a conflict.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ConflictOutcome {
|
||||
/// Use the server version
|
||||
UseServer,
|
||||
/// Use the local version (upload it)
|
||||
UseLocal,
|
||||
/// Keep both versions (rename one)
|
||||
KeepBoth { new_local_path: String },
|
||||
/// Requires manual intervention
|
||||
Manual,
|
||||
}
|
||||
|
||||
/// Resolve a conflict based on the configured strategy.
|
||||
pub fn resolve_conflict(
|
||||
conflict: &ConflictInfo,
|
||||
resolution: ConflictResolution,
|
||||
) -> ConflictOutcome {
|
||||
match resolution {
|
||||
ConflictResolution::ServerWins => ConflictOutcome::UseServer,
|
||||
ConflictResolution::ClientWins => ConflictOutcome::UseLocal,
|
||||
ConflictResolution::KeepBoth => {
|
||||
let new_path = generate_conflict_path(&conflict.path, &conflict.local_hash);
|
||||
ConflictOutcome::KeepBoth {
|
||||
new_local_path: new_path,
|
||||
}
|
||||
}
|
||||
ConflictResolution::Manual => ConflictOutcome::Manual,
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a new path for the conflicting local file.
|
||||
/// Format: filename.conflict-<short_hash>.ext
|
||||
fn generate_conflict_path(original_path: &str, local_hash: &str) -> String {
|
||||
let short_hash = &local_hash[..8.min(local_hash.len())];
|
||||
|
||||
if let Some((base, ext)) = original_path.rsplit_once('.') {
|
||||
format!("{}.conflict-{}.{}", base, short_hash, ext)
|
||||
} else {
|
||||
format!("{}.conflict-{}", original_path, short_hash)
|
||||
}
|
||||
}
|
||||
|
||||
/// Automatic conflict resolution based on modification times.
|
||||
/// Useful when ConflictResolution is set to a time-based strategy.
|
||||
pub fn resolve_by_mtime(conflict: &ConflictInfo) -> ConflictOutcome {
|
||||
match (conflict.local_mtime, conflict.server_mtime) {
|
||||
(Some(local), Some(server)) => {
|
||||
if local > server {
|
||||
ConflictOutcome::UseLocal
|
||||
} else {
|
||||
ConflictOutcome::UseServer
|
||||
}
|
||||
}
|
||||
(Some(_), None) => ConflictOutcome::UseLocal,
|
||||
(None, Some(_)) => ConflictOutcome::UseServer,
|
||||
(None, None) => ConflictOutcome::UseServer, // Default to server
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::sync::FileSyncStatus;
|
||||
|
||||
#[test]
|
||||
fn test_generate_conflict_path() {
|
||||
assert_eq!(
|
||||
generate_conflict_path("/path/to/file.txt", "abc12345"),
|
||||
"/path/to/file.conflict-abc12345.txt"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
generate_conflict_path("/path/to/file", "abc12345"),
|
||||
"/path/to/file.conflict-abc12345"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_conflict() {
|
||||
let state_no_conflict = DeviceSyncState {
|
||||
device_id: super::super::DeviceId::new(),
|
||||
path: "/test".to_string(),
|
||||
local_hash: Some("abc".to_string()),
|
||||
server_hash: Some("abc".to_string()),
|
||||
local_mtime: None,
|
||||
server_mtime: None,
|
||||
sync_status: FileSyncStatus::Synced,
|
||||
last_synced_at: None,
|
||||
conflict_info_json: None,
|
||||
};
|
||||
assert!(detect_conflict(&state_no_conflict).is_none());
|
||||
|
||||
let state_conflict = DeviceSyncState {
|
||||
device_id: super::super::DeviceId::new(),
|
||||
path: "/test".to_string(),
|
||||
local_hash: Some("abc".to_string()),
|
||||
server_hash: Some("def".to_string()),
|
||||
local_mtime: None,
|
||||
server_mtime: None,
|
||||
sync_status: FileSyncStatus::Conflict,
|
||||
last_synced_at: None,
|
||||
conflict_info_json: None,
|
||||
};
|
||||
assert!(detect_conflict(&state_conflict).is_some());
|
||||
}
|
||||
}
|
||||
14
crates/pinakes-core/src/sync/mod.rs
Normal file
14
crates/pinakes-core/src/sync/mod.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//! Cross-device synchronization module.
|
||||
//!
|
||||
//! Provides device registration, change tracking, and conflict resolution
|
||||
//! for syncing media libraries across multiple devices.
|
||||
|
||||
mod chunked;
|
||||
mod conflict;
|
||||
mod models;
|
||||
mod protocol;
|
||||
|
||||
pub use chunked::*;
|
||||
pub use conflict::*;
|
||||
pub use models::*;
|
||||
pub use protocol::*;
|
||||
380
crates/pinakes-core/src/sync/models.rs
Normal file
380
crates/pinakes-core/src/sync/models.rs
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
//! Sync domain models.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::ConflictResolution;
|
||||
use crate::model::{ContentHash, MediaId};
|
||||
use crate::users::UserId;
|
||||
|
||||
/// Unique identifier for a sync device.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct DeviceId(pub Uuid);
|
||||
|
||||
impl DeviceId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DeviceId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for DeviceId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Type of sync device.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DeviceType {
|
||||
Desktop,
|
||||
Mobile,
|
||||
Tablet,
|
||||
Server,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl Default for DeviceType {
|
||||
fn default() -> Self {
|
||||
Self::Other
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for DeviceType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Desktop => write!(f, "desktop"),
|
||||
Self::Mobile => write!(f, "mobile"),
|
||||
Self::Tablet => write!(f, "tablet"),
|
||||
Self::Server => write!(f, "server"),
|
||||
Self::Other => write!(f, "other"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for DeviceType {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"desktop" => Ok(Self::Desktop),
|
||||
"mobile" => Ok(Self::Mobile),
|
||||
"tablet" => Ok(Self::Tablet),
|
||||
"server" => Ok(Self::Server),
|
||||
"other" => Ok(Self::Other),
|
||||
_ => Err(format!("unknown device type: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A registered sync device.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncDevice {
|
||||
pub id: DeviceId,
|
||||
pub user_id: UserId,
|
||||
pub name: String,
|
||||
pub device_type: DeviceType,
|
||||
pub client_version: String,
|
||||
pub os_info: Option<String>,
|
||||
pub last_sync_at: Option<DateTime<Utc>>,
|
||||
pub last_seen_at: DateTime<Utc>,
|
||||
pub sync_cursor: Option<i64>,
|
||||
pub enabled: bool,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl SyncDevice {
|
||||
pub fn new(
|
||||
user_id: UserId,
|
||||
name: String,
|
||||
device_type: DeviceType,
|
||||
client_version: String,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: DeviceId::new(),
|
||||
user_id,
|
||||
name,
|
||||
device_type,
|
||||
client_version,
|
||||
os_info: None,
|
||||
last_sync_at: None,
|
||||
last_seen_at: now,
|
||||
sync_cursor: None,
|
||||
enabled: true,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Type of change recorded in the sync log.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SyncChangeType {
|
||||
Created,
|
||||
Modified,
|
||||
Deleted,
|
||||
Moved,
|
||||
MetadataUpdated,
|
||||
}
|
||||
|
||||
impl fmt::Display for SyncChangeType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Created => write!(f, "created"),
|
||||
Self::Modified => write!(f, "modified"),
|
||||
Self::Deleted => write!(f, "deleted"),
|
||||
Self::Moved => write!(f, "moved"),
|
||||
Self::MetadataUpdated => write!(f, "metadata_updated"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for SyncChangeType {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"created" => Ok(Self::Created),
|
||||
"modified" => Ok(Self::Modified),
|
||||
"deleted" => Ok(Self::Deleted),
|
||||
"moved" => Ok(Self::Moved),
|
||||
"metadata_updated" => Ok(Self::MetadataUpdated),
|
||||
_ => Err(format!("unknown sync change type: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An entry in the sync log tracking a change.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncLogEntry {
|
||||
pub id: Uuid,
|
||||
pub sequence: i64,
|
||||
pub change_type: SyncChangeType,
|
||||
pub media_id: Option<MediaId>,
|
||||
pub path: String,
|
||||
pub content_hash: Option<ContentHash>,
|
||||
pub file_size: Option<u64>,
|
||||
pub metadata_json: Option<String>,
|
||||
pub changed_by_device: Option<DeviceId>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl SyncLogEntry {
|
||||
pub fn new(
|
||||
change_type: SyncChangeType,
|
||||
path: String,
|
||||
media_id: Option<MediaId>,
|
||||
content_hash: Option<ContentHash>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
sequence: 0, // Will be assigned by database
|
||||
change_type,
|
||||
media_id,
|
||||
path,
|
||||
content_hash,
|
||||
file_size: None,
|
||||
metadata_json: None,
|
||||
changed_by_device: None,
|
||||
timestamp: Utc::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sync status for a file on a device.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FileSyncStatus {
|
||||
Synced,
|
||||
PendingUpload,
|
||||
PendingDownload,
|
||||
Conflict,
|
||||
Deleted,
|
||||
}
|
||||
|
||||
impl fmt::Display for FileSyncStatus {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Synced => write!(f, "synced"),
|
||||
Self::PendingUpload => write!(f, "pending_upload"),
|
||||
Self::PendingDownload => write!(f, "pending_download"),
|
||||
Self::Conflict => write!(f, "conflict"),
|
||||
Self::Deleted => write!(f, "deleted"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for FileSyncStatus {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"synced" => Ok(Self::Synced),
|
||||
"pending_upload" => Ok(Self::PendingUpload),
|
||||
"pending_download" => Ok(Self::PendingDownload),
|
||||
"conflict" => Ok(Self::Conflict),
|
||||
"deleted" => Ok(Self::Deleted),
|
||||
_ => Err(format!("unknown file sync status: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sync state for a specific file on a specific device.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DeviceSyncState {
|
||||
pub device_id: DeviceId,
|
||||
pub path: String,
|
||||
pub local_hash: Option<String>,
|
||||
pub server_hash: Option<String>,
|
||||
pub local_mtime: Option<i64>,
|
||||
pub server_mtime: Option<i64>,
|
||||
pub sync_status: FileSyncStatus,
|
||||
pub last_synced_at: Option<DateTime<Utc>>,
|
||||
pub conflict_info_json: Option<String>,
|
||||
}
|
||||
|
||||
/// A sync conflict that needs resolution.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncConflict {
|
||||
pub id: Uuid,
|
||||
pub device_id: DeviceId,
|
||||
pub path: String,
|
||||
pub local_hash: String,
|
||||
pub local_mtime: i64,
|
||||
pub server_hash: String,
|
||||
pub server_mtime: i64,
|
||||
pub detected_at: DateTime<Utc>,
|
||||
pub resolved_at: Option<DateTime<Utc>>,
|
||||
pub resolution: Option<ConflictResolution>,
|
||||
}
|
||||
|
||||
impl SyncConflict {
|
||||
pub fn new(
|
||||
device_id: DeviceId,
|
||||
path: String,
|
||||
local_hash: String,
|
||||
local_mtime: i64,
|
||||
server_hash: String,
|
||||
server_mtime: i64,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
device_id,
|
||||
path,
|
||||
local_hash,
|
||||
local_mtime,
|
||||
server_hash,
|
||||
server_mtime,
|
||||
detected_at: Utc::now(),
|
||||
resolved_at: None,
|
||||
resolution: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Status of an upload session.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UploadStatus {
|
||||
Pending,
|
||||
InProgress,
|
||||
Completed,
|
||||
Failed,
|
||||
Expired,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl fmt::Display for UploadStatus {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Pending => write!(f, "pending"),
|
||||
Self::InProgress => write!(f, "in_progress"),
|
||||
Self::Completed => write!(f, "completed"),
|
||||
Self::Failed => write!(f, "failed"),
|
||||
Self::Expired => write!(f, "expired"),
|
||||
Self::Cancelled => write!(f, "cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for UploadStatus {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"pending" => Ok(Self::Pending),
|
||||
"in_progress" => Ok(Self::InProgress),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"expired" => Ok(Self::Expired),
|
||||
"cancelled" => Ok(Self::Cancelled),
|
||||
_ => Err(format!("unknown upload status: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A chunked upload session.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UploadSession {
|
||||
pub id: Uuid,
|
||||
pub device_id: DeviceId,
|
||||
pub target_path: String,
|
||||
pub expected_hash: ContentHash,
|
||||
pub expected_size: u64,
|
||||
pub chunk_size: u64,
|
||||
pub chunk_count: u64,
|
||||
pub status: UploadStatus,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub last_activity: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl UploadSession {
|
||||
pub fn new(
|
||||
device_id: DeviceId,
|
||||
target_path: String,
|
||||
expected_hash: ContentHash,
|
||||
expected_size: u64,
|
||||
chunk_size: u64,
|
||||
timeout_hours: u64,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
let chunk_count = (expected_size + chunk_size - 1) / chunk_size;
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
device_id,
|
||||
target_path,
|
||||
expected_hash,
|
||||
expected_size,
|
||||
chunk_size,
|
||||
chunk_count,
|
||||
status: UploadStatus::Pending,
|
||||
created_at: now,
|
||||
expires_at: now + chrono::Duration::hours(timeout_hours as i64),
|
||||
last_activity: now,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Information about an uploaded chunk.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChunkInfo {
|
||||
pub upload_id: Uuid,
|
||||
pub chunk_index: u64,
|
||||
pub offset: u64,
|
||||
pub size: u64,
|
||||
pub hash: String,
|
||||
pub received_at: DateTime<Utc>,
|
||||
}
|
||||
215
crates/pinakes-core/src/sync/protocol.rs
Normal file
215
crates/pinakes-core/src/sync/protocol.rs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
//! Sync protocol implementation.
|
||||
//!
|
||||
//! Handles the bidirectional sync protocol between clients and server.
|
||||
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::model::{ContentHash, MediaId};
|
||||
use crate::storage::DynStorageBackend;
|
||||
|
||||
use super::{DeviceId, DeviceSyncState, FileSyncStatus, SyncChangeType, SyncLogEntry};
|
||||
|
||||
/// Request from client to get changes since a cursor.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChangesRequest {
|
||||
pub cursor: i64,
|
||||
pub limit: Option<u64>,
|
||||
}
|
||||
|
||||
/// Response containing changes since the cursor.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChangesResponse {
|
||||
pub changes: Vec<SyncLogEntry>,
|
||||
pub cursor: i64,
|
||||
pub has_more: bool,
|
||||
}
|
||||
|
||||
/// A change reported by the client.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClientChange {
|
||||
pub path: String,
|
||||
pub change_type: SyncChangeType,
|
||||
pub content_hash: Option<String>,
|
||||
pub file_size: Option<u64>,
|
||||
pub local_mtime: Option<i64>,
|
||||
pub metadata: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Request from client to report local changes.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReportChangesRequest {
|
||||
pub device_id: String,
|
||||
pub changes: Vec<ClientChange>,
|
||||
}
|
||||
|
||||
/// Result of processing a client change.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", rename_all = "snake_case")]
|
||||
pub enum ChangeResult {
|
||||
/// Change accepted, no action needed
|
||||
Accepted { path: String },
|
||||
/// Conflict detected, needs resolution
|
||||
Conflict {
|
||||
path: String,
|
||||
server_hash: String,
|
||||
server_mtime: i64,
|
||||
},
|
||||
/// Upload required for new/modified file
|
||||
UploadRequired {
|
||||
path: String,
|
||||
upload_url: String,
|
||||
session_id: String,
|
||||
},
|
||||
/// Error processing change
|
||||
Error { path: String, message: String },
|
||||
}
|
||||
|
||||
/// Response to a report changes request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReportChangesResponse {
|
||||
pub results: Vec<ChangeResult>,
|
||||
pub server_cursor: i64,
|
||||
}
|
||||
|
||||
/// Acknowledgment from client that changes have been processed.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AckRequest {
|
||||
pub device_id: String,
|
||||
pub cursor: i64,
|
||||
pub processed_paths: Vec<String>,
|
||||
}
|
||||
|
||||
/// Get changes since a cursor position.
|
||||
pub async fn get_changes(
|
||||
storage: &DynStorageBackend,
|
||||
cursor: i64,
|
||||
limit: u64,
|
||||
) -> Result<ChangesResponse> {
|
||||
let limit = limit.min(1000); // Cap at 1000
|
||||
let changes = storage.get_changes_since(cursor, limit + 1).await?;
|
||||
|
||||
let has_more = changes.len() > limit as usize;
|
||||
let changes: Vec<_> = changes.into_iter().take(limit as usize).collect();
|
||||
|
||||
let new_cursor = changes.last().map(|c| c.sequence).unwrap_or(cursor);
|
||||
|
||||
Ok(ChangesResponse {
|
||||
changes,
|
||||
cursor: new_cursor,
|
||||
has_more,
|
||||
})
|
||||
}
|
||||
|
||||
/// Record a change in the sync log.
|
||||
pub async fn record_change(
|
||||
storage: &DynStorageBackend,
|
||||
change_type: SyncChangeType,
|
||||
path: &str,
|
||||
media_id: Option<MediaId>,
|
||||
content_hash: Option<&ContentHash>,
|
||||
file_size: Option<u64>,
|
||||
changed_by_device: Option<DeviceId>,
|
||||
) -> Result<SyncLogEntry> {
|
||||
let entry = SyncLogEntry {
|
||||
id: Uuid::now_v7(),
|
||||
sequence: 0, // Will be assigned by database
|
||||
change_type,
|
||||
media_id,
|
||||
path: path.to_string(),
|
||||
content_hash: content_hash.cloned(),
|
||||
file_size,
|
||||
metadata_json: None,
|
||||
changed_by_device,
|
||||
timestamp: Utc::now(),
|
||||
};
|
||||
|
||||
storage.record_sync_change(&entry).await?;
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
/// Update device cursor after processing changes.
|
||||
pub async fn update_device_cursor(
|
||||
storage: &DynStorageBackend,
|
||||
device_id: DeviceId,
|
||||
cursor: i64,
|
||||
) -> Result<()> {
|
||||
let mut device = storage.get_device(device_id).await?;
|
||||
device.sync_cursor = Some(cursor);
|
||||
device.last_sync_at = Some(Utc::now());
|
||||
device.updated_at = Utc::now();
|
||||
storage.update_device(&device).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark a file as synced for a device.
|
||||
pub async fn mark_synced(
|
||||
storage: &DynStorageBackend,
|
||||
device_id: DeviceId,
|
||||
path: &str,
|
||||
hash: &str,
|
||||
mtime: Option<i64>,
|
||||
) -> Result<()> {
|
||||
let state = DeviceSyncState {
|
||||
device_id,
|
||||
path: path.to_string(),
|
||||
local_hash: Some(hash.to_string()),
|
||||
server_hash: Some(hash.to_string()),
|
||||
local_mtime: mtime,
|
||||
server_mtime: mtime,
|
||||
sync_status: FileSyncStatus::Synced,
|
||||
last_synced_at: Some(Utc::now()),
|
||||
conflict_info_json: None,
|
||||
};
|
||||
|
||||
storage.upsert_device_sync_state(&state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark a file as pending download for a device.
|
||||
pub async fn mark_pending_download(
|
||||
storage: &DynStorageBackend,
|
||||
device_id: DeviceId,
|
||||
path: &str,
|
||||
server_hash: &str,
|
||||
server_mtime: Option<i64>,
|
||||
) -> Result<()> {
|
||||
// Get existing state or create new
|
||||
let state = match storage.get_device_sync_state(device_id, path).await? {
|
||||
Some(mut s) => {
|
||||
s.server_hash = Some(server_hash.to_string());
|
||||
s.server_mtime = server_mtime;
|
||||
s.sync_status = FileSyncStatus::PendingDownload;
|
||||
s
|
||||
}
|
||||
None => DeviceSyncState {
|
||||
device_id,
|
||||
path: path.to_string(),
|
||||
local_hash: None,
|
||||
server_hash: Some(server_hash.to_string()),
|
||||
local_mtime: None,
|
||||
server_mtime,
|
||||
sync_status: FileSyncStatus::PendingDownload,
|
||||
last_synced_at: None,
|
||||
conflict_info_json: None,
|
||||
},
|
||||
};
|
||||
|
||||
storage.upsert_device_sync_state(&state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate a device token using UUIDs for randomness.
|
||||
pub fn generate_device_token() -> String {
|
||||
// Concatenate two UUIDs for 256 bits of randomness
|
||||
let uuid1 = uuid::Uuid::new_v4();
|
||||
let uuid2 = uuid::Uuid::new_v4();
|
||||
format!("{}{}", uuid1.simple(), uuid2.simple())
|
||||
}
|
||||
|
||||
/// Hash a device token for storage.
|
||||
pub fn hash_device_token(token: &str) -> String {
|
||||
blake3::hash(token.as_bytes()).to_hex().to_string()
|
||||
}
|
||||
265
crates/pinakes-core/src/upload.rs
Normal file
265
crates/pinakes-core/src/upload.rs
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
//! Upload processing for managed storage.
|
||||
//!
|
||||
//! Handles file uploads, metadata extraction, and MediaItem creation
|
||||
//! for files stored in managed content-addressable storage.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::{debug, info};
|
||||
|
||||
use crate::error::{PinakesError, Result};
|
||||
use crate::managed_storage::ManagedStorageService;
|
||||
use crate::media_type::MediaType;
|
||||
use crate::metadata;
|
||||
use crate::model::{MediaId, MediaItem, StorageMode, UploadResult};
|
||||
use crate::storage::DynStorageBackend;
|
||||
|
||||
/// Process an upload from an async reader.
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Stores the file in managed storage
|
||||
/// 2. Checks for duplicates by content hash
|
||||
/// 3. Extracts metadata from the file
|
||||
/// 4. Creates or updates the MediaItem
|
||||
pub async fn process_upload<R: AsyncRead + Unpin>(
|
||||
storage: &DynStorageBackend,
|
||||
managed: &ManagedStorageService,
|
||||
reader: R,
|
||||
original_filename: &str,
|
||||
mime_type: Option<&str>,
|
||||
) -> Result<UploadResult> {
|
||||
// Store the file
|
||||
let (content_hash, file_size) = managed.store_stream(reader).await?;
|
||||
|
||||
// Check if we already have a media item with this hash
|
||||
if let Some(existing) = storage.get_media_by_hash(&content_hash).await? {
|
||||
debug!(hash = %content_hash, media_id = %existing.id, "upload matched existing media item");
|
||||
return Ok(UploadResult {
|
||||
media_id: existing.id,
|
||||
content_hash,
|
||||
was_duplicate: true,
|
||||
file_size,
|
||||
});
|
||||
}
|
||||
|
||||
// Determine media type from filename
|
||||
let media_type = MediaType::from_path(Path::new(original_filename))
|
||||
.unwrap_or_else(|| MediaType::custom("unknown"));
|
||||
|
||||
// Get the actual file path in managed storage for metadata extraction
|
||||
let blob_path = managed.path(&content_hash);
|
||||
|
||||
// Extract metadata
|
||||
let extracted = metadata::extract_metadata(&blob_path, media_type.clone()).ok();
|
||||
|
||||
// Create or get blob record
|
||||
let mime = mime_type
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|| media_type.mime_type().to_string());
|
||||
let _blob = storage
|
||||
.get_or_create_blob(&content_hash, file_size, &mime)
|
||||
.await?;
|
||||
|
||||
// Create the media item
|
||||
let now = Utc::now();
|
||||
let media_id = MediaId::new();
|
||||
|
||||
let item = MediaItem {
|
||||
id: media_id,
|
||||
path: blob_path,
|
||||
file_name: sanitize_filename(original_filename),
|
||||
media_type,
|
||||
content_hash: content_hash.clone(),
|
||||
file_size,
|
||||
title: extracted.as_ref().and_then(|m| m.title.clone()),
|
||||
artist: extracted.as_ref().and_then(|m| m.artist.clone()),
|
||||
album: extracted.as_ref().and_then(|m| m.album.clone()),
|
||||
genre: extracted.as_ref().and_then(|m| m.genre.clone()),
|
||||
year: extracted.as_ref().and_then(|m| m.year),
|
||||
duration_secs: extracted.as_ref().and_then(|m| m.duration_secs),
|
||||
description: extracted.as_ref().and_then(|m| m.description.clone()),
|
||||
thumbnail_path: None,
|
||||
custom_fields: HashMap::new(),
|
||||
file_mtime: None,
|
||||
date_taken: extracted.as_ref().and_then(|m| m.date_taken),
|
||||
latitude: extracted.as_ref().and_then(|m| m.latitude),
|
||||
longitude: extracted.as_ref().and_then(|m| m.longitude),
|
||||
camera_make: extracted.as_ref().and_then(|m| m.camera_make.clone()),
|
||||
camera_model: extracted.as_ref().and_then(|m| m.camera_model.clone()),
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::Managed,
|
||||
original_filename: Some(original_filename.to_string()),
|
||||
uploaded_at: Some(now),
|
||||
storage_key: Some(content_hash.0.clone()),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
||||
// Store the media item
|
||||
storage.insert_managed_media(&item).await?;
|
||||
|
||||
info!(
|
||||
media_id = %media_id,
|
||||
hash = %content_hash,
|
||||
filename = %original_filename,
|
||||
size = file_size,
|
||||
"processed upload"
|
||||
);
|
||||
|
||||
Ok(UploadResult {
|
||||
media_id,
|
||||
content_hash,
|
||||
was_duplicate: false,
|
||||
file_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Process an upload from bytes.
|
||||
pub async fn process_upload_bytes(
|
||||
storage: &DynStorageBackend,
|
||||
managed: &ManagedStorageService,
|
||||
data: &[u8],
|
||||
original_filename: &str,
|
||||
mime_type: Option<&str>,
|
||||
) -> Result<UploadResult> {
|
||||
use std::io::Cursor;
|
||||
let cursor = Cursor::new(data);
|
||||
process_upload(storage, managed, cursor, original_filename, mime_type).await
|
||||
}
|
||||
|
||||
/// Process an upload from a local file path.
|
||||
///
|
||||
/// This is useful for migrating existing external files to managed storage.
|
||||
pub async fn process_upload_file(
|
||||
storage: &DynStorageBackend,
|
||||
managed: &ManagedStorageService,
|
||||
path: &Path,
|
||||
original_filename: Option<&str>,
|
||||
) -> Result<UploadResult> {
|
||||
let file = tokio::fs::File::open(path).await?;
|
||||
let reader = tokio::io::BufReader::new(file);
|
||||
|
||||
let filename = original_filename.unwrap_or_else(|| {
|
||||
path.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown")
|
||||
});
|
||||
|
||||
let mime = mime_guess::from_path(path).first().map(|m| m.to_string());
|
||||
|
||||
process_upload(storage, managed, reader, filename, mime.as_deref()).await
|
||||
}
|
||||
|
||||
/// Migrate an existing external media item to managed storage.
|
||||
pub async fn migrate_to_managed(
|
||||
storage: &DynStorageBackend,
|
||||
managed: &ManagedStorageService,
|
||||
media_id: MediaId,
|
||||
) -> Result<()> {
|
||||
let item = storage.get_media(media_id).await?;
|
||||
|
||||
if item.storage_mode == StorageMode::Managed {
|
||||
return Err(PinakesError::InvalidOperation(
|
||||
"media item is already in managed storage".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// Check if the external file exists
|
||||
if !item.path.exists() {
|
||||
return Err(PinakesError::FileNotFound(item.path.clone()));
|
||||
}
|
||||
|
||||
// Store the file in managed storage
|
||||
let (new_hash, new_size) = managed.store_file(&item.path).await?;
|
||||
|
||||
// Verify the hash matches (it should, unless the file changed)
|
||||
if new_hash.0 != item.content_hash.0 {
|
||||
return Err(PinakesError::StorageIntegrity(format!(
|
||||
"hash changed during migration: {} -> {}",
|
||||
item.content_hash, new_hash
|
||||
)));
|
||||
}
|
||||
|
||||
// Get or create blob record
|
||||
let mime = item.media_type.mime_type().to_string();
|
||||
let _blob = storage
|
||||
.get_or_create_blob(&new_hash, new_size, &mime)
|
||||
.await?;
|
||||
|
||||
// Update the media item
|
||||
let mut updated = item.clone();
|
||||
updated.storage_mode = StorageMode::Managed;
|
||||
updated.storage_key = Some(new_hash.0.clone());
|
||||
updated.uploaded_at = Some(Utc::now());
|
||||
updated.path = managed.path(&new_hash);
|
||||
updated.updated_at = Utc::now();
|
||||
|
||||
storage.update_media(&updated).await?;
|
||||
|
||||
info!(
|
||||
media_id = %media_id,
|
||||
hash = %new_hash,
|
||||
"migrated media item to managed storage"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sanitize a filename for storage.
|
||||
fn sanitize_filename(name: &str) -> String {
|
||||
// Remove path separators and null bytes
|
||||
name.replace(['/', '\\', '\0'], "_")
|
||||
// Trim whitespace
|
||||
.trim()
|
||||
// Truncate to reasonable length
|
||||
.chars()
|
||||
.take(255)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Delete a managed media item and clean up the blob if orphaned.
|
||||
pub async fn delete_managed_media(
|
||||
storage: &DynStorageBackend,
|
||||
managed: &ManagedStorageService,
|
||||
media_id: MediaId,
|
||||
) -> Result<()> {
|
||||
let item = storage.get_media(media_id).await?;
|
||||
|
||||
if item.storage_mode != StorageMode::Managed {
|
||||
return Err(PinakesError::InvalidOperation(
|
||||
"media item is not in managed storage".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// Decrement blob reference count
|
||||
let should_delete = storage.decrement_blob_ref(&item.content_hash).await?;
|
||||
|
||||
// Delete the media item
|
||||
storage.delete_media(media_id).await?;
|
||||
|
||||
// If blob is orphaned, delete it from storage
|
||||
if should_delete {
|
||||
managed.delete(&item.content_hash).await?;
|
||||
storage.delete_blob(&item.content_hash).await?;
|
||||
info!(hash = %item.content_hash, "deleted orphaned blob");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_filename() {
|
||||
assert_eq!(sanitize_filename("test.txt"), "test.txt");
|
||||
assert_eq!(sanitize_filename("path/to/file.txt"), "path_to_file.txt");
|
||||
assert_eq!(sanitize_filename(" spaces "), "spaces");
|
||||
assert_eq!(sanitize_filename("a".repeat(300).as_str()), "a".repeat(255));
|
||||
}
|
||||
}
|
||||
|
|
@ -43,6 +43,10 @@ async fn test_media_crud() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -129,6 +133,10 @@ async fn test_tags() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -189,6 +197,10 @@ async fn test_collections() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -244,6 +256,10 @@ async fn test_custom_fields() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -318,6 +334,10 @@ async fn test_search() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -457,6 +477,10 @@ async fn test_library_statistics_with_data() {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
|
@ -501,6 +525,10 @@ fn make_test_media(hash: &str) -> MediaItem {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use std::sync::Arc;
|
|||
|
||||
use pinakes_core::integrity::detect_orphans;
|
||||
use pinakes_core::media_type::{BuiltinMediaType, MediaType};
|
||||
use pinakes_core::model::{ContentHash, MediaId, MediaItem};
|
||||
use pinakes_core::model::{ContentHash, MediaId, MediaItem, StorageMode};
|
||||
use pinakes_core::storage::{DynStorageBackend, StorageBackend, sqlite::SqliteBackend};
|
||||
use tempfile::TempDir;
|
||||
use uuid::Uuid;
|
||||
|
|
@ -46,6 +46,10 @@ fn create_test_media_item(path: PathBuf, hash: &str) -> MediaItem {
|
|||
camera_model: None,
|
||||
rating: None,
|
||||
perceptual_hash: None,
|
||||
storage_mode: StorageMode::External,
|
||||
original_filename: None,
|
||||
uploaded_at: None,
|
||||
storage_key: None,
|
||||
created_at: chrono::Utc::now(),
|
||||
updated_at: chrono::Utc::now(),
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue