NotAShelf/pinakes
1
0
Fork 0
Code Issues 6 Pull requests Projects Releases Packages Wiki Activity Actions

pinakes: import in parallel; various UI improvements

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I1eb47cd79cd4145c56af966f6756fe1d6a6a6964
This commit is contained in:
raf 2026-02-03 10:31:20 +03:00
commit 116fe7b059
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
42 changed files with 4316 additions and 316 deletions
Download patch file Download diff file Expand all files Collapse all files

528
crates/pinakes-core/src/cache.rs
View file

@ -1,91 +1,501 @@
use std::collections::HashMap;
//! High-performance caching layer using moka.
//!
//! This module provides a comprehensive caching solution with:
//! - LRU eviction with configurable size limits
//! - TTL-based expiration
//! - Smart cache invalidation
//! - Metrics tracking (hit rate, size, evictions)
//! - Specialized caches for different data types
use std::hash::Hash;
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::sync::RwLock;
use moka::future::Cache as MokaCache;
struct CacheEntry<V> {
value: V,
inserted_at: Instant,
use crate::model::MediaId;
/// Cache statistics for monitoring and debugging.
#[derive(Debug, Clone, Default)]
pub struct CacheStats {
pub hits: u64,
pub misses: u64,
pub evictions: u64,
pub size: u64,
}
/// A simple TTL-based in-memory cache with periodic eviction.
pub struct Cache<K, V> {
entries: Arc<RwLock<HashMap<K, CacheEntry<V>>>>,
ttl: Duration,
impl CacheStats {
pub fn hit_rate(&self) -> f64 {
let total = self.hits + self.misses;
if total == 0 {
0.0
} else {
self.hits as f64 / total as f64
}
}
}
/// Atomic counters for cache metrics.
struct CacheMetrics {
hits: AtomicU64,
misses: AtomicU64,
}
impl Default for CacheMetrics {
fn default() -> Self {
Self {
hits: AtomicU64::new(0),
misses: AtomicU64::new(0),
}
}
}
impl CacheMetrics {
fn record_hit(&self) {
self.hits.fetch_add(1, Ordering::Relaxed);
}
fn record_miss(&self) {
self.misses.fetch_add(1, Ordering::Relaxed);
}
fn stats(&self) -> (u64, u64) {
(
self.hits.load(Ordering::Relaxed),
self.misses.load(Ordering::Relaxed),
)
}
}
/// A high-performance cache with LRU eviction and TTL support.
pub struct Cache<K, V>
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
inner: MokaCache<K, V>,
metrics: Arc<CacheMetrics>,
}
impl<K, V> Cache<K, V>
where
K: Eq + Hash + Clone + Send + Sync + 'static,
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
pub fn new(ttl: Duration) -> Self {
let cache = Self {
entries: Arc::new(RwLock::new(HashMap::new())),
ttl,
};
/// Create a new cache with the specified TTL and maximum capacity.
pub fn new(ttl: Duration, max_capacity: u64) -> Self {
let inner = MokaCache::builder()
.time_to_live(ttl)
.max_capacity(max_capacity)
.build();
// Spawn periodic eviction task
let entries = cache.entries.clone();
let ttl = cache.ttl;
tokio::spawn(async move {
let mut interval = tokio::time::interval(ttl);
loop {
interval.tick().await;
let now = Instant::now();
let mut map = entries.write().await;
map.retain(|_, entry| now.duration_since(entry.inserted_at) < ttl);
}
});
cache
}
pub async fn get(&self, key: &K) -> Option<V> {
let map = self.entries.read().await;
if let Some(entry) = map.get(key)
&& entry.inserted_at.elapsed() < self.ttl
{
return Some(entry.value.clone());
Self {
inner,
metrics: Arc::new(CacheMetrics::default()),
}
None
}
/// Create a new cache with TTL, max capacity, and time-to-idle.
pub fn new_with_idle(ttl: Duration, tti: Duration, max_capacity: u64) -> Self {
let inner = MokaCache::builder()
.time_to_live(ttl)
.time_to_idle(tti)
.max_capacity(max_capacity)
.build();
Self {
inner,
metrics: Arc::new(CacheMetrics::default()),
}
}
/// Get a value from the cache.
pub async fn get(&self, key: &K) -> Option<V> {
match self.inner.get(key).await {
Some(value) => {
self.metrics.record_hit();
Some(value)
}
None => {
self.metrics.record_miss();
None
}
}
}
/// Insert a value into the cache.
pub async fn insert(&self, key: K, value: V) {
let mut map = self.entries.write().await;
map.insert(
key,
CacheEntry {
value,
inserted_at: Instant::now(),
},
);
self.inner.insert(key, value).await;
}
/// Remove a specific key from the cache.
pub async fn invalidate(&self, key: &K) {
let mut map = self.entries.write().await;
map.remove(key);
self.inner.invalidate(key).await;
}
/// Clear all entries from the cache.
pub async fn invalidate_all(&self) {
self.inner.invalidate_all();
// Run pending tasks to ensure immediate invalidation
self.inner.run_pending_tasks().await;
}
/// Get the current number of entries in the cache.
pub fn entry_count(&self) -> u64 {
self.inner.entry_count()
}
/// Get cache statistics.
pub fn stats(&self) -> CacheStats {
let (hits, misses) = self.metrics.stats();
CacheStats {
hits,
misses,
evictions: 0, // Moka doesn't expose this directly
size: self.entry_count(),
}
}
}
/// Specialized cache for search query results.
pub struct QueryCache {
/// Cache keyed by (query_hash, offset, limit)
inner: Cache<String, String>,
}
impl QueryCache {
pub fn new(ttl: Duration, max_capacity: u64) -> Self {
Self {
inner: Cache::new(ttl, max_capacity),
}
}
/// Generate a cache key from query parameters.
fn make_key(query: &str, offset: u64, limit: u64, sort: Option<&str>) -> String {
use std::hash::{DefaultHasher, Hasher};
let mut hasher = DefaultHasher::new();
hasher.write(query.as_bytes());
hasher.write(&offset.to_le_bytes());
hasher.write(&limit.to_le_bytes());
if let Some(s) = sort {
hasher.write(s.as_bytes());
}
format!("q:{:016x}", hasher.finish())
}
pub async fn get(
&self,
query: &str,
offset: u64,
limit: u64,
sort: Option<&str>,
) -> Option<String> {
let key = Self::make_key(query, offset, limit, sort);
self.inner.get(&key).await
}
pub async fn insert(
&self,
query: &str,
offset: u64,
limit: u64,
sort: Option<&str>,
result: String,
) {
let key = Self::make_key(query, offset, limit, sort);
self.inner.insert(key, result).await;
}
pub async fn invalidate_all(&self) {
let mut map = self.entries.write().await;
map.clear();
self.inner.invalidate_all().await;
}
pub fn stats(&self) -> CacheStats {
self.inner.stats()
}
}
/// Application-level cache layer wrapping multiple caches for different data types.
pub struct CacheLayer {
/// Cache for serialized API responses, keyed by request path + query string.
pub responses: Cache<String, String>,
/// Specialized cache for metadata extraction results.
pub struct MetadataCache {
/// Cache keyed by content hash
inner: Cache<String, String>,
}
impl CacheLayer {
pub fn new(ttl_secs: u64) -> Self {
let ttl = Duration::from_secs(ttl_secs);
impl MetadataCache {
pub fn new(ttl: Duration, max_capacity: u64) -> Self {
Self {
responses: Cache::new(ttl),
inner: Cache::new(ttl, max_capacity),
}
}
pub async fn get(&self, content_hash: &str) -> Option<String> {
self.inner.get(&content_hash.to_string()).await
}
pub async fn insert(&self, content_hash: &str, metadata_json: String) {
self.inner
.insert(content_hash.to_string(), metadata_json)
.await;
}
pub async fn invalidate(&self, content_hash: &str) {
self.inner.invalidate(&content_hash.to_string()).await;
}
pub fn stats(&self) -> CacheStats {
self.inner.stats()
}
}
/// Specialized cache for media item data.
pub struct MediaCache {
inner: Cache<String, String>,
}
impl MediaCache {
pub fn new(ttl: Duration, max_capacity: u64) -> Self {
Self {
inner: Cache::new(ttl, max_capacity),
}
}
pub async fn get(&self, media_id: MediaId) -> Option<String> {
self.inner.get(&media_id.to_string()).await
}
pub async fn insert(&self, media_id: MediaId, item_json: String) {
self.inner.insert(media_id.to_string(), item_json).await;
}
pub async fn invalidate(&self, media_id: MediaId) {
self.inner.invalidate(&media_id.to_string()).await;
}
pub async fn invalidate_all(&self) {
self.inner.invalidate_all().await;
}
pub fn stats(&self) -> CacheStats {
self.inner.stats()
}
}
/// Configuration for the cache layer.
#[derive(Debug, Clone)]
pub struct CacheConfig {
/// TTL for response cache in seconds
pub response_ttl_secs: u64,
/// Maximum number of cached responses
pub response_max_entries: u64,
/// TTL for query cache in seconds
pub query_ttl_secs: u64,
/// Maximum number of cached query results
pub query_max_entries: u64,
/// TTL for metadata cache in seconds
pub metadata_ttl_secs: u64,
/// Maximum number of cached metadata entries
pub metadata_max_entries: u64,
/// TTL for media cache in seconds
pub media_ttl_secs: u64,
/// Maximum number of cached media items
pub media_max_entries: u64,
}
impl Default for CacheConfig {
fn default() -> Self {
Self {
response_ttl_secs: 60,
response_max_entries: 1000,
query_ttl_secs: 300,
query_max_entries: 500,
metadata_ttl_secs: 3600,
metadata_max_entries: 10000,
media_ttl_secs: 300,
media_max_entries: 5000,
}
}
}
/// Application-level cache layer wrapping multiple specialized caches.
pub struct CacheLayer {
/// Cache for serialized API responses
pub responses: Cache<String, String>,
/// Cache for search query results
pub queries: QueryCache,
/// Cache for metadata extraction results
pub metadata: MetadataCache,
/// Cache for individual media items
pub media: MediaCache,
/// Configuration
config: CacheConfig,
}
impl CacheLayer {
/// Create a new cache layer with the specified TTL (using defaults for other settings).
pub fn new(ttl_secs: u64) -> Self {
let config = CacheConfig {
response_ttl_secs: ttl_secs,
..Default::default()
};
Self::with_config(config)
}
/// Create a new cache layer with full configuration.
pub fn with_config(config: CacheConfig) -> Self {
Self {
responses: Cache::new(
Duration::from_secs(config.response_ttl_secs),
config.response_max_entries,
),
queries: QueryCache::new(
Duration::from_secs(config.query_ttl_secs),
config.query_max_entries,
),
metadata: MetadataCache::new(
Duration::from_secs(config.metadata_ttl_secs),
config.metadata_max_entries,
),
media: MediaCache::new(
Duration::from_secs(config.media_ttl_secs),
config.media_max_entries,
),
config,
}
}
/// Invalidate all caches related to a media item update.
pub async fn invalidate_for_media_update(&self, media_id: MediaId) {
self.media.invalidate(media_id).await;
// Query cache should be invalidated as search results may change
self.queries.invalidate_all().await;
}
/// Invalidate all caches related to a media item deletion.
pub async fn invalidate_for_media_delete(&self, media_id: MediaId) {
self.media.invalidate(media_id).await;
self.queries.invalidate_all().await;
}
/// Invalidate all caches (useful after bulk imports or major changes).
pub async fn invalidate_all(&self) {
self.responses.invalidate_all().await;
self.queries.invalidate_all().await;
self.media.invalidate_all().await;
// Keep metadata cache as it's keyed by content hash which doesn't change
}
/// Get aggregated statistics for all caches.
pub fn stats(&self) -> CacheLayerStats {
CacheLayerStats {
responses: self.responses.stats(),
queries: self.queries.stats(),
metadata: self.metadata.stats(),
media: self.media.stats(),
}
}
/// Get the current configuration.
pub fn config(&self) -> &CacheConfig {
&self.config
}
}
/// Aggregated statistics for the entire cache layer.
#[derive(Debug, Clone)]
pub struct CacheLayerStats {
pub responses: CacheStats,
pub queries: CacheStats,
pub metadata: CacheStats,
pub media: CacheStats,
}
impl CacheLayerStats {
/// Get the overall hit rate across all caches.
pub fn overall_hit_rate(&self) -> f64 {
let total_hits =
self.responses.hits + self.queries.hits + self.metadata.hits + self.media.hits;
let total_requests = total_hits
+ self.responses.misses
+ self.queries.misses
+ self.metadata.misses
+ self.media.misses;
if total_requests == 0 {
0.0
} else {
total_hits as f64 / total_requests as f64
}
}
/// Get the total number of entries across all caches.
pub fn total_entries(&self) -> u64 {
self.responses.size + self.queries.size + self.metadata.size + self.media.size
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_cache_basic_operations() {
let cache: Cache<String, String> = Cache::new(Duration::from_secs(60), 100);
// Insert and get
cache.insert("key1".to_string(), "value1".to_string()).await;
assert_eq!(
cache.get(&"key1".to_string()).await,
Some("value1".to_string())
);
// Miss
assert_eq!(cache.get(&"key2".to_string()).await, None);
// Invalidate
cache.invalidate(&"key1".to_string()).await;
assert_eq!(cache.get(&"key1".to_string()).await, None);
}
#[tokio::test]
async fn test_cache_stats() {
let cache: Cache<String, String> = Cache::new(Duration::from_secs(60), 100);
cache.insert("key1".to_string(), "value1".to_string()).await;
let _ = cache.get(&"key1".to_string()).await; // hit
let _ = cache.get(&"key2".to_string()).await; // miss
let stats = cache.stats();
assert_eq!(stats.hits, 1);
assert_eq!(stats.misses, 1);
assert!((stats.hit_rate() - 0.5).abs() < 0.01);
}
#[tokio::test]
async fn test_query_cache() {
let cache = QueryCache::new(Duration::from_secs(60), 100);
cache
.insert("test query", 0, 10, Some("name"), "results".to_string())
.await;
assert_eq!(
cache.get("test query", 0, 10, Some("name")).await,
Some("results".to_string())
);
// Different parameters should miss
assert_eq!(cache.get("test query", 10, 10, Some("name")).await, None);
}
#[tokio::test]
async fn test_cache_layer() {
let layer = CacheLayer::new(60);
let media_id = MediaId::new();
layer.media.insert(media_id, "{}".to_string()).await;
assert!(layer.media.get(media_id).await.is_some());
layer.invalidate_for_media_delete(media_id).await;
assert!(layer.media.get(media_id).await.is_none());
}
}

82
crates/pinakes-core/src/config.rs
View file

@ -484,6 +484,85 @@ pub struct ServerConfig {
/// If set, all requests (except /health) must include `Authorization: Bearer <key>`.
/// Can also be set via `PINAKES_API_KEY` environment variable.
pub api_key: Option<String>,
/// TLS/HTTPS configuration
#[serde(default)]
pub tls: TlsConfig,
}
/// TLS/HTTPS configuration for secure connections
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TlsConfig {
/// Enable TLS (HTTPS)
#[serde(default)]
pub enabled: bool,
/// Path to the TLS certificate file (PEM format)
#[serde(default)]
pub cert_path: Option<PathBuf>,
/// Path to the TLS private key file (PEM format)
#[serde(default)]
pub key_path: Option<PathBuf>,
/// Enable HTTP to HTTPS redirect (starts a second listener on http_port)
#[serde(default)]
pub redirect_http: bool,
/// Port for HTTP redirect listener (default: 80)
#[serde(default = "default_http_port")]
pub http_port: u16,
/// Enable HSTS (HTTP Strict Transport Security) header
#[serde(default = "default_true")]
pub hsts_enabled: bool,
/// HSTS max-age in seconds (default: 1 year)
#[serde(default = "default_hsts_max_age")]
pub hsts_max_age: u64,
}
fn default_http_port() -> u16 {
80
}
fn default_hsts_max_age() -> u64 {
31536000 // 1 year in seconds
}
impl Default for TlsConfig {
fn default() -> Self {
Self {
enabled: false,
cert_path: None,
key_path: None,
redirect_http: false,
http_port: default_http_port(),
hsts_enabled: true,
hsts_max_age: default_hsts_max_age(),
}
}
}
impl TlsConfig {
/// Validate TLS configuration
pub fn validate(&self) -> Result<(), String> {
if self.enabled {
if self.cert_path.is_none() {
return Err("TLS enabled but cert_path not specified".into());
}
if self.key_path.is_none() {
return Err("TLS enabled but key_path not specified".into());
}
if let Some(ref cert_path) = self.cert_path {
if !cert_path.exists() {
return Err(format!(
"TLS certificate file not found: {}",
cert_path.display()
));
}
}
if let Some(ref key_path) = self.key_path {
if !key_path.exists() {
return Err(format!("TLS key file not found: {}", key_path.display()));
}
}
}
Ok(())
}
}
impl Config {
@ -564,6 +643,8 @@ impl Config {
if self.scanning.import_concurrency == 0 || self.scanning.import_concurrency > 256 {
return Err("import_concurrency must be between 1 and 256".into());
}
// Validate TLS configuration
self.server.tls.validate()?;
Ok(())
}
@ -609,6 +690,7 @@ impl Default for Config {
host: "127.0.0.1".to_string(),
port: 3000,
api_key: None,
tls: TlsConfig::default(),
},
ui: UiConfig::default(),
accounts: AccountsConfig::default(),

3
crates/pinakes-core/src/error.rs
View file

@ -48,6 +48,9 @@ pub enum PinakesError {
#[error("authorization error: {0}")]
Authorization(String),
#[error("path not allowed: {0}")]
PathNotAllowed(String),
}
impl From<rusqlite::Error> for PinakesError {

92
crates/pinakes-core/src/import.rs
View file

@ -1,4 +1,5 @@
use std::path::{Path, PathBuf};
use std::time::SystemTime;
use tracing::info;
@ -14,9 +15,29 @@ use crate::thumbnail;
pub struct ImportResult {
pub media_id: MediaId,
pub was_duplicate: bool,
/// True if the file was skipped because it hasn't changed since last scan
pub was_skipped: bool,
pub path: PathBuf,
}
/// Options for import operations
#[derive(Debug, Clone, Default)]
pub struct ImportOptions {
/// Skip files that haven't changed since last scan (based on mtime)
pub incremental: bool,
/// Force re-import even if mtime hasn't changed
pub force: bool,
}
/// Get the modification time of a file as a Unix timestamp
fn get_file_mtime(path: &Path) -> Option<i64> {
std::fs::metadata(path)
.ok()
.and_then(|m| m.modified().ok())
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
}
/// Check that a canonicalized path falls under at least one configured root directory.
/// If no roots are configured, all paths are allowed (for ad-hoc imports).
pub async fn validate_path_in_roots(storage: &DynStorageBackend, path: &Path) -> Result<()> {
@ -38,6 +59,15 @@ pub async fn validate_path_in_roots(storage: &DynStorageBackend, path: &Path) ->
}
pub async fn import_file(storage: &DynStorageBackend, path: &Path) -> Result<ImportResult> {
import_file_with_options(storage, path, &ImportOptions::default()).await
}
/// Import a file with configurable options for incremental scanning
pub async fn import_file_with_options(
storage: &DynStorageBackend,
path: &Path,
options: &ImportOptions,
) -> Result<ImportResult> {
let path = path.canonicalize()?;
if !path.exists() {
@ -49,12 +79,38 @@ pub async fn import_file(storage: &DynStorageBackend, path: &Path) -> Result<Imp
let media_type = MediaType::from_path(&path)
.ok_or_else(|| PinakesError::UnsupportedMediaType(path.clone()))?;
let current_mtime = get_file_mtime(&path);
// Check for incremental scan: skip if file hasn't changed
if options.incremental && !options.force {
if let Some(existing) = storage.get_media_by_path(&path).await? {
// Compare mtimes - if they match, skip this file
if let (Some(stored_mtime), Some(curr_mtime)) = (existing.file_mtime, current_mtime) {
if stored_mtime == curr_mtime {
return Ok(ImportResult {
media_id: existing.id,
was_duplicate: false,
was_skipped: true,
path: path.clone(),
});
}
}
}
}
let content_hash = compute_file_hash(&path).await?;
if let Some(existing) = storage.get_media_by_hash(&content_hash).await? {
// Update the mtime even for duplicates so incremental scan works
if current_mtime.is_some() && existing.file_mtime != current_mtime {
let mut updated = existing.clone();
updated.file_mtime = current_mtime;
let _ = storage.update_media(&updated).await;
}
return Ok(ImportResult {
media_id: existing.id,
was_duplicate: true,
was_skipped: false,
path: path.clone(),
});
}
@ -109,6 +165,7 @@ pub async fn import_file(storage: &DynStorageBackend, path: &Path) -> Result<Imp
description: extracted.description,
thumbnail_path: thumb_path,
custom_fields: std::collections::HashMap::new(),
file_mtime: current_mtime,
created_at: now,
updated_at: now,
};
@ -144,6 +201,7 @@ pub async fn import_file(storage: &DynStorageBackend, path: &Path) -> Result<Imp
Ok(ImportResult {
media_id,
was_duplicate: false,
was_skipped: false,
path: path.clone(),
})
}
@ -180,8 +238,14 @@ pub async fn import_directory(
dir: &Path,
ignore_patterns: &[String],
) -> Result<Vec<std::result::Result<ImportResult, PinakesError>>> {
import_directory_with_concurrency(storage, dir, ignore_patterns, DEFAULT_IMPORT_CONCURRENCY)
.await
import_directory_with_options(
storage,
dir,
ignore_patterns,
DEFAULT_IMPORT_CONCURRENCY,
&ImportOptions::default(),
)
.await
}
pub async fn import_directory_with_concurrency(
@ -189,10 +253,29 @@ pub async fn import_directory_with_concurrency(
dir: &Path,
ignore_patterns: &[String],
concurrency: usize,
) -> Result<Vec<std::result::Result<ImportResult, PinakesError>>> {
import_directory_with_options(
storage,
dir,
ignore_patterns,
concurrency,
&ImportOptions::default(),
)
.await
}
/// Import a directory with full options including incremental scanning support
pub async fn import_directory_with_options(
storage: &DynStorageBackend,
dir: &Path,
ignore_patterns: &[String],
concurrency: usize,
options: &ImportOptions,
) -> Result<Vec<std::result::Result<ImportResult, PinakesError>>> {
let concurrency = concurrency.clamp(1, 256);
let dir = dir.to_path_buf();
let patterns = ignore_patterns.to_vec();
let options = options.clone();
let entries: Vec<PathBuf> = {
let dir = dir.clone();
@ -213,15 +296,14 @@ pub async fn import_directory_with_concurrency(
let mut results = Vec::with_capacity(entries.len());
let mut join_set = tokio::task::JoinSet::new();
let mut pending_paths: Vec<PathBuf> = Vec::new();
for entry_path in entries {
let storage = storage.clone();
let path = entry_path.clone();
pending_paths.push(entry_path);
let opts = options.clone();
join_set.spawn(async move {
let result = import_file(&storage, &path).await;
let result = import_file_with_options(&storage, &path, &opts).await;
(path, result)
});

37
crates/pinakes-core/src/jobs.rs
View file

@ -231,4 +231,41 @@ impl JobQueue {
job.updated_at = Utc::now();
}
}
/// Get job queue statistics
pub async fn stats(&self) -> JobQueueStats {
let jobs = self.jobs.read().await;
let mut pending = 0;
let mut running = 0;
let mut completed = 0;
let mut failed = 0;
for job in jobs.values() {
match job.status {
JobStatus::Pending => pending += 1,
JobStatus::Running { .. } => running += 1,
JobStatus::Completed { .. } => completed += 1,
JobStatus::Failed { .. } => failed += 1,
JobStatus::Cancelled => {} // Don't count cancelled jobs
}
}
JobQueueStats {
pending,
running,
completed,
failed,
total: jobs.len(),
}
}
}
/// Statistics about the job queue
#[derive(Debug, Clone, Default)]
pub struct JobQueueStats {
pub pending: usize,
pub running: usize,
pub completed: usize,
pub failed: usize,
pub total: usize,
}

1
crates/pinakes-core/src/lib.rs
View file

@ -15,6 +15,7 @@ pub mod media_type;
pub mod metadata;
pub mod model;
pub mod opener;
pub mod path_validation;
pub mod playlists;
pub mod plugin;
pub mod scan;

80
crates/pinakes-core/src/model.rs
View file

@ -61,6 +61,8 @@ pub struct MediaItem {
pub description: Option<String>,
pub thumbnail_path: Option<PathBuf>,
pub custom_fields: HashMap<String, CustomField>,
/// File modification time (Unix timestamp in seconds), used for incremental scanning
pub file_mtime: Option<i64>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@ -126,6 +128,7 @@ pub struct AuditEntry {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AuditAction {
// Media actions
Imported,
Updated,
Deleted,
@ -135,11 +138,50 @@ pub enum AuditAction {
RemovedFromCollection,
Opened,
Scanned,
// Authentication actions
LoginSuccess,
LoginFailed,
Logout,
SessionExpired,
// Authorization actions
PermissionDenied,
RoleChanged,
LibraryAccessGranted,
LibraryAccessRevoked,
// User management
UserCreated,
UserUpdated,
UserDeleted,
// Plugin actions
PluginInstalled,
PluginUninstalled,
PluginEnabled,
PluginDisabled,
// Configuration actions
ConfigChanged,
RootDirectoryAdded,
RootDirectoryRemoved,
// Social/Sharing actions
ShareLinkCreated,
ShareLinkAccessed,
// System actions
DatabaseVacuumed,
DatabaseCleared,
ExportCompleted,
IntegrityCheckCompleted,
}
impl fmt::Display for AuditAction {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
// Media actions
Self::Imported => "imported",
Self::Updated => "updated",
Self::Deleted => "deleted",
@ -149,6 +191,44 @@ impl fmt::Display for AuditAction {
Self::RemovedFromCollection => "removed_from_collection",
Self::Opened => "opened",
Self::Scanned => "scanned",
// Authentication actions
Self::LoginSuccess => "login_success",
Self::LoginFailed => "login_failed",
Self::Logout => "logout",
Self::SessionExpired => "session_expired",
// Authorization actions
Self::PermissionDenied => "permission_denied",
Self::RoleChanged => "role_changed",
Self::LibraryAccessGranted => "library_access_granted",
Self::LibraryAccessRevoked => "library_access_revoked",
// User management
Self::UserCreated => "user_created",
Self::UserUpdated => "user_updated",
Self::UserDeleted => "user_deleted",
// Plugin actions
Self::PluginInstalled => "plugin_installed",
Self::PluginUninstalled => "plugin_uninstalled",
Self::PluginEnabled => "plugin_enabled",
Self::PluginDisabled => "plugin_disabled",
// Configuration actions
Self::ConfigChanged => "config_changed",
Self::RootDirectoryAdded => "root_directory_added",
Self::RootDirectoryRemoved => "root_directory_removed",
// Social/Sharing actions
Self::ShareLinkCreated => "share_link_created",
Self::ShareLinkAccessed => "share_link_accessed",
// System actions
Self::DatabaseVacuumed => "database_vacuumed",
Self::DatabaseCleared => "database_cleared",
Self::ExportCompleted => "export_completed",
Self::IntegrityCheckCompleted => "integrity_check_completed",
};
write!(f, "{s}")
}

310
crates/pinakes-core/src/path_validation.rs Normal file
View file

@ -0,0 +1,310 @@
//! Path validation utilities to prevent path traversal attacks.
//!
//! This module provides functions to validate and sanitize file paths,
//! ensuring they remain within allowed root directories and don't contain
//! malicious path traversal sequences.
use std::path::{Path, PathBuf};
use crate::error::{PinakesError, Result};
/// Validates that a path is within one of the allowed root directories.
///
/// This function:
/// 1. Canonicalizes the path to resolve any symlinks and `..` sequences
/// 2. Checks that the canonical path starts with one of the allowed roots
/// 3. Returns the canonical path if valid, or an error if not
///
/// # Security
///
/// This prevents path traversal attacks where an attacker might try to
/// access files outside the allowed directories using sequences like:
/// - `../../../etc/passwd`
/// - `/media/../../../etc/passwd`
/// - Symlinks pointing outside allowed roots
///
/// # Arguments
///
/// * `path` - The path to validate
/// * `allowed_roots` - List of allowed root directories
///
/// # Returns
///
/// The canonicalized path if valid, or a `PathNotAllowed` error if the path
/// is outside all allowed roots.
///
/// # Example
///
/// ```no_run
/// use std::path::PathBuf;
/// use pinakes_core::path_validation::validate_path;
///
/// let allowed_roots = vec![PathBuf::from("/media"), PathBuf::from("/home/user/documents")];
/// let path = PathBuf::from("/media/music/song.mp3");
///
/// let validated = validate_path(&path, &allowed_roots).unwrap();
/// ```
pub fn validate_path(path: &Path, allowed_roots: &[PathBuf]) -> Result<PathBuf> {
// Handle the case where no roots are configured
if allowed_roots.is_empty() {
return Err(PinakesError::PathNotAllowed(
"no allowed roots configured".to_string(),
));
}
// First check if the path exists
if !path.exists() {
return Err(PinakesError::PathNotAllowed(format!(
"path does not exist: {}",
path.display()
)));
}
// Canonicalize to resolve symlinks and relative components
let canonical = path.canonicalize().map_err(|e| {
PinakesError::PathNotAllowed(format!(
"failed to canonicalize path {}: {}",
path.display(),
e
))
})?;
// Check if the canonical path is within any allowed root
let canonical_roots: Vec<PathBuf> = allowed_roots
.iter()
.filter_map(|root| root.canonicalize().ok())
.collect();
if canonical_roots.is_empty() {
return Err(PinakesError::PathNotAllowed(
"no accessible allowed roots".to_string(),
));
}
let is_allowed = canonical_roots
.iter()
.any(|root| canonical.starts_with(root));
if is_allowed {
Ok(canonical)
} else {
Err(PinakesError::PathNotAllowed(format!(
"path {} is outside allowed roots",
path.display()
)))
}
}
/// Validates a path relative to a single root directory.
///
/// This is a convenience wrapper for `validate_path` when you only have one root.
pub fn validate_path_single_root(path: &Path, root: &Path) -> Result<PathBuf> {
validate_path(path, &[root.to_path_buf()])
}
/// Checks if a path appears to contain traversal sequences without canonicalizing.
///
/// This is a quick pre-check that can reject obviously malicious paths without
/// hitting the filesystem. It should be used in addition to `validate_path`,
/// not as a replacement.
///
/// # Arguments
///
/// * `path` - The path string to check
///
/// # Returns
///
/// `true` if the path appears safe (no obvious traversal sequences),
/// `false` if it contains suspicious patterns.
pub fn path_looks_safe(path: &str) -> bool {
// Reject paths with obvious traversal patterns
!path.contains("..")
&& !path.contains("//")
&& !path.starts_with('/')
&& path.chars().filter(|c| *c == '/').count() < 50 // Reasonable depth limit
}
/// Sanitizes a filename by removing or replacing dangerous characters.
///
/// This removes:
/// - Path separators (`/`, `\`)
/// - Null bytes
/// - Control characters
/// - Leading dots (to prevent hidden files)
///
/// # Arguments
///
/// * `filename` - The filename to sanitize
///
/// # Returns
///
/// A sanitized filename safe for use on most filesystems.
pub fn sanitize_filename(filename: &str) -> String {
let sanitized: String = filename
.chars()
.filter(|c| {
// Allow alphanumeric, common punctuation, and unicode letters
c.is_alphanumeric() || matches!(*c, '-' | '_' | '.' | ' ' | '(' | ')' | '[' | ']')
})
.collect();
// Remove leading dots to prevent hidden files
let sanitized = sanitized.trim_start_matches('.');
// Remove leading/trailing whitespace
let sanitized = sanitized.trim();
// Ensure the filename isn't empty after sanitization
if sanitized.is_empty() {
"unnamed".to_string()
} else {
sanitized.to_string()
}
}
/// Joins a base path with a relative path safely.
///
/// This ensures the resulting path doesn't escape the base directory
/// through use of `..` or absolute paths in the relative component.
///
/// # Arguments
///
/// * `base` - The base directory
/// * `relative` - The relative path to join
///
/// # Returns
///
/// The joined path if safe, or an error if the relative path would escape the base.
pub fn safe_join(base: &Path, relative: &str) -> Result<PathBuf> {
// Reject absolute paths in the relative component
if relative.starts_with('/') || relative.starts_with('\\') {
return Err(PinakesError::PathNotAllowed(
"relative path cannot be absolute".to_string(),
));
}
// Reject paths with .. traversal
if relative.contains("..") {
return Err(PinakesError::PathNotAllowed(
"relative path cannot contain '..'".to_string(),
));
}
// Build the path and validate it stays within base
let joined = base.join(relative);
// Canonicalize base for comparison
let canonical_base = base.canonicalize().map_err(|e| {
PinakesError::PathNotAllowed(format!(
"failed to canonicalize base {}: {}",
base.display(),
e
))
})?;
// The joined path might not exist yet, so we can't canonicalize it directly.
// Instead, we check each component
let mut current = canonical_base.clone();
for component in Path::new(relative).components() {
use std::path::Component;
match component {
Component::Normal(name) => {
current = current.join(name);
}
Component::ParentDir => {
return Err(PinakesError::PathNotAllowed(
"path traversal detected".to_string(),
));
}
Component::CurDir => continue,
_ => {
return Err(PinakesError::PathNotAllowed(
"invalid path component".to_string(),
));
}
}
}
Ok(joined)
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::TempDir;
fn setup_test_dirs() -> TempDir {
let temp = TempDir::new().unwrap();
fs::create_dir_all(temp.path().join("allowed")).unwrap();
fs::create_dir_all(temp.path().join("forbidden")).unwrap();
fs::write(temp.path().join("allowed/file.txt"), "test").unwrap();
fs::write(temp.path().join("forbidden/secret.txt"), "secret").unwrap();
temp
}
#[test]
fn test_validate_path_allowed() {
let temp = setup_test_dirs();
let allowed_roots = vec![temp.path().join("allowed")];
let path = temp.path().join("allowed/file.txt");
let result = validate_path(&path, &allowed_roots);
assert!(result.is_ok());
}
#[test]
fn test_validate_path_forbidden() {
let temp = setup_test_dirs();
let allowed_roots = vec![temp.path().join("allowed")];
let path = temp.path().join("forbidden/secret.txt");
let result = validate_path(&path, &allowed_roots);
assert!(result.is_err());
}
#[test]
fn test_validate_path_traversal() {
let temp = setup_test_dirs();
let allowed_roots = vec![temp.path().join("allowed")];
let path = temp.path().join("allowed/../forbidden/secret.txt");
let result = validate_path(&path, &allowed_roots);
assert!(result.is_err());
}
#[test]
fn test_sanitize_filename() {
assert_eq!(sanitize_filename("normal.txt"), "normal.txt");
assert_eq!(sanitize_filename("../../../etc/passwd"), "etcpasswd");
assert_eq!(sanitize_filename(".hidden"), "hidden");
assert_eq!(sanitize_filename("file<with>bad:chars"), "filewithbadchars");
assert_eq!(sanitize_filename(""), "unnamed");
assert_eq!(sanitize_filename("..."), "unnamed");
}
#[test]
fn test_path_looks_safe() {
assert!(path_looks_safe("normal/path/file.txt"));
assert!(!path_looks_safe("../../../etc/passwd"));
assert!(!path_looks_safe("path//double/slash"));
}
#[test]
fn test_safe_join() {
let temp = TempDir::new().unwrap();
let base = temp.path();
// Valid join
let result = safe_join(base, "subdir/file.txt");
assert!(result.is_ok());
// Traversal attempt
let result = safe_join(base, "../etc/passwd");
assert!(result.is_err());
// Absolute path attempt
let result = safe_join(base, "/etc/passwd");
assert!(result.is_err());
}
}

117
crates/pinakes-core/src/scan.rs
View file

@ -14,9 +14,20 @@ pub struct ScanStatus {
pub scanning: bool,
pub files_found: usize,
pub files_processed: usize,
/// Number of files skipped because they haven't changed (incremental scan)
pub files_skipped: usize,
pub errors: Vec<String>,
}
/// Options for scanning operations
#[derive(Debug, Clone, Default)]
pub struct ScanOptions {
/// Use incremental scanning (skip unchanged files based on mtime)
pub incremental: bool,
/// Force full rescan even for incremental mode
pub force_full: bool,
}
/// Shared scan progress that can be read by the status endpoint while a scan runs.
#[derive(Clone)]
pub struct ScanProgress {
@ -50,6 +61,7 @@ impl ScanProgress {
scanning: self.is_scanning.load(Ordering::Acquire),
files_found: self.files_found.load(Ordering::Acquire),
files_processed: self.files_processed.load(Ordering::Acquire),
files_skipped: 0, // Not tracked in real-time progress
errors,
}
}
@ -89,7 +101,20 @@ pub async fn scan_directory(
dir: &Path,
ignore_patterns: &[String],
) -> Result<ScanStatus> {
scan_directory_with_progress(storage, dir, ignore_patterns, None).await
scan_directory_with_options(storage, dir, ignore_patterns, None, &ScanOptions::default()).await
}
/// Scan a directory with incremental scanning support
pub async fn scan_directory_incremental(
storage: &DynStorageBackend,
dir: &Path,
ignore_patterns: &[String],
) -> Result<ScanStatus> {
let options = ScanOptions {
incremental: true,
force_full: false,
};
scan_directory_with_options(storage, dir, ignore_patterns, None, &options).await
}
pub async fn scan_directory_with_progress(
@ -98,20 +123,62 @@ pub async fn scan_directory_with_progress(
ignore_patterns: &[String],
progress: Option<&ScanProgress>,
) -> Result<ScanStatus> {
info!(dir = %dir.display(), "starting directory scan");
scan_directory_with_options(
storage,
dir,
ignore_patterns,
progress,
&ScanOptions::default(),
)
.await
}
/// Scan a directory with full options including progress tracking and incremental mode
pub async fn scan_directory_with_options(
storage: &DynStorageBackend,
dir: &Path,
ignore_patterns: &[String],
progress: Option<&ScanProgress>,
scan_options: &ScanOptions,
) -> Result<ScanStatus> {
info!(
dir = %dir.display(),
incremental = scan_options.incremental,
force = scan_options.force_full,
"starting directory scan"
);
if let Some(p) = progress {
p.begin();
}
let results = import::import_directory(storage, dir, ignore_patterns).await?;
// Note: for configurable concurrency, use import_directory_with_concurrency directly
// Convert scan options to import options
let import_options = import::ImportOptions {
incremental: scan_options.incremental && !scan_options.force_full,
force: scan_options.force_full,
};
let results = import::import_directory_with_options(
storage,
dir,
ignore_patterns,
8, // Default concurrency
&import_options,
)
.await?;
let mut errors = Vec::new();
let mut processed = 0;
let mut skipped = 0;
for result in &results {
match result {
Ok(_) => processed += 1,
Ok(r) => {
if r.was_skipped {
skipped += 1;
} else {
processed += 1;
}
}
Err(e) => {
let msg = e.to_string();
if let Some(p) = progress {
@ -132,9 +199,20 @@ pub async fn scan_directory_with_progress(
scanning: false,
files_found: results.len(),
files_processed: processed,
files_skipped: skipped,
errors,
};
if scan_options.incremental {
info!(
dir = %dir.display(),
found = status.files_found,
processed = status.files_processed,
skipped = status.files_skipped,
"incremental scan complete"
);
}
Ok(status)
}
@ -142,19 +220,43 @@ pub async fn scan_all_roots(
storage: &DynStorageBackend,
ignore_patterns: &[String],
) -> Result<Vec<ScanStatus>> {
scan_all_roots_with_progress(storage, ignore_patterns, None).await
scan_all_roots_with_options(storage, ignore_patterns, None, &ScanOptions::default()).await
}
/// Scan all roots incrementally (skip unchanged files)
pub async fn scan_all_roots_incremental(
storage: &DynStorageBackend,
ignore_patterns: &[String],
) -> Result<Vec<ScanStatus>> {
let options = ScanOptions {
incremental: true,
force_full: false,
};
scan_all_roots_with_options(storage, ignore_patterns, None, &options).await
}
pub async fn scan_all_roots_with_progress(
storage: &DynStorageBackend,
ignore_patterns: &[String],
progress: Option<&ScanProgress>,
) -> Result<Vec<ScanStatus>> {
scan_all_roots_with_options(storage, ignore_patterns, progress, &ScanOptions::default()).await
}
/// Scan all roots with full options including progress and incremental mode
pub async fn scan_all_roots_with_options(
storage: &DynStorageBackend,
ignore_patterns: &[String],
progress: Option<&ScanProgress>,
scan_options: &ScanOptions,
) -> Result<Vec<ScanStatus>> {
let roots = storage.list_root_dirs().await?;
let mut statuses = Vec::new();
for root in roots {
match scan_directory_with_progress(storage, &root, ignore_patterns, progress).await {
match scan_directory_with_options(storage, &root, ignore_patterns, progress, scan_options)
.await
{
Ok(status) => statuses.push(status),
Err(e) => {
warn!(root = %root.display(), error = %e, "failed to scan root directory");
@ -162,6 +264,7 @@ pub async fn scan_all_roots_with_progress(
scanning: false,
files_found: 0,
files_processed: 0,
files_skipped: 0,
errors: vec![e.to_string()],
});
}

308
crates/pinakes-core/src/search.rs
View file

@ -6,7 +6,10 @@ use winnow::{ModalResult, Parser};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum SearchQuery {
FullText(String),
FieldMatch { field: String, value: String },
FieldMatch {
field: String,
value: String,
},
And(Vec<SearchQuery>),
Or(Vec<SearchQuery>),
Not(Box<SearchQuery>),
@ -14,6 +17,45 @@ pub enum SearchQuery {
Fuzzy(String),
TypeFilter(String),
TagFilter(String),
/// Range query: field:start..end (inclusive)
RangeQuery {
field: String,
start: Option<i64>,
end: Option<i64>,
},
/// Comparison query: field:>value, field:<value, field:>=value, field:<=value
CompareQuery {
field: String,
op: CompareOp,
value: i64,
},
/// Date query: created:today, modified:last-week, etc.
DateQuery {
field: String,
value: DateValue,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum CompareOp {
GreaterThan,
GreaterOrEqual,
LessThan,
LessOrEqual,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum DateValue {
Today,
Yesterday,
ThisWeek,
LastWeek,
ThisMonth,
LastMonth,
ThisYear,
LastYear,
/// Days ago: last-7d, last-30d
DaysAgo(u32),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -69,14 +111,143 @@ fn not_expr(input: &mut &str) -> ModalResult<SearchQuery> {
.parse_next(input)
}
/// Parse a date value like "today", "yesterday", "last-week", "last-30d"
fn parse_date_value(s: &str) -> Option<DateValue> {
match s.to_lowercase().as_str() {
"today" => Some(DateValue::Today),
"yesterday" => Some(DateValue::Yesterday),
"this-week" | "thisweek" => Some(DateValue::ThisWeek),
"last-week" | "lastweek" => Some(DateValue::LastWeek),
"this-month" | "thismonth" => Some(DateValue::ThisMonth),
"last-month" | "lastmonth" => Some(DateValue::LastMonth),
"this-year" | "thisyear" => Some(DateValue::ThisYear),
"last-year" | "lastyear" => Some(DateValue::LastYear),
other => {
// Try to parse "last-Nd" format (e.g., "last-7d", "last-30d")
if let Some(rest) = other.strip_prefix("last-") {
if let Some(days_str) = rest.strip_suffix('d') {
if let Ok(days) = days_str.parse::<u32>() {
return Some(DateValue::DaysAgo(days));
}
}
}
None
}
}
}
/// Parse size strings like "10MB", "1GB", "500KB" to bytes
fn parse_size_value(s: &str) -> Option<i64> {
let s = s.to_uppercase();
if let Some(num) = s.strip_suffix("GB") {
num.parse::<i64>().ok().map(|n| n * 1024 * 1024 * 1024)
} else if let Some(num) = s.strip_suffix("MB") {
num.parse::<i64>().ok().map(|n| n * 1024 * 1024)
} else if let Some(num) = s.strip_suffix("KB") {
num.parse::<i64>().ok().map(|n| n * 1024)
} else if let Some(num) = s.strip_suffix('B') {
num.parse::<i64>().ok()
} else {
s.parse::<i64>().ok()
}
}
fn field_match(input: &mut &str) -> ModalResult<SearchQuery> {
let field_name =
take_while(1.., |c: char| c.is_alphanumeric() || c == '_').map(|s: &str| s.to_string());
(field_name, ':', word_or_quoted)
.map(|(field, _, value)| match field.as_str() {
"type" => SearchQuery::TypeFilter(value),
"tag" => SearchQuery::TagFilter(value),
_ => SearchQuery::FieldMatch { field, value },
.map(|(field, _, value)| {
// Handle special field types
match field.as_str() {
"type" => return SearchQuery::TypeFilter(value),
"tag" => return SearchQuery::TagFilter(value),
_ => {}
}
// Check for range queries: field:start..end
if value.contains("..") {
let parts: Vec<&str> = value.split("..").collect();
if parts.len() == 2 {
let start = if parts[0].is_empty() {
None
} else if field == "size" {
parse_size_value(parts[0])
} else {
parts[0].parse().ok()
};
let end = if parts[1].is_empty() {
None
} else if field == "size" {
parse_size_value(parts[1])
} else {
parts[1].parse().ok()
};
return SearchQuery::RangeQuery { field, start, end };
}
}
// Check for comparison queries: >=, <=, >, <
if let Some(rest) = value.strip_prefix(">=") {
let val = if field == "size" {
parse_size_value(rest).unwrap_or(0)
} else {
rest.parse().unwrap_or(0)
};
return SearchQuery::CompareQuery {
field,
op: CompareOp::GreaterOrEqual,
value: val,
};
}
if let Some(rest) = value.strip_prefix("<=") {
let val = if field == "size" {
parse_size_value(rest).unwrap_or(0)
} else {
rest.parse().unwrap_or(0)
};
return SearchQuery::CompareQuery {
field,
op: CompareOp::LessOrEqual,
value: val,
};
}
if let Some(rest) = value.strip_prefix('>') {
let val = if field == "size" {
parse_size_value(rest).unwrap_or(0)
} else {
rest.parse().unwrap_or(0)
};
return SearchQuery::CompareQuery {
field,
op: CompareOp::GreaterThan,
value: val,
};
}
if let Some(rest) = value.strip_prefix('<') {
let val = if field == "size" {
parse_size_value(rest).unwrap_or(0)
} else {
rest.parse().unwrap_or(0)
};
return SearchQuery::CompareQuery {
field,
op: CompareOp::LessThan,
value: val,
};
}
// Check for date queries on created/modified fields
if field == "created" || field == "modified" {
if let Some(date_val) = parse_date_value(&value) {
return SearchQuery::DateQuery {
field,
value: date_val,
};
}
}
// Default: simple field match
SearchQuery::FieldMatch { field, value }
})
.parse_next(input)
}
@ -253,4 +424,131 @@ mod tests {
let q = parse_search_query("\"hello world\"").unwrap();
assert_eq!(q, SearchQuery::FullText("hello world".into()));
}
#[test]
fn test_range_query_year() {
let q = parse_search_query("year:2020..2023").unwrap();
assert_eq!(
q,
SearchQuery::RangeQuery {
field: "year".into(),
start: Some(2020),
end: Some(2023)
}
);
}
#[test]
fn test_range_query_open_start() {
let q = parse_search_query("year:..2023").unwrap();
assert_eq!(
q,
SearchQuery::RangeQuery {
field: "year".into(),
start: None,
end: Some(2023)
}
);
}
#[test]
fn test_range_query_open_end() {
let q = parse_search_query("year:2020..").unwrap();
assert_eq!(
q,
SearchQuery::RangeQuery {
field: "year".into(),
start: Some(2020),
end: None
}
);
}
#[test]
fn test_compare_greater_than() {
let q = parse_search_query("year:>2020").unwrap();
assert_eq!(
q,
SearchQuery::CompareQuery {
field: "year".into(),
op: CompareOp::GreaterThan,
value: 2020
}
);
}
#[test]
fn test_compare_less_or_equal() {
let q = parse_search_query("year:<=2023").unwrap();
assert_eq!(
q,
SearchQuery::CompareQuery {
field: "year".into(),
op: CompareOp::LessOrEqual,
value: 2023
}
);
}
#[test]
fn test_size_compare_mb() {
let q = parse_search_query("size:>10MB").unwrap();
assert_eq!(
q,
SearchQuery::CompareQuery {
field: "size".into(),
op: CompareOp::GreaterThan,
value: 10 * 1024 * 1024
}
);
}
#[test]
fn test_size_range_gb() {
let q = parse_search_query("size:1GB..2GB").unwrap();
assert_eq!(
q,
SearchQuery::RangeQuery {
field: "size".into(),
start: Some(1024 * 1024 * 1024),
end: Some(2 * 1024 * 1024 * 1024)
}
);
}
#[test]
fn test_date_query_today() {
let q = parse_search_query("created:today").unwrap();
assert_eq!(
q,
SearchQuery::DateQuery {
field: "created".into(),
value: DateValue::Today
}
);
}
#[test]
fn test_date_query_last_week() {
let q = parse_search_query("modified:last-week").unwrap();
assert_eq!(
q,
SearchQuery::DateQuery {
field: "modified".into(),
value: DateValue::LastWeek
}
);
}
#[test]
fn test_date_query_days_ago() {
let q = parse_search_query("created:last-30d").unwrap();
assert_eq!(
q,
SearchQuery::DateQuery {
field: "created".into(),
value: DateValue::DaysAgo(30)
}
);
}
}

55
crates/pinakes-core/src/storage/mod.rs
View file

@ -46,6 +46,8 @@ pub trait StorageBackend: Send + Sync + 'static {
async fn get_media(&self, id: MediaId) -> Result<MediaItem>;
async fn count_media(&self) -> Result<u64>;
async fn get_media_by_hash(&self, hash: &ContentHash) -> Result<Option<MediaItem>>;
/// Get a media item by its file path (used for incremental scanning)
async fn get_media_by_path(&self, path: &std::path::Path) -> Result<Option<MediaItem>>;
async fn list_media(&self, pagination: &Pagination) -> Result<Vec<MediaItem>>;
async fn update_media(&self, item: &MediaItem) -> Result<()>;
async fn delete_media(&self, id: MediaId) -> Result<()>;
@ -232,6 +234,59 @@ pub trait StorageBackend: Send + Sync + 'static {
root_path: &str,
) -> Result<()>;
/// Check if a user has access to a specific media item based on library permissions.
/// Returns the permission level if access is granted, or an error if denied.
/// Admin users (role=admin) bypass library checks and have full access.
async fn check_library_access(
&self,
user_id: crate::users::UserId,
media_id: crate::model::MediaId,
) -> Result<crate::users::LibraryPermission> {
// Default implementation: get the media item's path and check against user's library access
let media = self.get_media(media_id).await?;
let path_str = media.path.to_string_lossy().to_string();
// Get user's library permissions
let libraries = self.get_user_libraries(user_id).await?;
// If user has no library restrictions, they have no access (unless they're admin)
// This default impl requires at least one matching library permission
for lib in &libraries {
if path_str.starts_with(&lib.root_path) {
return Ok(lib.permission);
}
}
Err(crate::error::PinakesError::Authorization(format!(
"user {} has no access to media {}",
user_id, media_id
)))
}
/// Check if a user has at least read access to a media item
async fn has_media_read_access(
&self,
user_id: crate::users::UserId,
media_id: crate::model::MediaId,
) -> Result<bool> {
match self.check_library_access(user_id, media_id).await {
Ok(perm) => Ok(perm.can_read()),
Err(_) => Ok(false),
}
}
/// Check if a user has write access to a media item
async fn has_media_write_access(
&self,
user_id: crate::users::UserId,
media_id: crate::model::MediaId,
) -> Result<bool> {
match self.check_library_access(user_id, media_id).await {
Ok(perm) => Ok(perm.can_write()),
Err(_) => Ok(false),
}
}
// ===== Ratings =====
async fn rate_media(
&self,

268
crates/pinakes-core/src/storage/postgres.rs
View file

@ -114,6 +114,7 @@ fn row_to_media_item(row: &Row) -> Result<MediaItem> {
.get::<_, Option<String>>("thumbnail_path")
.map(PathBuf::from),
custom_fields: HashMap::new(),
file_mtime: row.get("file_mtime"),
created_at: row.get("created_at"),
updated_at: row.get("updated_at"),
})
@ -198,11 +199,61 @@ fn build_search_inner(
if text.is_empty() {
return Ok("TRUE".to_string());
}
let idx = *offset;
// Combine FTS with trigram similarity and ILIKE for comprehensive fuzzy matching
// This allows partial matches like "mus" -> "music"
let idx_fts = *offset;
*offset += 1;
let idx_prefix = *offset;
*offset += 1;
let idx_ilike = *offset;
*offset += 1;
let idx_sim_title = *offset;
*offset += 1;
let idx_sim_artist = *offset;
*offset += 1;
let idx_sim_album = *offset;
*offset += 1;
let idx_sim_filename = *offset;
*offset += 1;
// Sanitize for tsquery prefix matching
let sanitized = text.replace(['&', '|', '!', '(', ')', ':', '*', '\\', '\''], "");
let prefix_query = if sanitized.contains(' ') {
// For multi-word, join with & and add :* to last word
let words: Vec<&str> = sanitized.split_whitespace().collect();
if let Some((last, rest)) = words.split_last() {
let prefix_parts: Vec<String> = rest.iter().map(|w| w.to_string()).collect();
if prefix_parts.is_empty() {
format!("{}:*", last)
} else {
format!("{} & {}:*", prefix_parts.join(" & "), last)
}
} else {
format!("{}:*", sanitized)
}
} else {
format!("{}:*", sanitized)
};
params.push(Box::new(text.clone()));
params.push(Box::new(prefix_query));
params.push(Box::new(format!("%{}%", text)));
params.push(Box::new(text.clone()));
params.push(Box::new(text.clone()));
params.push(Box::new(text.clone()));
params.push(Box::new(text.clone()));
Ok(format!(
"search_vector @@ plainto_tsquery('english', ${idx})"
"(\
search_vector @@ plainto_tsquery('english', ${idx_fts}) OR \
search_vector @@ to_tsquery('english', ${idx_prefix}) OR \
LOWER(COALESCE(title, '')) LIKE LOWER(${idx_ilike}) OR \
LOWER(COALESCE(file_name, '')) LIKE LOWER(${idx_ilike}) OR \
similarity(COALESCE(title, ''), ${idx_sim_title}) > 0.3 OR \
similarity(COALESCE(artist, ''), ${idx_sim_artist}) > 0.3 OR \
similarity(COALESCE(album, ''), ${idx_sim_album}) > 0.3 OR \
similarity(COALESCE(file_name, ''), ${idx_sim_filename}) > 0.25\
)"
))
}
SearchQuery::Prefix(term) => {
@ -214,14 +265,31 @@ fn build_search_inner(
Ok(format!("search_vector @@ to_tsquery('english', ${idx})"))
}
SearchQuery::Fuzzy(term) => {
// Use trigram similarity on multiple fields
let idx_title = *offset;
*offset += 1;
let idx_artist = *offset;
*offset += 1;
let idx_album = *offset;
*offset += 1;
let idx_filename = *offset;
*offset += 1;
let idx_ilike = *offset;
*offset += 1;
params.push(Box::new(term.clone()));
params.push(Box::new(term.clone()));
params.push(Box::new(term.clone()));
params.push(Box::new(term.clone()));
params.push(Box::new(format!("%{}%", term)));
Ok(format!(
"(similarity(COALESCE(title, ''), ${idx_title}) > 0.3 OR similarity(COALESCE(artist, ''), ${idx_artist}) > 0.3)"
"(\
similarity(COALESCE(title, ''), ${idx_title}) > 0.3 OR \
similarity(COALESCE(artist, ''), ${idx_artist}) > 0.3 OR \
similarity(COALESCE(album, ''), ${idx_album}) > 0.3 OR \
similarity(COALESCE(file_name, ''), ${idx_filename}) > 0.25 OR \
LOWER(COALESCE(title, '')) LIKE LOWER(${idx_ilike}) OR \
LOWER(COALESCE(file_name, '')) LIKE LOWER(${idx_ilike})\
)"
))
}
SearchQuery::FieldMatch { field, value } => {
@ -277,6 +345,86 @@ fn build_search_inner(
let frag = build_search_inner(inner, offset, params, type_filters, tag_filters)?;
Ok(format!("NOT ({frag})"))
}
SearchQuery::RangeQuery { field, start, end } => {
let col = match field.as_str() {
"year" => "year",
"size" | "file_size" => "file_size",
"duration" => "duration_secs",
_ => return Ok("TRUE".to_string()), // Unknown field, ignore
};
match (start, end) {
(Some(s), Some(e)) => {
let idx_start = *offset;
*offset += 1;
let idx_end = *offset;
*offset += 1;
params.push(Box::new(*s));
params.push(Box::new(*e));
Ok(format!("({col} >= ${idx_start} AND {col} <= ${idx_end})"))
}
(Some(s), None) => {
let idx = *offset;
*offset += 1;
params.push(Box::new(*s));
Ok(format!("{col} >= ${idx}"))
}
(None, Some(e)) => {
let idx = *offset;
*offset += 1;
params.push(Box::new(*e));
Ok(format!("{col} <= ${idx}"))
}
(None, None) => Ok("TRUE".to_string()),
}
}
SearchQuery::CompareQuery { field, op, value } => {
let col = match field.as_str() {
"year" => "year",
"size" | "file_size" => "file_size",
"duration" => "duration_secs",
_ => return Ok("TRUE".to_string()), // Unknown field, ignore
};
let op_sql = match op {
crate::search::CompareOp::GreaterThan => ">",
crate::search::CompareOp::GreaterOrEqual => ">=",
crate::search::CompareOp::LessThan => "<",
crate::search::CompareOp::LessOrEqual => "<=",
};
let idx = *offset;
*offset += 1;
params.push(Box::new(*value));
Ok(format!("{col} {op_sql} ${idx}"))
}
SearchQuery::DateQuery { field, value } => {
let col = match field.as_str() {
"created" => "created_at",
"modified" | "updated" => "updated_at",
_ => return Ok("TRUE".to_string()),
};
Ok(date_value_to_postgres_expr(col, value))
}
}
}
/// Convert a DateValue to a PostgreSQL datetime comparison expression
fn date_value_to_postgres_expr(col: &str, value: &crate::search::DateValue) -> String {
use crate::search::DateValue;
match value {
DateValue::Today => format!("{col}::date = CURRENT_DATE"),
DateValue::Yesterday => format!("{col}::date = CURRENT_DATE - INTERVAL '1 day'"),
DateValue::ThisWeek => format!("{col} >= date_trunc('week', CURRENT_DATE)"),
DateValue::LastWeek => format!(
"{col} >= date_trunc('week', CURRENT_DATE) - INTERVAL '7 days' AND {col} < date_trunc('week', CURRENT_DATE)"
),
DateValue::ThisMonth => format!("{col} >= date_trunc('month', CURRENT_DATE)"),
DateValue::LastMonth => format!(
"{col} >= date_trunc('month', CURRENT_DATE) - INTERVAL '1 month' AND {col} < date_trunc('month', CURRENT_DATE)"
),
DateValue::ThisYear => format!("{col} >= date_trunc('year', CURRENT_DATE)"),
DateValue::LastYear => format!(
"{col} >= date_trunc('year', CURRENT_DATE) - INTERVAL '1 year' AND {col} < date_trunc('year', CURRENT_DATE)"
),
DateValue::DaysAgo(days) => format!("{col} >= CURRENT_DATE - INTERVAL '{days} days'"),
}
}
@ -478,7 +626,7 @@ impl StorageBackend for PostgresBackend {
.query_opt(
"SELECT id, path, file_name, media_type, content_hash, file_size,
title, artist, album, genre, year, duration_secs, description,
thumbnail_path, created_at, updated_at
thumbnail_path, file_mtime, created_at, updated_at
FROM media_items WHERE content_hash = $1",
&[&hash.0],
)
@ -494,6 +642,34 @@ impl StorageBackend for PostgresBackend {
}
}
async fn get_media_by_path(&self, path: &std::path::Path) -> Result<Option<MediaItem>> {
let path_str = path.to_string_lossy().to_string();
let client = self
.pool
.get()
.await
.map_err(|e| PinakesError::Database(format!("pool error: {e}")))?;
let row = client
.query_opt(
"SELECT id, path, file_name, media_type, content_hash, file_size,
title, artist, album, genre, year, duration_secs, description,
thumbnail_path, file_mtime, created_at, updated_at
FROM media_items WHERE path = $1",
&[&path_str],
)
.await?;
match row {
Some(r) => {
let mut item = row_to_media_item(&r)?;
item.custom_fields = self.get_custom_fields(item.id).await?;
Ok(Some(item))
}
None => Ok(None),
}
}
async fn list_media(&self, pagination: &Pagination) -> Result<Vec<MediaItem>> {
let client = self
.pool
@ -671,6 +847,59 @@ impl StorageBackend for PostgresBackend {
Ok(count as u64)
}
// ---- Batch Operations ----
async fn batch_delete_media(&self, ids: &[MediaId]) -> Result<u64> {
if ids.is_empty() {
return Ok(0);
}
let client = self
.pool
.get()
.await
.map_err(|e| PinakesError::Database(format!("pool error: {e}")))?;
// Use ANY with array for efficient batch delete
let uuids: Vec<Uuid> = ids.iter().map(|id| id.0).collect();
let rows = client
.execute("DELETE FROM media_items WHERE id = ANY($1)", &[&uuids])
.await?;
Ok(rows)
}
async fn batch_tag_media(&self, media_ids: &[MediaId], tag_ids: &[Uuid]) -> Result<u64> {
if media_ids.is_empty() || tag_ids.is_empty() {
return Ok(0);
}
let client = self
.pool
.get()
.await
.map_err(|e| PinakesError::Database(format!("pool error: {e}")))?;
// Use UNNEST for efficient batch insert
let mut media_uuids = Vec::new();
let mut tag_uuids = Vec::new();
for mid in media_ids {
for tid in tag_ids {
media_uuids.push(mid.0);
tag_uuids.push(*tid);
}
}
let rows = client
.execute(
"INSERT INTO media_tags (media_id, tag_id)
SELECT * FROM UNNEST($1::uuid[], $2::uuid[])
ON CONFLICT DO NOTHING",
&[&media_uuids, &tag_uuids],
)
.await?;
Ok(rows)
}
// ---- Tags ----
async fn create_tag(&self, name: &str, parent_id: Option<Uuid>) -> Result<Tag> {
@ -3155,6 +3384,9 @@ fn query_has_fts(query: &SearchQuery) -> bool {
SearchQuery::FieldMatch { .. } => false,
SearchQuery::TypeFilter(_) => false,
SearchQuery::TagFilter(_) => false,
SearchQuery::RangeQuery { .. } => false,
SearchQuery::CompareQuery { .. } => false,
SearchQuery::DateQuery { .. } => false,
SearchQuery::And(children) | SearchQuery::Or(children) => {
children.iter().any(query_has_fts)
}
@ -3173,7 +3405,7 @@ fn find_first_fts_param(query: &SearchQuery) -> i32 {
None
} else {
let idx = *offset;
*offset += 1;
*offset += 7; // FullText now uses 7 params (fts, prefix, ilike, sim_title, sim_artist, sim_album, sim_filename)
Some(idx)
}
}
@ -3183,7 +3415,7 @@ fn find_first_fts_param(query: &SearchQuery) -> i32 {
Some(idx)
}
SearchQuery::Fuzzy(_) => {
*offset += 2; // fuzzy uses two params
*offset += 5; // Fuzzy now uses 5 params (sim_title, sim_artist, sim_album, sim_filename, ilike)
None
}
SearchQuery::FieldMatch { .. } => {
@ -3191,6 +3423,21 @@ fn find_first_fts_param(query: &SearchQuery) -> i32 {
None
}
SearchQuery::TypeFilter(_) | SearchQuery::TagFilter(_) => None,
SearchQuery::RangeQuery { start, end, .. } => {
// Range queries use 0-2 params depending on bounds
if start.is_some() {
*offset += 1;
}
if end.is_some() {
*offset += 1;
}
None
}
SearchQuery::CompareQuery { .. } => {
*offset += 1;
None
}
SearchQuery::DateQuery { .. } => None, // No params, uses inline SQL
SearchQuery::And(children) | SearchQuery::Or(children) => {
for child in children {
if let Some(idx) = find_inner(child, offset) {
@ -3255,10 +3502,15 @@ mod tests {
let mut offset = 1;
let mut params: Vec<Box<dyn ToSql + Sync + Send>> = Vec::new();
let (clause, types, tags) = build_search_clause(&query, &mut offset, &mut params).unwrap();
assert_eq!(clause, "search_vector @@ plainto_tsquery('english', $1)");
// Fuzzy search combines FTS, prefix, ILIKE, and trigram similarity
assert!(clause.contains("plainto_tsquery"));
assert!(clause.contains("to_tsquery"));
assert!(clause.contains("LIKE"));
assert!(clause.contains("similarity"));
assert!(types.is_empty());
assert!(tags.is_empty());
assert_eq!(offset, 2);
// FullText now uses 7 parameters
assert_eq!(offset, 8);
}
#[test]

186
crates/pinakes-core/src/storage/sqlite.rs
View file

@ -111,6 +111,8 @@ fn row_to_media_item(row: &Row) -> rusqlite::Result<MediaItem> {
.get::<_, Option<String>>("thumbnail_path")?
.map(PathBuf::from),
custom_fields: HashMap::new(), // loaded separately
// file_mtime may not be present in all queries, so handle gracefully
file_mtime: row.get::<_, Option<i64>>("file_mtime").unwrap_or(None),
created_at: parse_datetime(&created_str),
updated_at: parse_datetime(&updated_str),
})
@ -312,18 +314,22 @@ fn load_custom_fields_batch(db: &Connection, items: &mut [MediaItem]) -> rusqlit
/// Translate a `SearchQuery` into components that can be assembled into SQL.
///
/// Returns `(fts_expr, where_clauses, join_clauses)` where:
/// Returns `(fts_expr, like_terms, where_clauses, join_clauses, params)` where:
/// - `fts_expr` is an FTS5 MATCH expression (may be empty),
/// - `like_terms` are search terms for LIKE fallback matching,
/// - `where_clauses` are extra WHERE predicates (e.g. type filters),
/// - `join_clauses` are extra JOIN snippets (e.g. tag filters).
/// - `params` are bind parameter values corresponding to `?` placeholders in
/// where_clauses and join_clauses.
fn search_query_to_fts(query: &SearchQuery) -> (String, Vec<String>, Vec<String>, Vec<String>) {
fn search_query_to_fts(
query: &SearchQuery,
) -> (String, Vec<String>, Vec<String>, Vec<String>, Vec<String>) {
let mut wheres = Vec::new();
let mut joins = Vec::new();
let mut params = Vec::new();
let fts = build_fts_expr(query, &mut wheres, &mut joins, &mut params);
(fts, wheres, joins, params)
let mut like_terms = Vec::new();
let fts = build_fts_expr(query, &mut wheres, &mut joins, &mut params, &mut like_terms);
(fts, like_terms, wheres, joins, params)
}
fn build_fts_expr(
@ -331,21 +337,35 @@ fn build_fts_expr(
wheres: &mut Vec<String>,
joins: &mut Vec<String>,
params: &mut Vec<String>,
like_terms: &mut Vec<String>,
) -> String {
match query {
SearchQuery::FullText(text) => {
if text.is_empty() {
String::new()
} else {
sanitize_fts_token(text)
// Collect term for LIKE fallback matching
like_terms.push(text.clone());
// Add implicit prefix matching for better partial matches
// This allows "mus" to match "music", "musician", etc.
let sanitized = sanitize_fts_token(text);
// If it's a single word, add prefix matching
if !sanitized.contains(' ') && !sanitized.contains('"') {
format!("{}*", sanitized)
} else {
// For phrases, use as-is but also add NEAR for proximity
sanitized
}
}
}
SearchQuery::Prefix(prefix) => {
like_terms.push(prefix.clone());
format!("{}*", sanitize_fts_token(prefix))
}
SearchQuery::Fuzzy(term) => {
// FTS5 does not natively support fuzzy; fall back to prefix match
// FTS5 does not natively support fuzzy; use prefix match
// as a best-effort approximation.
like_terms.push(term.clone());
format!("{}*", sanitize_fts_token(term))
}
SearchQuery::FieldMatch { field, value } => {
@ -355,7 +375,7 @@ fn build_fts_expr(
format!("{safe_field}:{safe_value}")
}
SearchQuery::Not(inner) => {
let inner_expr = build_fts_expr(inner, wheres, joins, params);
let inner_expr = build_fts_expr(inner, wheres, joins, params, like_terms);
if inner_expr.is_empty() {
String::new()
} else {
@ -365,7 +385,7 @@ fn build_fts_expr(
SearchQuery::And(terms) => {
let parts: Vec<String> = terms
.iter()
.map(|t| build_fts_expr(t, wheres, joins, params))
.map(|t| build_fts_expr(t, wheres, joins, params, like_terms))
.filter(|s| !s.is_empty())
.collect();
parts.join(" ")
@ -373,7 +393,7 @@ fn build_fts_expr(
SearchQuery::Or(terms) => {
let parts: Vec<String> = terms
.iter()
.map(|t| build_fts_expr(t, wheres, joins, params))
.map(|t| build_fts_expr(t, wheres, joins, params, like_terms))
.filter(|s| !s.is_empty())
.collect();
if parts.len() <= 1 {
@ -399,6 +419,82 @@ fn build_fts_expr(
params.push(tag_name.clone());
String::new()
}
SearchQuery::RangeQuery { field, start, end } => {
let col = match field.as_str() {
"year" => "m.year",
"size" | "file_size" => "m.file_size",
"duration" => "m.duration_secs",
_ => return String::new(), // Unknown field, ignore
};
match (start, end) {
(Some(s), Some(e)) => {
wheres.push(format!("{col} >= ? AND {col} <= ?"));
params.push(s.to_string());
params.push(e.to_string());
}
(Some(s), None) => {
wheres.push(format!("{col} >= ?"));
params.push(s.to_string());
}
(None, Some(e)) => {
wheres.push(format!("{col} <= ?"));
params.push(e.to_string());
}
(None, None) => {}
}
String::new()
}
SearchQuery::CompareQuery { field, op, value } => {
let col = match field.as_str() {
"year" => "m.year",
"size" | "file_size" => "m.file_size",
"duration" => "m.duration_secs",
_ => return String::new(), // Unknown field, ignore
};
let op_sql = match op {
crate::search::CompareOp::GreaterThan => ">",
crate::search::CompareOp::GreaterOrEqual => ">=",
crate::search::CompareOp::LessThan => "<",
crate::search::CompareOp::LessOrEqual => "<=",
};
wheres.push(format!("{col} {op_sql} ?"));
params.push(value.to_string());
String::new()
}
SearchQuery::DateQuery { field, value } => {
let col = match field.as_str() {
"created" => "m.created_at",
"modified" | "updated" => "m.updated_at",
_ => return String::new(),
};
let sql = date_value_to_sqlite_expr(col, value);
if !sql.is_empty() {
wheres.push(sql);
}
String::new()
}
}
}
/// Convert a DateValue to a SQLite datetime comparison expression
fn date_value_to_sqlite_expr(col: &str, value: &crate::search::DateValue) -> String {
use crate::search::DateValue;
match value {
DateValue::Today => format!("date({col}) = date('now')"),
DateValue::Yesterday => format!("date({col}) = date('now', '-1 day')"),
DateValue::ThisWeek => format!("{col} >= datetime('now', 'weekday 0', '-7 days')"),
DateValue::LastWeek => format!(
"{col} >= datetime('now', 'weekday 0', '-14 days') AND {col} < datetime('now', 'weekday 0', '-7 days')"
),
DateValue::ThisMonth => format!("{col} >= datetime('now', 'start of month')"),
DateValue::LastMonth => format!(
"{col} >= datetime('now', 'start of month', '-1 month') AND {col} < datetime('now', 'start of month')"
),
DateValue::ThisYear => format!("{col} >= datetime('now', 'start of year')"),
DateValue::LastYear => format!(
"{col} >= datetime('now', 'start of year', '-1 year') AND {col} < datetime('now', 'start of year')"
),
DateValue::DaysAgo(days) => format!("{col} >= datetime('now', '-{days} days')"),
}
}
@ -514,8 +610,8 @@ impl StorageBackend for SqliteBackend {
db.execute(
"INSERT INTO media_items (id, path, file_name, media_type, content_hash, \
file_size, title, artist, album, genre, year, duration_secs, description, \
thumbnail_path, created_at, updated_at) \
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
thumbnail_path, file_mtime, created_at, updated_at) \
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)",
params![
item.id.0.to_string(),
item.path.to_string_lossy().as_ref(),
@ -533,6 +629,7 @@ impl StorageBackend for SqliteBackend {
item.thumbnail_path
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
item.file_mtime,
item.created_at.to_rfc3339(),
item.updated_at.to_rfc3339(),
],
@ -566,7 +663,7 @@ impl StorageBackend for SqliteBackend {
let mut stmt = db.prepare(
"SELECT id, path, file_name, media_type, content_hash, file_size, \
title, artist, album, genre, year, duration_secs, description, \
thumbnail_path, created_at, updated_at FROM media_items WHERE id = ?1",
thumbnail_path, file_mtime, created_at, updated_at FROM media_items WHERE id = ?1",
)?;
let mut item = stmt
.query_row(params![id.0.to_string()], row_to_media_item)
@ -593,7 +690,7 @@ impl StorageBackend for SqliteBackend {
let mut stmt = db.prepare(
"SELECT id, path, file_name, media_type, content_hash, file_size, \
title, artist, album, genre, year, duration_secs, description, \
thumbnail_path, created_at, updated_at FROM media_items WHERE content_hash = ?1",
thumbnail_path, file_mtime, created_at, updated_at FROM media_items WHERE content_hash = ?1",
)?;
let result = stmt
.query_row(params![hash.0], row_to_media_item)
@ -609,6 +706,32 @@ impl StorageBackend for SqliteBackend {
.map_err(|e| PinakesError::Database(e.to_string()))?
}
async fn get_media_by_path(&self, path: &std::path::Path) -> Result<Option<MediaItem>> {
let path_str = path.to_string_lossy().to_string();
let conn = Arc::clone(&self.conn);
tokio::task::spawn_blocking(move || {
let db = conn
.lock()
.map_err(|e| PinakesError::Database(e.to_string()))?;
let mut stmt = db.prepare(
"SELECT id, path, file_name, media_type, content_hash, file_size, \
title, artist, album, genre, year, duration_secs, description, \
thumbnail_path, file_mtime, created_at, updated_at FROM media_items WHERE path = ?1",
)?;
let result = stmt
.query_row(params![path_str], row_to_media_item)
.optional()?;
if let Some(mut item) = result {
item.custom_fields = load_custom_fields_sync(&db, item.id)?;
Ok(Some(item))
} else {
Ok(None)
}
})
.await
.map_err(|e| PinakesError::Database(e.to_string()))?
}
async fn list_media(&self, pagination: &Pagination) -> Result<Vec<MediaItem>> {
let pagination = pagination.clone();
let conn = Arc::clone(&self.conn);
@ -630,7 +753,7 @@ impl StorageBackend for SqliteBackend {
let sql = format!(
"SELECT id, path, file_name, media_type, content_hash, file_size, \
title, artist, album, genre, year, duration_secs, description, \
thumbnail_path, created_at, updated_at FROM media_items \
thumbnail_path, file_mtime, created_at, updated_at FROM media_items \
ORDER BY {order_by} LIMIT ?1 OFFSET ?2"
);
let mut stmt = db.prepare(&sql)?;
@ -658,7 +781,7 @@ impl StorageBackend for SqliteBackend {
"UPDATE media_items SET path = ?2, file_name = ?3, media_type = ?4, \
content_hash = ?5, file_size = ?6, title = ?7, artist = ?8, album = ?9, \
genre = ?10, year = ?11, duration_secs = ?12, description = ?13, \
thumbnail_path = ?14, updated_at = ?15 WHERE id = ?1",
thumbnail_path = ?14, file_mtime = ?15, updated_at = ?16 WHERE id = ?1",
params![
item.id.0.to_string(),
item.path.to_string_lossy().as_ref(),
@ -676,6 +799,7 @@ impl StorageBackend for SqliteBackend {
item.thumbnail_path
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
item.file_mtime,
item.updated_at.to_rfc3339(),
],
)?;
@ -1067,7 +1191,7 @@ impl StorageBackend for SqliteBackend {
.lock()
.map_err(|e| PinakesError::Database(e.to_string()))?;
let (fts_expr, where_clauses, join_clauses, bind_params) =
let (fts_expr, _like_terms, where_clauses, join_clauses, bind_params) =
search_query_to_fts(&request.query);
let use_fts = !fts_expr.is_empty();
@ -1309,16 +1433,30 @@ impl StorageBackend for SqliteBackend {
}
async fn batch_delete_media(&self, ids: &[MediaId]) -> Result<u64> {
if ids.is_empty() {
return Ok(0);
}
let ids: Vec<String> = ids.iter().map(|id| id.0.to_string()).collect();
let conn = Arc::clone(&self.conn);
tokio::task::spawn_blocking(move || {
let db = conn
.lock()
.map_err(|e| PinakesError::Database(e.to_string()))?;
// Use IN clause for batch delete - much faster than individual deletes
// SQLite has a limit of ~500-1000 items in IN clause, so chunk if needed
const CHUNK_SIZE: usize = 500;
db.execute_batch("BEGIN IMMEDIATE")?;
let mut count = 0u64;
for id in &ids {
let rows = db.execute("DELETE FROM media_items WHERE id = ?1", params![id])?;
for chunk in ids.chunks(CHUNK_SIZE) {
let placeholders: Vec<String> =
(1..=chunk.len()).map(|i| format!("?{}", i)).collect();
let sql = format!(
"DELETE FROM media_items WHERE id IN ({})",
placeholders.join(", ")
);
let params: Vec<&dyn rusqlite::ToSql> =
chunk.iter().map(|s| s as &dyn rusqlite::ToSql).collect();
let rows = db.execute(&sql, params.as_slice())?;
count += rows as u64;
}
db.execute_batch("COMMIT")?;
@ -1329,6 +1467,9 @@ impl StorageBackend for SqliteBackend {
}
async fn batch_tag_media(&self, media_ids: &[MediaId], tag_ids: &[Uuid]) -> Result<u64> {
if media_ids.is_empty() || tag_ids.is_empty() {
return Ok(0);
}
let media_ids: Vec<String> = media_ids.iter().map(|id| id.0.to_string()).collect();
let tag_ids: Vec<String> = tag_ids.iter().map(|id| id.to_string()).collect();
let conn = Arc::clone(&self.conn);
@ -1337,13 +1478,14 @@ impl StorageBackend for SqliteBackend {
.lock()
.map_err(|e| PinakesError::Database(e.to_string()))?;
db.execute_batch("BEGIN IMMEDIATE")?;
// Prepare statement once for reuse
let mut stmt = db.prepare_cached(
"INSERT OR IGNORE INTO media_tags (media_id, tag_id) VALUES (?1, ?2)",
)?;
let mut count = 0u64;
for mid in &media_ids {
for tid in &tag_ids {
db.execute(
"INSERT OR IGNORE INTO media_tags (media_id, tag_id) VALUES (?1, ?2)",
params![mid, tid],
)?;
stmt.execute(params![mid, tid])?;
count += 1;
}
}