pinakes: import in parallel; various UI improvements

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I1eb47cd79cd4145c56af966f6756fe1d6a6a6964

crates/pinakes-core/src/scan.rs

@@ -14,9 +14,20 @@ pub struct ScanStatus {
     pub scanning: bool,
     pub files_found: usize,
     pub files_processed: usize,
+    /// Number of files skipped because they haven't changed (incremental scan)
+    pub files_skipped: usize,
     pub errors: Vec<String>,
 }
 
+/// Options for scanning operations
+#[derive(Debug, Clone, Default)]
+pub struct ScanOptions {
+    /// Use incremental scanning (skip unchanged files based on mtime)
+    pub incremental: bool,
+    /// Force full rescan even for incremental mode
+    pub force_full: bool,
+}
+
 /// Shared scan progress that can be read by the status endpoint while a scan runs.
 #[derive(Clone)]
 pub struct ScanProgress {
@@ -50,6 +61,7 @@ impl ScanProgress {
             scanning: self.is_scanning.load(Ordering::Acquire),
             files_found: self.files_found.load(Ordering::Acquire),
             files_processed: self.files_processed.load(Ordering::Acquire),
+            files_skipped: 0, // Not tracked in real-time progress
             errors,
         }
     }
@@ -89,7 +101,20 @@ pub async fn scan_directory(
     dir: &Path,
     ignore_patterns: &[String],
 ) -> Result<ScanStatus> {
-    scan_directory_with_progress(storage, dir, ignore_patterns, None).await
+    scan_directory_with_options(storage, dir, ignore_patterns, None, &ScanOptions::default()).await
+}
+
+/// Scan a directory with incremental scanning support
+pub async fn scan_directory_incremental(
+    storage: &DynStorageBackend,
+    dir: &Path,
+    ignore_patterns: &[String],
+) -> Result<ScanStatus> {
+    let options = ScanOptions {
+        incremental: true,
+        force_full: false,
+    };
+    scan_directory_with_options(storage, dir, ignore_patterns, None, &options).await
 }
 
 pub async fn scan_directory_with_progress(
@@ -98,20 +123,62 @@ pub async fn scan_directory_with_progress(
     ignore_patterns: &[String],
     progress: Option<&ScanProgress>,
 ) -> Result<ScanStatus> {
-    info!(dir = %dir.display(), "starting directory scan");
+    scan_directory_with_options(
+        storage,
+        dir,
+        ignore_patterns,
+        progress,
+        &ScanOptions::default(),
+    )
+    .await
+}
+
+/// Scan a directory with full options including progress tracking and incremental mode
+pub async fn scan_directory_with_options(
+    storage: &DynStorageBackend,
+    dir: &Path,
+    ignore_patterns: &[String],
+    progress: Option<&ScanProgress>,
+    scan_options: &ScanOptions,
+) -> Result<ScanStatus> {
+    info!(
+        dir = %dir.display(),
+        incremental = scan_options.incremental,
+        force = scan_options.force_full,
+        "starting directory scan"
+    );
 
     if let Some(p) = progress {
         p.begin();
     }
 
-    let results = import::import_directory(storage, dir, ignore_patterns).await?;
-    // Note: for configurable concurrency, use import_directory_with_concurrency directly
+    // Convert scan options to import options
+    let import_options = import::ImportOptions {
+        incremental: scan_options.incremental && !scan_options.force_full,
+        force: scan_options.force_full,
+    };
+
+    let results = import::import_directory_with_options(
+        storage,
+        dir,
+        ignore_patterns,
+        8, // Default concurrency
+        &import_options,
+    )
+    .await?;
 
     let mut errors = Vec::new();
     let mut processed = 0;
+    let mut skipped = 0;
     for result in &results {
         match result {
-            Ok(_) => processed += 1,
+            Ok(r) => {
+                if r.was_skipped {
+                    skipped += 1;
+                } else {
+                    processed += 1;
+                }
+            }
             Err(e) => {
                 let msg = e.to_string();
                 if let Some(p) = progress {
@@ -132,9 +199,20 @@ pub async fn scan_directory_with_progress(
         scanning: false,
         files_found: results.len(),
         files_processed: processed,
+        files_skipped: skipped,
         errors,
     };
 
+    if scan_options.incremental {
+        info!(
+            dir = %dir.display(),
+            found = status.files_found,
+            processed = status.files_processed,
+            skipped = status.files_skipped,
+            "incremental scan complete"
+        );
+    }
+
     Ok(status)
 }
 
@@ -142,19 +220,43 @@ pub async fn scan_all_roots(
     storage: &DynStorageBackend,
     ignore_patterns: &[String],
 ) -> Result<Vec<ScanStatus>> {
-    scan_all_roots_with_progress(storage, ignore_patterns, None).await
+    scan_all_roots_with_options(storage, ignore_patterns, None, &ScanOptions::default()).await
+}
+
+/// Scan all roots incrementally (skip unchanged files)
+pub async fn scan_all_roots_incremental(
+    storage: &DynStorageBackend,
+    ignore_patterns: &[String],
+) -> Result<Vec<ScanStatus>> {
+    let options = ScanOptions {
+        incremental: true,
+        force_full: false,
+    };
+    scan_all_roots_with_options(storage, ignore_patterns, None, &options).await
 }
 
 pub async fn scan_all_roots_with_progress(
     storage: &DynStorageBackend,
     ignore_patterns: &[String],
     progress: Option<&ScanProgress>,
+) -> Result<Vec<ScanStatus>> {
+    scan_all_roots_with_options(storage, ignore_patterns, progress, &ScanOptions::default()).await
+}
+
+/// Scan all roots with full options including progress and incremental mode
+pub async fn scan_all_roots_with_options(
+    storage: &DynStorageBackend,
+    ignore_patterns: &[String],
+    progress: Option<&ScanProgress>,
+    scan_options: &ScanOptions,
 ) -> Result<Vec<ScanStatus>> {
     let roots = storage.list_root_dirs().await?;
     let mut statuses = Vec::new();
 
     for root in roots {
-        match scan_directory_with_progress(storage, &root, ignore_patterns, progress).await {
+        match scan_directory_with_options(storage, &root, ignore_patterns, progress, scan_options)
+            .await
+        {
             Ok(status) => statuses.push(status),
             Err(e) => {
                 warn!(root = %root.display(), error = %e, "failed to scan root directory");
@@ -162,6 +264,7 @@ pub async fn scan_all_roots_with_progress(
                     scanning: false,
                     files_found: 0,
                     files_processed: 0,
+                    files_skipped: 0,
                     errors: vec![e.to_string()],
                 });
             }