treewide: better cross-device sync capabilities; in-database storage

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: Id99798df6f7e4470caae8a193c2654aa6a6a6964

migrations/postgres/V15__managed_storage.sql

@@ -0,0 +1,30 @@
+-- V15: Managed File Storage
+-- Adds server-side content-addressable storage for uploaded files
+
+-- Add storage mode to media_items (external = file on disk, managed = in content-addressable storage)
+ALTER TABLE media_items ADD COLUMN storage_mode TEXT NOT NULL DEFAULT 'external';
+
+-- Original filename for managed uploads (preserved separately from file_name which may be normalized)
+ALTER TABLE media_items ADD COLUMN original_filename TEXT;
+
+-- When the file was uploaded to managed storage
+ALTER TABLE media_items ADD COLUMN uploaded_at TIMESTAMPTZ;
+
+-- Storage key for looking up the blob (usually same as content_hash for deduplication)
+ALTER TABLE media_items ADD COLUMN storage_key TEXT;
+
+-- Managed blobs table - tracks deduplicated file storage
+CREATE TABLE managed_blobs (
+    content_hash TEXT PRIMARY KEY NOT NULL,
+    file_size BIGINT NOT NULL,
+    mime_type TEXT NOT NULL,
+    reference_count INTEGER NOT NULL DEFAULT 1,
+    stored_at TIMESTAMPTZ NOT NULL,
+    last_verified TIMESTAMPTZ
+);
+
+-- Index for finding managed media items
+CREATE INDEX idx_media_storage_mode ON media_items(storage_mode);
+
+-- Index for finding orphaned blobs (reference_count = 0)
+CREATE INDEX idx_blobs_reference_count ON managed_blobs(reference_count);

migrations/postgres/V16__sync_system.sql

@@ -0,0 +1,103 @@
+-- V16: Cross-Device Sync System
+-- Adds device registration, change tracking, and chunked upload support
+
+-- Sync devices table
+CREATE TABLE sync_devices (
+    id TEXT PRIMARY KEY NOT NULL,
+    user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    name TEXT NOT NULL,
+    device_type TEXT NOT NULL,
+    client_version TEXT NOT NULL,
+    os_info TEXT,
+    device_token_hash TEXT NOT NULL UNIQUE,
+    last_sync_at TIMESTAMPTZ,
+    last_seen_at TIMESTAMPTZ NOT NULL,
+    sync_cursor BIGINT DEFAULT 0,
+    enabled BOOLEAN NOT NULL DEFAULT TRUE,
+    created_at TIMESTAMPTZ NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX idx_sync_devices_user ON sync_devices(user_id);
+CREATE INDEX idx_sync_devices_token ON sync_devices(device_token_hash);
+
+-- Sync log table - tracks all changes for sync
+CREATE TABLE sync_log (
+    id TEXT PRIMARY KEY NOT NULL,
+    sequence BIGSERIAL UNIQUE NOT NULL,
+    change_type TEXT NOT NULL,
+    media_id TEXT REFERENCES media_items(id) ON DELETE SET NULL,
+    path TEXT NOT NULL,
+    content_hash TEXT,
+    file_size BIGINT,
+    metadata_json TEXT,
+    changed_by_device TEXT REFERENCES sync_devices(id) ON DELETE SET NULL,
+    timestamp TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX idx_sync_log_sequence ON sync_log(sequence);
+CREATE INDEX idx_sync_log_path ON sync_log(path);
+CREATE INDEX idx_sync_log_timestamp ON sync_log(timestamp);
+
+-- Device sync state - tracks sync status per device per file
+CREATE TABLE device_sync_state (
+    device_id TEXT NOT NULL REFERENCES sync_devices(id) ON DELETE CASCADE,
+    path TEXT NOT NULL,
+    local_hash TEXT,
+    server_hash TEXT,
+    local_mtime BIGINT,
+    server_mtime BIGINT,
+    sync_status TEXT NOT NULL,
+    last_synced_at TIMESTAMPTZ,
+    conflict_info_json TEXT,
+    PRIMARY KEY (device_id, path)
+);
+
+CREATE INDEX idx_device_sync_status ON device_sync_state(device_id, sync_status);
+
+-- Upload sessions for chunked uploads
+CREATE TABLE upload_sessions (
+    id TEXT PRIMARY KEY NOT NULL,
+    device_id TEXT NOT NULL REFERENCES sync_devices(id) ON DELETE CASCADE,
+    target_path TEXT NOT NULL,
+    expected_hash TEXT NOT NULL,
+    expected_size BIGINT NOT NULL,
+    chunk_size BIGINT NOT NULL,
+    chunk_count BIGINT NOT NULL,
+    status TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL,
+    expires_at TIMESTAMPTZ NOT NULL,
+    last_activity TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX idx_upload_sessions_device ON upload_sessions(device_id);
+CREATE INDEX idx_upload_sessions_status ON upload_sessions(status);
+CREATE INDEX idx_upload_sessions_expires ON upload_sessions(expires_at);
+
+-- Upload chunks - tracks received chunks
+CREATE TABLE upload_chunks (
+    upload_id TEXT NOT NULL REFERENCES upload_sessions(id) ON DELETE CASCADE,
+    chunk_index BIGINT NOT NULL,
+    offset BIGINT NOT NULL,
+    size BIGINT NOT NULL,
+    hash TEXT NOT NULL,
+    received_at TIMESTAMPTZ NOT NULL,
+    PRIMARY KEY (upload_id, chunk_index)
+);
+
+-- Sync conflicts
+CREATE TABLE sync_conflicts (
+    id TEXT PRIMARY KEY NOT NULL,
+    device_id TEXT NOT NULL REFERENCES sync_devices(id) ON DELETE CASCADE,
+    path TEXT NOT NULL,
+    local_hash TEXT NOT NULL,
+    local_mtime BIGINT NOT NULL,
+    server_hash TEXT NOT NULL,
+    server_mtime BIGINT NOT NULL,
+    detected_at TIMESTAMPTZ NOT NULL,
+    resolved_at TIMESTAMPTZ,
+    resolution TEXT
+);
+
+CREATE INDEX idx_sync_conflicts_device ON sync_conflicts(device_id);
+CREATE INDEX idx_sync_conflicts_unresolved ON sync_conflicts(device_id) WHERE resolved_at IS NULL;

migrations/postgres/V17__enhanced_sharing.sql

@@ -0,0 +1,83 @@
+-- V17: Enhanced Sharing System
+-- Replaces simple share_links with comprehensive sharing capabilities
+
+-- Enhanced shares table
+CREATE TABLE shares (
+    id TEXT PRIMARY KEY NOT NULL,
+    target_type TEXT NOT NULL CHECK (target_type IN ('media', 'collection', 'tag', 'saved_search')),
+    target_id TEXT NOT NULL,
+    owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    recipient_type TEXT NOT NULL CHECK (recipient_type IN ('public_link', 'user', 'group', 'federated')),
+    recipient_user_id TEXT REFERENCES users(id) ON DELETE CASCADE,
+    recipient_group_id TEXT,
+    recipient_federated_handle TEXT,
+    recipient_federated_server TEXT,
+    public_token TEXT UNIQUE,
+    public_password_hash TEXT,
+    perm_view BOOLEAN NOT NULL DEFAULT TRUE,
+    perm_download BOOLEAN NOT NULL DEFAULT FALSE,
+    perm_edit BOOLEAN NOT NULL DEFAULT FALSE,
+    perm_delete BOOLEAN NOT NULL DEFAULT FALSE,
+    perm_reshare BOOLEAN NOT NULL DEFAULT FALSE,
+    perm_add BOOLEAN NOT NULL DEFAULT FALSE,
+    note TEXT,
+    expires_at TIMESTAMPTZ,
+    access_count BIGINT NOT NULL DEFAULT 0,
+    last_accessed TIMESTAMPTZ,
+    inherit_to_children BOOLEAN NOT NULL DEFAULT TRUE,
+    parent_share_id TEXT REFERENCES shares(id) ON DELETE CASCADE,
+    created_at TIMESTAMPTZ NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL,
+    UNIQUE(owner_id, target_type, target_id, recipient_type, recipient_user_id)
+);
+
+CREATE INDEX idx_shares_owner ON shares(owner_id);
+CREATE INDEX idx_shares_recipient_user ON shares(recipient_user_id);
+CREATE INDEX idx_shares_target ON shares(target_type, target_id);
+CREATE INDEX idx_shares_token ON shares(public_token);
+CREATE INDEX idx_shares_expires ON shares(expires_at);
+
+-- Share activity log
+CREATE TABLE share_activity (
+    id TEXT PRIMARY KEY NOT NULL,
+    share_id TEXT NOT NULL REFERENCES shares(id) ON DELETE CASCADE,
+    actor_id TEXT REFERENCES users(id) ON DELETE SET NULL,
+    actor_ip TEXT,
+    action TEXT NOT NULL,
+    details TEXT,
+    timestamp TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX idx_share_activity_share ON share_activity(share_id);
+CREATE INDEX idx_share_activity_timestamp ON share_activity(timestamp);
+
+-- Share notifications
+CREATE TABLE share_notifications (
+    id TEXT PRIMARY KEY NOT NULL,
+    user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    share_id TEXT NOT NULL REFERENCES shares(id) ON DELETE CASCADE,
+    notification_type TEXT NOT NULL,
+    is_read BOOLEAN NOT NULL DEFAULT FALSE,
+    created_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX idx_share_notifications_user ON share_notifications(user_id);
+CREATE INDEX idx_share_notifications_unread ON share_notifications(user_id) WHERE is_read = FALSE;
+
+-- Migrate existing share_links to new shares table
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'share_links') THEN
+        INSERT INTO shares (
+            id, target_type, target_id, owner_id, recipient_type,
+            public_token, public_password_hash, perm_view, perm_download,
+            access_count, expires_at, created_at, updated_at
+        )
+        SELECT
+            id, 'media', media_id, created_by, 'public_link',
+            token, password_hash, TRUE, TRUE,
+            view_count, expires_at, created_at, created_at
+        FROM share_links
+        ON CONFLICT DO NOTHING;
+    END IF;
+END $$;

migrations/sqlite/V15__managed_storage.sql

@@ -0,0 +1,30 @@
+-- V15: Managed File Storage
+-- Adds server-side content-addressable storage for uploaded files
+
+-- Add storage mode to media_items (external = file on disk, managed = in content-addressable storage)
+ALTER TABLE media_items ADD COLUMN storage_mode TEXT NOT NULL DEFAULT 'external';
+
+-- Original filename for managed uploads (preserved separately from file_name which may be normalized)
+ALTER TABLE media_items ADD COLUMN original_filename TEXT;
+
+-- When the file was uploaded to managed storage
+ALTER TABLE media_items ADD COLUMN uploaded_at TEXT;
+
+-- Storage key for looking up the blob (usually same as content_hash for deduplication)
+ALTER TABLE media_items ADD COLUMN storage_key TEXT;
+
+-- Managed blobs table - tracks deduplicated file storage
+CREATE TABLE managed_blobs (
+    content_hash TEXT PRIMARY KEY NOT NULL,
+    file_size INTEGER NOT NULL,
+    mime_type TEXT NOT NULL,
+    reference_count INTEGER NOT NULL DEFAULT 1,
+    stored_at TEXT NOT NULL,
+    last_verified TEXT
+);
+
+-- Index for finding managed media items
+CREATE INDEX idx_media_storage_mode ON media_items(storage_mode);
+
+-- Index for finding orphaned blobs (reference_count = 0)
+CREATE INDEX idx_blobs_reference_count ON managed_blobs(reference_count);

migrations/sqlite/V16__sync_system.sql

@@ -0,0 +1,117 @@
+-- V16: Cross-Device Sync System
+-- Adds device registration, change tracking, and chunked upload support
+
+-- Sync devices table
+CREATE TABLE sync_devices (
+    id TEXT PRIMARY KEY NOT NULL,
+    user_id TEXT NOT NULL,
+    name TEXT NOT NULL,
+    device_type TEXT NOT NULL,
+    client_version TEXT NOT NULL,
+    os_info TEXT,
+    device_token_hash TEXT NOT NULL UNIQUE,
+    last_sync_at TEXT,
+    last_seen_at TEXT NOT NULL,
+    sync_cursor INTEGER DEFAULT 0,
+    enabled INTEGER NOT NULL DEFAULT 1,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_sync_devices_user ON sync_devices(user_id);
+CREATE INDEX idx_sync_devices_token ON sync_devices(device_token_hash);
+
+-- Sync log table - tracks all changes for sync
+CREATE TABLE sync_log (
+    id TEXT PRIMARY KEY NOT NULL,
+    sequence INTEGER NOT NULL UNIQUE,
+    change_type TEXT NOT NULL,
+    media_id TEXT,
+    path TEXT NOT NULL,
+    content_hash TEXT,
+    file_size INTEGER,
+    metadata_json TEXT,
+    changed_by_device TEXT,
+    timestamp TEXT NOT NULL,
+    FOREIGN KEY (media_id) REFERENCES media_items(id) ON DELETE SET NULL,
+    FOREIGN KEY (changed_by_device) REFERENCES sync_devices(id) ON DELETE SET NULL
+);
+
+CREATE INDEX idx_sync_log_sequence ON sync_log(sequence);
+CREATE INDEX idx_sync_log_path ON sync_log(path);
+CREATE INDEX idx_sync_log_timestamp ON sync_log(timestamp);
+
+-- Sequence counter for sync log
+CREATE TABLE sync_sequence (
+    id INTEGER PRIMARY KEY CHECK (id = 1),
+    current_value INTEGER NOT NULL DEFAULT 0
+);
+INSERT INTO sync_sequence (id, current_value) VALUES (1, 0);
+
+-- Device sync state - tracks sync status per device per file
+CREATE TABLE device_sync_state (
+    device_id TEXT NOT NULL,
+    path TEXT NOT NULL,
+    local_hash TEXT,
+    server_hash TEXT,
+    local_mtime INTEGER,
+    server_mtime INTEGER,
+    sync_status TEXT NOT NULL,
+    last_synced_at TEXT,
+    conflict_info_json TEXT,
+    PRIMARY KEY (device_id, path),
+    FOREIGN KEY (device_id) REFERENCES sync_devices(id) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_device_sync_status ON device_sync_state(device_id, sync_status);
+
+-- Upload sessions for chunked uploads
+CREATE TABLE upload_sessions (
+    id TEXT PRIMARY KEY NOT NULL,
+    device_id TEXT NOT NULL,
+    target_path TEXT NOT NULL,
+    expected_hash TEXT NOT NULL,
+    expected_size INTEGER NOT NULL,
+    chunk_size INTEGER NOT NULL,
+    chunk_count INTEGER NOT NULL,
+    status TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    expires_at TEXT NOT NULL,
+    last_activity TEXT NOT NULL,
+    FOREIGN KEY (device_id) REFERENCES sync_devices(id) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_upload_sessions_device ON upload_sessions(device_id);
+CREATE INDEX idx_upload_sessions_status ON upload_sessions(status);
+CREATE INDEX idx_upload_sessions_expires ON upload_sessions(expires_at);
+
+-- Upload chunks - tracks received chunks
+CREATE TABLE upload_chunks (
+    upload_id TEXT NOT NULL,
+    chunk_index INTEGER NOT NULL,
+    offset INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    hash TEXT NOT NULL,
+    received_at TEXT NOT NULL,
+    PRIMARY KEY (upload_id, chunk_index),
+    FOREIGN KEY (upload_id) REFERENCES upload_sessions(id) ON DELETE CASCADE
+);
+
+-- Sync conflicts
+CREATE TABLE sync_conflicts (
+    id TEXT PRIMARY KEY NOT NULL,
+    device_id TEXT NOT NULL,
+    path TEXT NOT NULL,
+    local_hash TEXT NOT NULL,
+    local_mtime INTEGER NOT NULL,
+    server_hash TEXT NOT NULL,
+    server_mtime INTEGER NOT NULL,
+    detected_at TEXT NOT NULL,
+    resolved_at TEXT,
+    resolution TEXT,
+    FOREIGN KEY (device_id) REFERENCES sync_devices(id) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_sync_conflicts_device ON sync_conflicts(device_id);
+CREATE INDEX idx_sync_conflicts_unresolved ON sync_conflicts(device_id, resolved_at) WHERE resolved_at IS NULL;

migrations/sqlite/V17__enhanced_sharing.sql

@@ -0,0 +1,85 @@
+-- V17: Enhanced Sharing System
+-- Replaces simple share_links with comprehensive sharing capabilities
+
+-- Enhanced shares table
+CREATE TABLE shares (
+    id TEXT PRIMARY KEY NOT NULL,
+    target_type TEXT NOT NULL CHECK (target_type IN ('media', 'collection', 'tag', 'saved_search')),
+    target_id TEXT NOT NULL,
+    owner_id TEXT NOT NULL,
+    recipient_type TEXT NOT NULL CHECK (recipient_type IN ('public_link', 'user', 'group', 'federated')),
+    recipient_user_id TEXT,
+    recipient_group_id TEXT,
+    recipient_federated_handle TEXT,
+    recipient_federated_server TEXT,
+    public_token TEXT UNIQUE,
+    public_password_hash TEXT,
+    perm_view INTEGER NOT NULL DEFAULT 1,
+    perm_download INTEGER NOT NULL DEFAULT 0,
+    perm_edit INTEGER NOT NULL DEFAULT 0,
+    perm_delete INTEGER NOT NULL DEFAULT 0,
+    perm_reshare INTEGER NOT NULL DEFAULT 0,
+    perm_add INTEGER NOT NULL DEFAULT 0,
+    note TEXT,
+    expires_at TEXT,
+    access_count INTEGER NOT NULL DEFAULT 0,
+    last_accessed TEXT,
+    inherit_to_children INTEGER NOT NULL DEFAULT 1,
+    parent_share_id TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (recipient_user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (parent_share_id) REFERENCES shares(id) ON DELETE CASCADE,
+    UNIQUE(owner_id, target_type, target_id, recipient_type, recipient_user_id)
+);
+
+CREATE INDEX idx_shares_owner ON shares(owner_id);
+CREATE INDEX idx_shares_recipient_user ON shares(recipient_user_id);
+CREATE INDEX idx_shares_target ON shares(target_type, target_id);
+CREATE INDEX idx_shares_token ON shares(public_token);
+CREATE INDEX idx_shares_expires ON shares(expires_at);
+
+-- Share activity log
+CREATE TABLE share_activity (
+    id TEXT PRIMARY KEY NOT NULL,
+    share_id TEXT NOT NULL,
+    actor_id TEXT,
+    actor_ip TEXT,
+    action TEXT NOT NULL,
+    details TEXT,
+    timestamp TEXT NOT NULL,
+    FOREIGN KEY (share_id) REFERENCES shares(id) ON DELETE CASCADE,
+    FOREIGN KEY (actor_id) REFERENCES users(id) ON DELETE SET NULL
+);
+
+CREATE INDEX idx_share_activity_share ON share_activity(share_id);
+CREATE INDEX idx_share_activity_timestamp ON share_activity(timestamp);
+
+-- Share notifications
+CREATE TABLE share_notifications (
+    id TEXT PRIMARY KEY NOT NULL,
+    user_id TEXT NOT NULL,
+    share_id TEXT NOT NULL,
+    notification_type TEXT NOT NULL,
+    is_read INTEGER NOT NULL DEFAULT 0,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (share_id) REFERENCES shares(id) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_share_notifications_user ON share_notifications(user_id);
+CREATE INDEX idx_share_notifications_unread ON share_notifications(user_id, is_read) WHERE is_read = 0;
+
+-- Migrate existing share_links to new shares table (if share_links exists)
+INSERT OR IGNORE INTO shares (
+    id, target_type, target_id, owner_id, recipient_type,
+    public_token, public_password_hash, perm_view, perm_download,
+    access_count, expires_at, created_at, updated_at
+)
+SELECT
+    id, 'media', media_id, created_by, 'public_link',
+    token, password_hash, 1, 1,
+    view_count, expires_at, created_at, created_at
+FROM share_links
+WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type='table' AND name='share_links');