NotAShelf/stash
1
0
Fork 0
mirror of https://github.com/NotAShelf/stash.git synced 2026-04-13 14:33:47 +00:00
Code Issues Projects Releases Packages Wiki Activity

db: switch to sqlite as the primary backend

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I6a6a69648f81d0d094e11a3e0f0a19d3b8eccd5d
This commit is contained in:
raf 2025-08-12 18:54:07 +03:00
commit 4f725425fc
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
12 changed files with 246 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/commands/decode.rs
View file

@ -1,4 +1,4 @@
use crate::db::{ClipboardDb, SledClipboardDb};
use crate::db::{ClipboardDb, SqliteClipboardDb};
use std::io::{Read, Write};
@ -13,7 +13,7 @@ pub trait DecodeCommand {
) -> Result<(), StashError>;
}
impl DecodeCommand for SledClipboardDb {
impl DecodeCommand for SqliteClipboardDb {
fn decode(
&self,
in_: impl Read,

4
src/commands/delete.rs
View file

@ -1,4 +1,4 @@
use crate::db::{ClipboardDb, SledClipboardDb, StashError};
use crate::db::{ClipboardDb, SqliteClipboardDb, StashError};
use std::io::Read;
@ -6,7 +6,7 @@ pub trait DeleteCommand {
fn delete(&self, input: impl Read) -> Result<usize, StashError>;
}
impl DeleteCommand for SledClipboardDb {
impl DeleteCommand for SqliteClipboardDb {
fn delete(&self, input: impl Read) -> Result<usize, StashError> {
match self.delete_entries(input) {
Ok(deleted) => {

4
src/commands/list.rs
View file

@ -1,11 +1,11 @@
use crate::db::{ClipboardDb, SledClipboardDb};
use crate::db::{ClipboardDb, SqliteClipboardDb};
use std::io::Write;
pub trait ListCommand {
fn list(&self, out: impl Write, preview_width: u32) -> Result<(), crate::db::StashError>;
}
impl ListCommand for SledClipboardDb {
impl ListCommand for SqliteClipboardDb {
fn list(&self, out: impl Write, preview_width: u32) -> Result<(), crate::db::StashError> {
self.list_entries(out, preview_width)?;
log::info!("Listed clipboard entries");

6
src/commands/query.rs
View file

@ -1,4 +1,4 @@
use crate::db::{ClipboardDb, SledClipboardDb};
use crate::db::{ClipboardDb, SqliteClipboardDb};
use crate::db::StashError;
@ -6,8 +6,8 @@ pub trait QueryCommand {
fn query_delete(&self, query: &str) -> Result<usize, StashError>;
}
impl QueryCommand for SledClipboardDb {
impl QueryCommand for SqliteClipboardDb {
fn query_delete(&self, query: &str) -> Result<usize, StashError> {
<SledClipboardDb as ClipboardDb>::delete_query(self, query)
<SqliteClipboardDb as ClipboardDb>::delete_query(self, query)
}
}

4
src/commands/store.rs
View file

@ -1,4 +1,4 @@
use crate::db::{ClipboardDb, SledClipboardDb};
use crate::db::{ClipboardDb, SqliteClipboardDb};
use std::io::Read;
@ -12,7 +12,7 @@ pub trait StoreCommand {
) -> Result<(), crate::db::StashError>;
}
impl StoreCommand for SledClipboardDb {
impl StoreCommand for SqliteClipboardDb {
fn store(
&self,
input: impl Read,

4
src/commands/wipe.rs
View file

@ -1,4 +1,4 @@
use crate::db::{ClipboardDb, SledClipboardDb};
use crate::db::{ClipboardDb, SqliteClipboardDb};
use crate::db::StashError;
@ -6,7 +6,7 @@ pub trait WipeCommand {
fn wipe(&self) -> Result<(), StashError>;
}
impl WipeCommand for SledClipboardDb {
impl WipeCommand for SqliteClipboardDb {
fn wipe(&self) -> Result<(), StashError> {
self.wipe_db()?;
log::info!("Database wiped");

277
src/db/mod.rs
View file

@ -4,9 +4,9 @@ use std::str;
use image::{GenericImageView, ImageFormat};
use log::{error, info};
use rmp_serde::{decode::from_read, encode::to_vec};
use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize};
use sled::{Db, IVec};
use thiserror::Error;
#[derive(Error, Debug)]
@ -15,8 +15,7 @@ pub enum StashError {
EmptyOrTooLarge,
#[error("Input is all whitespace, skipping store.")]
AllWhitespace,
#[error("Failed to serialize entry: {0}")]
Serialize(String),
#[error("Failed to store entry: {0}")]
Store(String),
#[error("Error reading entry during deduplication: {0}")]
@ -41,10 +40,7 @@ pub enum StashError {
DecodeExtractId(String),
#[error("Failed to get entry for decode: {0}")]
DecodeGet(String),
#[error("No entry found for id {0}")]
DecodeNoEntry(u64),
#[error("Failed to decode entry: {0}")]
DecodeDecode(String),
#[error("Failed to write decoded entry: {0}")]
DecodeWrite(String),
#[error("Failed to delete entry during query delete: {0}")]
@ -89,11 +85,25 @@ impl fmt::Display for Entry {
}
}
pub struct SledClipboardDb {
pub db: Db,
pub struct SqliteClipboardDb {
pub conn: Connection,
}
impl ClipboardDb for SledClipboardDb {
impl SqliteClipboardDb {
pub fn new(conn: Connection) -> Result<Self, StashError> {
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS clipboard (
id INTEGER PRIMARY KEY AUTOINCREMENT,
contents BLOB NOT NULL,
mime TEXT
);",
)
.map_err(|e| StashError::Store(e.to_string()))?;
Ok(Self { conn })
}
}
impl ClipboardDb for SqliteClipboardDb {
fn store_entry(
&self,
mut input: impl Read,
@ -112,98 +122,111 @@ impl ClipboardDb for SledClipboardDb {
self.deduplicate(&buf, max_dedupe_search)?;
let entry = Entry {
contents: buf.clone(),
mime,
};
let id = self.next_sequence();
let enc = to_vec(&entry).map_err(|e| StashError::Serialize(e.to_string()))?;
self.db
.insert(u64_to_ivec(id), enc)
self.conn
.execute(
"INSERT INTO clipboard (contents, mime) VALUES (?1, ?2)",
params![buf, mime],
)
.map_err(|e| StashError::Store(e.to_string()))?;
self.trim_db(max_items)?;
Ok(id)
Ok(self.next_sequence())
}
fn deduplicate(&self, buf: &[u8], max: u64) -> Result<usize, StashError> {
let mut count = 0;
let mut stmt = self
.conn
.prepare("SELECT id, contents FROM clipboard ORDER BY id DESC LIMIT ?1")
.map_err(|e| StashError::DeduplicationRead(e.to_string()))?;
let mut rows = stmt
.query(params![i64::try_from(max).unwrap_or(i64::MAX)])
.map_err(|e| StashError::DeduplicationRead(e.to_string()))?;
let mut deduped = 0;
for item in self
.db
.iter()
.rev()
.take(usize::try_from(max).unwrap_or(usize::MAX))
while let Some(row) = rows
.next()
.map_err(|e| StashError::DeduplicationRead(e.to_string()))?
{
let (k, v) = match item {
Ok((k, v)) => (k, v),
Err(e) => return Err(StashError::DeduplicationRead(e.to_string())),
};
let entry: Entry = match from_read(v.as_ref()) {
Ok(e) => e,
Err(e) => return Err(StashError::DeduplicationDecode(e.to_string())),
};
if entry.contents == buf {
self.db
.remove(k)
.map(|_| {
deduped += 1;
})
let id: u64 = row
.get(0)
.map_err(|e| StashError::DeduplicationDecode(e.to_string()))?;
let contents: Vec<u8> = row
.get(1)
.map_err(|e| StashError::DeduplicationDecode(e.to_string()))?;
if contents == buf {
self.conn
.execute("DELETE FROM clipboard WHERE id = ?1", params![id])
.map_err(|e| StashError::DeduplicationRemove(e.to_string()))?;
}
count += 1;
if count >= max {
break;
deduped += 1;
}
}
Ok(deduped)
}
fn trim_db(&self, max: u64) -> Result<(), StashError> {
let mut keys: Vec<_> = self
.db
.iter()
.rev()
.filter_map(|kv| match kv {
Ok((k, _)) => Some(k),
Err(_e) => None,
})
.collect();
if keys.len() as u64 > max {
for k in keys.drain(usize::try_from(max).unwrap_or(0)..) {
self.db
.remove(k)
.map_err(|e| StashError::Trim(e.to_string()))?;
}
let count: u64 = self
.conn
.query_row("SELECT COUNT(*) FROM clipboard", [], |row| row.get(0))
.map_err(|e| StashError::Trim(e.to_string()))?;
if count > max {
let to_delete = count - max;
self.conn.execute(
"DELETE FROM clipboard WHERE id IN (SELECT id FROM clipboard ORDER BY id ASC LIMIT ?1)",
params![i64::try_from(to_delete).unwrap_or(i64::MAX)],
).map_err(|e| StashError::Trim(e.to_string()))?;
}
Ok(())
}
fn delete_last(&self) -> Result<(), StashError> {
if let Some((k, _)) = self.db.iter().next_back().and_then(Result::ok) {
self.db
.remove(k)
.map(|_| ())
.map_err(|e| StashError::DeleteLast(e.to_string()))
let id: Option<u64> = self
.conn
.query_row(
"SELECT id FROM clipboard ORDER BY id DESC LIMIT 1",
[],
|row| row.get(0),
)
.optional()
.map_err(|e| StashError::DeleteLast(e.to_string()))?;
if let Some(id) = id {
self.conn
.execute("DELETE FROM clipboard WHERE id = ?1", params![id])
.map_err(|e| StashError::DeleteLast(e.to_string()))?;
Ok(())
} else {
Err(StashError::NoEntriesToDelete)
}
}
fn wipe_db(&self) -> Result<(), StashError> {
self.db.clear().map_err(|e| StashError::Wipe(e.to_string()))
self.conn
.execute("DELETE FROM clipboard", [])
.map_err(|e| StashError::Wipe(e.to_string()))?;
Ok(())
}
fn list_entries(&self, mut out: impl Write, preview_width: u32) -> Result<usize, StashError> {
let mut stmt = self
.conn
.prepare("SELECT id, contents, mime FROM clipboard ORDER BY id DESC")
.map_err(|e| StashError::ListDecode(e.to_string()))?;
let mut rows = stmt
.query([])
.map_err(|e| StashError::ListDecode(e.to_string()))?;
let mut listed = 0;
for (k, v) in self.db.iter().rev().filter_map(Result::ok) {
let id = ivec_to_u64(&k);
let entry: Entry = match from_read(v.as_ref()) {
Ok(e) => e,
Err(e) => return Err(StashError::ListDecode(e.to_string())),
};
let preview = preview_entry(&entry.contents, entry.mime.as_deref(), preview_width);
while let Some(row) = rows
.next()
.map_err(|e| StashError::ListDecode(e.to_string()))?
{
let id: u64 = row
.get(0)
.map_err(|e| StashError::ListDecode(e.to_string()))?;
let contents: Vec<u8> = row
.get(1)
.map_err(|e| StashError::ListDecode(e.to_string()))?;
let mime: Option<String> = row
.get(2)
.map_err(|e| StashError::ListDecode(e.to_string()))?;
let preview = preview_entry(&contents, mime.as_deref(), preview_width);
if writeln!(out, "{id}\t{preview}").is_ok() {
listed += 1;
}
@ -226,38 +249,44 @@ impl ClipboardDb for SledClipboardDb {
buf
};
let id = extract_id(&s).map_err(|e| StashError::DecodeExtractId(e.to_string()))?;
let v = self
.db
.get(u64_to_ivec(id))
.map_err(|e| StashError::DecodeGet(e.to_string()))?
.ok_or(StashError::DecodeNoEntry(id))?;
let entry: Entry =
from_read(v.as_ref()).map_err(|e| StashError::DecodeDecode(e.to_string()))?;
out.write_all(&entry.contents)
let (contents, _mime): (Vec<u8>, Option<String>) = self
.conn
.query_row(
"SELECT contents, mime FROM clipboard WHERE id = ?1",
params![id],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.map_err(|e| StashError::DecodeGet(e.to_string()))?;
out.write_all(&contents)
.map_err(|e| StashError::DecodeWrite(e.to_string()))?;
info!("Decoded entry with id {id}");
Ok(())
}
fn delete_query(&self, query: &str) -> Result<usize, StashError> {
let mut stmt = self
.conn
.prepare("SELECT id, contents FROM clipboard")
.map_err(|e| StashError::QueryDelete(e.to_string()))?;
let mut rows = stmt
.query([])
.map_err(|e| StashError::QueryDelete(e.to_string()))?;
let mut deleted = 0;
for (k, v) in self.db.iter().filter_map(Result::ok) {
let entry: Entry = match from_read(v.as_ref()) {
Ok(e) => e,
Err(_) => continue,
};
if entry
.contents
.windows(query.len())
.any(|w| w == query.as_bytes())
{
self.db
.remove(k)
.map(|_| {
deleted += 1;
})
while let Some(row) = rows
.next()
.map_err(|e| StashError::QueryDelete(e.to_string()))?
{
let id: u64 = row
.get(0)
.map_err(|e| StashError::QueryDelete(e.to_string()))?;
let contents: Vec<u8> = row
.get(1)
.map_err(|e| StashError::QueryDelete(e.to_string()))?;
if contents.windows(query.len()).any(|w| w == query.as_bytes()) {
self.conn
.execute("DELETE FROM clipboard WHERE id = ?1", params![id])
.map_err(|e| StashError::QueryDelete(e.to_string()))?;
deleted += 1;
}
}
Ok(deleted)
@ -268,25 +297,24 @@ impl ClipboardDb for SledClipboardDb {
let mut deleted = 0;
for line in reader.lines().map_while(Result::ok) {
if let Ok(id) = extract_id(&line) {
self.db
.remove(u64_to_ivec(id))
.map(|_| {
deleted += 1;
})
self.conn
.execute("DELETE FROM clipboard WHERE id = ?1", params![id])
.map_err(|e| StashError::DeleteEntry(id, e.to_string()))?;
deleted += 1;
}
}
Ok(deleted)
}
fn next_sequence(&self) -> u64 {
let last = self
.db
.iter()
.next_back()
.and_then(std::result::Result::ok)
.map(|(k, _)| ivec_to_u64(&k));
last.unwrap_or(0) + 1
match self
.conn
.query_row("SELECT MAX(id) FROM clipboard", [], |row| {
row.get::<_, Option<u64>>(0)
}) {
Ok(Some(max_id)) => max_id + 1,
Ok(None) | Err(_) => 1,
}
}
}
@ -296,20 +324,6 @@ pub fn extract_id(input: &str) -> Result<u64, &'static str> {
id_str.parse().map_err(|_| "invalid id")
}
pub fn u64_to_ivec(v: u64) -> IVec {
IVec::from(&v.to_be_bytes()[..])
}
pub fn ivec_to_u64(v: &IVec) -> u64 {
let arr: [u8; 8] = if let Ok(arr) = v.as_ref().try_into() {
arr
} else {
error!("Failed to convert IVec to u64: invalid length");
return 0;
};
u64::from_be_bytes(arr)
}
pub fn detect_mime(data: &[u8]) -> Option<String> {
if image::guess_format(data).is_ok() {
match image::guess_format(data) {
@ -347,7 +361,13 @@ pub fn preview_entry(data: &[u8], mime: Option<&str>, width: u32) -> String {
);
}
} else if mime == "application/json" || mime.starts_with("text/") {
let s = str::from_utf8(data).unwrap_or("");
let s = match str::from_utf8(data) {
Ok(s) => s,
Err(e) => {
error!("Failed to decode UTF-8 clipboard data: {e}");
""
}
};
let s = s.trim().replace(|c: char| c.is_whitespace(), " ");
return truncate(&s, width as usize, "…");
}
@ -366,7 +386,12 @@ pub fn truncate(s: &str, max: usize, ellip: &str) -> String {
pub fn size_str(size: usize) -> String {
let units = ["B", "KiB", "MiB"];
let mut fsize = f64::from(u32::try_from(size).unwrap_or(u32::MAX));
let mut fsize = if let Ok(val) = u32::try_from(size) {
f64::from(val)
} else {
error!("Clipboard entry size too large for display: {size}");
f64::from(u32::MAX)
};
let mut i = 0;
while fsize >= 1024.0 && i < units.len() - 1 {
fsize /= 1024.0;

24
src/import.rs
View file

@ -1,4 +1,4 @@
use crate::db::{Entry, SledClipboardDb, detect_mime, u64_to_ivec};
use crate::db::{Entry, SqliteClipboardDb, detect_mime};
use log::{error, info};
use std::io::{self, BufRead};
@ -6,31 +6,27 @@ pub trait ImportCommand {
fn import_tsv(&self, input: impl io::Read);
}
impl ImportCommand for SledClipboardDb {
impl ImportCommand for SqliteClipboardDb {
fn import_tsv(&self, input: impl io::Read) {
let reader = io::BufReader::new(input);
let mut imported = 0;
for line in reader.lines().map_while(Result::ok) {
let mut parts = line.splitn(2, '\t');
if let (Some(id_str), Some(val)) = (parts.next(), parts.next()) {
if let Ok(id) = id_str.parse::<u64>() {
if let Ok(_id) = id_str.parse::<u64>() {
let entry = Entry {
contents: val.as_bytes().to_vec(),
mime: detect_mime(val.as_bytes()),
};
let enc = match rmp_serde::encode::to_vec(&entry) {
Ok(enc) => enc,
Err(e) => {
error!("Failed to encode entry for id {id}: {e}");
continue;
}
};
match self.db.insert(u64_to_ivec(id), enc) {
match self.conn.execute(
"INSERT INTO clipboard (contents, mime) VALUES (?1, ?2)",
rusqlite::params![entry.contents, entry.mime],
) {
Ok(_) => {
imported += 1;
info!("Imported entry with id {id}");
info!("Imported entry from TSV");
}
Err(e) => error!("Failed to insert entry with id {id}: {e}"),
Err(e) => error!("Failed to insert entry: {e}"),
}
} else {
error!("Failed to parse id from line: {id_str}");
@ -39,6 +35,6 @@ impl ImportCommand for SledClipboardDb {
error!("Malformed TSV line: {line:?}");
}
}
info!("Imported {imported} records from TSV into sled database.");
info!("Imported {imported} records from TSV into SQLite database.");
}
}

14
src/main.rs
View file

@ -95,7 +95,7 @@ fn report_error<T>(result: Result<T, impl std::fmt::Display>, context: &str) ->
}
/// Watch clipboard and store changes
fn run_daemon(db: &db::SledClipboardDb, max_dedupe_search: u64, max_items: u64) {
fn run_daemon(db: &db::SqliteClipboardDb, max_dedupe_search: u64, max_items: u64) {
log::info!("Starting clipboard watch daemon (Wayland)");
let mut last_contents: Option<Vec<u8>> = None;
@ -149,12 +149,18 @@ fn main() {
.join("db")
});
let sled_db = sled::open(&db_path).unwrap_or_else(|e| {
log::error!("Failed to open database: {e}");
let conn = rusqlite::Connection::open(&db_path).unwrap_or_else(|e| {
log::error!("Failed to open SQLite database: {e}");
process::exit(1);
});
let db = db::SledClipboardDb { db: sled_db };
let db = match db::SqliteClipboardDb::new(conn) {
Ok(db) => db,
Err(e) => {
log::error!("Failed to initialize SQLite database: {e}");
process::exit(1);
}
};
match cli.command {
Some(Command::Store) => {