cache: add SQLite route persistence; initial TTL and LRU eviction implementation
Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: I0370d6c114d5490634905c1a831a31526a6a6964
This commit is contained in:
parent
9f264fbef1
commit
663f9995b2
GPG key ID:
29D95B64378DB4BF
8 changed files with 674 additions and 5 deletions
168
internal/cache/db.go
vendored
168
internal/cache/db.go
vendored
|
|
@ -1 +1,169 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Core routing decision persisted per store path.
|
||||
type RouteEntry struct {
|
||||
StorePath string
|
||||
UpstreamURL string
|
||||
LatencyMs float64
|
||||
LatencyEMA float64
|
||||
LastVerified time.Time
|
||||
QueryCount uint32
|
||||
FailureCount uint32
|
||||
TTL time.Time
|
||||
NarHash string
|
||||
NarSize uint64
|
||||
}
|
||||
|
||||
// Returns true if the entry exists and hasn't expired.
|
||||
func (r *RouteEntry) IsValid() bool {
|
||||
return r != nil && time.Now().Before(r.TTL)
|
||||
}
|
||||
|
||||
// SQLite-backed store for route persistence.
|
||||
type DB struct {
|
||||
db *sql.DB
|
||||
maxEntries int
|
||||
}
|
||||
|
||||
// Opens or creates the SQLite database at path with WAL mode.
|
||||
func Open(path string, maxEntries int) (*DB, error) {
|
||||
db, err := sql.Open("sqlite", path+"?_journal=WAL&_busy_timeout=5000")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1) // SQLite WAL allows 1 writer
|
||||
|
||||
if err := migrate(db); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("migrate: %w", err)
|
||||
}
|
||||
|
||||
return &DB{db: db, maxEntries: maxEntries}, nil
|
||||
}
|
||||
|
||||
// Closes the database.
|
||||
func (d *DB) Close() error {
|
||||
return d.db.Close()
|
||||
}
|
||||
|
||||
func migrate(db *sql.DB) error {
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS routes (
|
||||
store_path TEXT PRIMARY KEY,
|
||||
upstream_url TEXT NOT NULL,
|
||||
latency_ms REAL DEFAULT 0,
|
||||
latency_ema REAL DEFAULT 0,
|
||||
query_count INTEGER DEFAULT 1,
|
||||
failure_count INTEGER DEFAULT 0,
|
||||
last_verified INTEGER DEFAULT 0,
|
||||
ttl INTEGER NOT NULL,
|
||||
nar_hash TEXT DEFAULT '',
|
||||
nar_size INTEGER DEFAULT 0,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_routes_ttl ON routes(ttl);
|
||||
CREATE INDEX IF NOT EXISTS idx_routes_last_verified ON routes(last_verified);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS upstream_health (
|
||||
url TEXT PRIMARY KEY,
|
||||
ema_latency REAL DEFAULT 0,
|
||||
last_probe INTEGER DEFAULT 0,
|
||||
consecutive_fails INTEGER DEFAULT 0,
|
||||
total_queries INTEGER DEFAULT 0,
|
||||
success_rate REAL DEFAULT 1.0
|
||||
);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// Returns the route for storePath, or nil if not found.
|
||||
func (d *DB) GetRoute(storePath string) (*RouteEntry, error) {
|
||||
row := d.db.QueryRow(`
|
||||
SELECT store_path, upstream_url, latency_ms, latency_ema,
|
||||
query_count, failure_count, last_verified, ttl, nar_hash, nar_size
|
||||
FROM routes WHERE store_path = ?`, storePath)
|
||||
|
||||
var e RouteEntry
|
||||
var lastVerifiedUnix, ttlUnix int64
|
||||
err := row.Scan(
|
||||
&e.StorePath, &e.UpstreamURL, &e.LatencyMs, &e.LatencyEMA,
|
||||
&e.QueryCount, &e.FailureCount, &lastVerifiedUnix, &ttlUnix,
|
||||
&e.NarHash, &e.NarSize,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.LastVerified = time.Unix(lastVerifiedUnix, 0).UTC()
|
||||
e.TTL = time.Unix(ttlUnix, 0).UTC()
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
// Inserts or updates a route entry.
|
||||
func (d *DB) SetRoute(entry *RouteEntry) error {
|
||||
_, err := d.db.Exec(`
|
||||
INSERT INTO routes
|
||||
(store_path, upstream_url, latency_ms, latency_ema,
|
||||
query_count, failure_count, last_verified, ttl, nar_hash, nar_size)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(store_path) DO UPDATE SET
|
||||
upstream_url = excluded.upstream_url,
|
||||
latency_ms = excluded.latency_ms,
|
||||
latency_ema = excluded.latency_ema,
|
||||
query_count = excluded.query_count,
|
||||
failure_count = excluded.failure_count,
|
||||
last_verified = excluded.last_verified,
|
||||
ttl = excluded.ttl,
|
||||
nar_hash = excluded.nar_hash,
|
||||
nar_size = excluded.nar_size`,
|
||||
entry.StorePath, entry.UpstreamURL,
|
||||
entry.LatencyMs, entry.LatencyEMA,
|
||||
entry.QueryCount, entry.FailureCount,
|
||||
entry.LastVerified.Unix(), entry.TTL.Unix(),
|
||||
entry.NarHash, entry.NarSize,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.evictIfNeeded()
|
||||
}
|
||||
|
||||
// Deletes routes whose TTL has passed.
|
||||
func (d *DB) ExpireOldRoutes() error {
|
||||
_, err := d.db.Exec(`DELETE FROM routes WHERE ttl < ?`, time.Now().Unix())
|
||||
return err
|
||||
}
|
||||
|
||||
// Returns the total number of stored routes.
|
||||
func (d *DB) RouteCount() (int, error) {
|
||||
var count int
|
||||
err := d.db.QueryRow(`SELECT COUNT(*) FROM routes`).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// Deletes the oldest routes (by last_verified) when over capacity.
|
||||
func (d *DB) evictIfNeeded() error {
|
||||
count, err := d.RouteCount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count <= d.maxEntries {
|
||||
return nil
|
||||
}
|
||||
excess := count - d.maxEntries
|
||||
_, err = d.db.Exec(`
|
||||
DELETE FROM routes WHERE store_path IN (
|
||||
SELECT store_path FROM routes ORDER BY last_verified ASC LIMIT ?
|
||||
)`, excess)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
157
internal/cache/db_test.go
vendored
Normal file
157
internal/cache/db_test.go
vendored
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
package cache_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"notashelf.dev/ncro/internal/cache"
|
||||
)
|
||||
|
||||
func newTestDB(t *testing.T) *cache.DB {
|
||||
t.Helper()
|
||||
f, err := os.CreateTemp("", "ncro-test-*.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
t.Cleanup(func() { os.Remove(f.Name()) })
|
||||
|
||||
db, err := cache.Open(f.Name(), 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestGetSetRoute(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
|
||||
entry := &cache.RouteEntry{
|
||||
StorePath: "abc123xyz-hello-2.12",
|
||||
UpstreamURL: "https://cache.nixos.org",
|
||||
LatencyMs: 12.5,
|
||||
LatencyEMA: 12.5,
|
||||
LastVerified: time.Now().UTC().Truncate(time.Second),
|
||||
QueryCount: 1,
|
||||
TTL: time.Now().Add(time.Hour).UTC().Truncate(time.Second),
|
||||
}
|
||||
|
||||
if err := db.SetRoute(entry); err != nil {
|
||||
t.Fatalf("SetRoute: %v", err)
|
||||
}
|
||||
|
||||
got, err := db.GetRoute("abc123xyz-hello-2.12")
|
||||
if err != nil {
|
||||
t.Fatalf("GetRoute: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("GetRoute returned nil")
|
||||
}
|
||||
if got.UpstreamURL != entry.UpstreamURL {
|
||||
t.Errorf("upstream = %q, want %q", got.UpstreamURL, entry.UpstreamURL)
|
||||
}
|
||||
if got.QueryCount != 1 {
|
||||
t.Errorf("query_count = %d, want 1", got.QueryCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRouteNotFound(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
got, err := db.GetRoute("nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected nil, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRouteUpsert(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
|
||||
entry := &cache.RouteEntry{
|
||||
StorePath: "abc123-pkg",
|
||||
UpstreamURL: "https://cache.nixos.org",
|
||||
LatencyMs: 20.0,
|
||||
LatencyEMA: 20.0,
|
||||
QueryCount: 1,
|
||||
TTL: time.Now().Add(time.Hour),
|
||||
}
|
||||
db.SetRoute(entry)
|
||||
|
||||
entry.LatencyEMA = 18.0
|
||||
entry.QueryCount = 2
|
||||
if err := db.SetRoute(entry); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
got, _ := db.GetRoute("abc123-pkg")
|
||||
if got.LatencyEMA != 18.0 {
|
||||
t.Errorf("ema = %f, want 18.0", got.LatencyEMA)
|
||||
}
|
||||
if got.QueryCount != 2 {
|
||||
t.Errorf("query_count = %d, want 2", got.QueryCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpireOldRoutes(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
|
||||
// Insert expired route
|
||||
expired := &cache.RouteEntry{
|
||||
StorePath: "expired-pkg",
|
||||
UpstreamURL: "https://cache.nixos.org",
|
||||
TTL: time.Now().Add(-time.Minute), // already expired
|
||||
}
|
||||
db.SetRoute(expired)
|
||||
|
||||
// Insert valid route
|
||||
valid := &cache.RouteEntry{
|
||||
StorePath: "valid-pkg",
|
||||
UpstreamURL: "https://cache.nixos.org",
|
||||
TTL: time.Now().Add(time.Hour),
|
||||
}
|
||||
db.SetRoute(valid)
|
||||
|
||||
if err := db.ExpireOldRoutes(); err != nil {
|
||||
t.Fatalf("ExpireOldRoutes: %v", err)
|
||||
}
|
||||
|
||||
got, _ := db.GetRoute("expired-pkg")
|
||||
if got != nil {
|
||||
t.Error("expired route should have been deleted")
|
||||
}
|
||||
got2, _ := db.GetRoute("valid-pkg")
|
||||
if got2 == nil {
|
||||
t.Error("valid route should still exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLRUEviction(t *testing.T) {
|
||||
// Use maxEntries=3 to trigger eviction easily
|
||||
f, _ := os.CreateTemp("", "ncro-lru-*.db")
|
||||
f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
db, _ := cache.Open(f.Name(), 3)
|
||||
defer db.Close()
|
||||
|
||||
for i := range 4 {
|
||||
db.SetRoute(&cache.RouteEntry{
|
||||
StorePath: "pkg-" + string(rune('a'+i)),
|
||||
UpstreamURL: "https://cache.nixos.org",
|
||||
LastVerified: time.Now().Add(time.Duration(i) * time.Second),
|
||||
TTL: time.Now().Add(time.Hour),
|
||||
})
|
||||
}
|
||||
|
||||
count, err := db.RouteCount()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count > 3 {
|
||||
t.Errorf("expected count <= 3 after LRU eviction, got %d", count)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue