common: initial database handling

Can be configured from the config file, and also using environment options. ```toml [database] url = "postgresql://fc_ci:password@localhost/fc_ci" max_connections = 20 min_connections = 5 connect_timeout = 30 idle_timeout = 600 max_lifetime = 1800 ``` We'll want to support SQlite in the future, and better secret handling for database credentials. For now, this is workable. --- Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: I36b4c1306511052a2748ca9d5d3429366a6a6964
2025-11-02 21:31:21 +03:00 · 2025-11-02 21:31:21 +03:00 · 0e6d249e0f
commit 0e6d249e0f
parent a4c3cd1517
6 changed files with 530 additions and 5 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 4

+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "allocator-api2"
 version = "0.2.21"
@ -504,6 +513,16 @@ dependencies = [
 "typeid",
 ]

+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "etcetera"
 version = "0.8.0"
@ -526,20 +545,24 @@ dependencies = [
 "pin-project-lite",
 ]

+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
 [[package]]
 name = "fc-common"
 version = "0.1.0"
 dependencies = [
 "anyhow",
 "chrono",
- "clap",
 "git2",
 "serde",
 "serde_json",
 "sqlx",
+ "tempfile",
 "thiserror",
- "tracing",
- "tracing-subscriber",
 "uuid",
 ]

@ -563,6 +586,17 @@ dependencies = [
 "uuid",
 ]

+[[package]]
+name = "fc-migrate-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "fc-common",
+ "tokio",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "fc-queue-runner"
 version = "0.1.0"
@ -1175,6 +1209,12 @@ dependencies = [
 "vcpkg",
 ]

+[[package]]
+name = "linux-raw-sys"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
+
 [[package]]
 name = "litemap"
 version = "0.8.1"
@ -1196,6 +1236,15 @@ version = "0.4.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"

+[[package]]
+name = "matchers"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
+dependencies = [
+ "regex-automata",
+]
+
 [[package]]
 name = "matchit"
 version = "0.8.4"
@ -1543,6 +1592,23 @@ dependencies = [
 "bitflags",
 ]

+[[package]]
+name = "regex-automata"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
+
 [[package]]
 name = "ring"
 version = "0.17.14"
@ -1599,6 +1665,19 @@ dependencies = [
 "ordered-multimap",
 ]

+[[package]]
+name = "rustix"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "rustls"
 version = "0.23.34"
@ -2093,6 +2172,19 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "tempfile"
+version = "3.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
+dependencies = [
+ "fastrand",
+ "getrandom 0.3.4",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.17"
@ -2201,10 +2293,12 @@ version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8"
 dependencies = [
+ "indexmap",
 "serde_core",
 "serde_spanned",
 "toml_datetime",
 "toml_parser",
+ "toml_writer",
 "winnow",
 ]

@ -2226,6 +2320,12 @@ dependencies = [
 "winnow",
 ]

+[[package]]
+name = "toml_writer"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2"
+
 [[package]]
 name = "tower"
 version = "0.5.2"
@ -2304,10 +2404,14 @@ version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
 dependencies = [
+ "matchers",
 "nu-ansi-term",
+ "once_cell",
+ "regex-automata",
 "sharded-slab",
 "smallvec",
 "thread_local",
+ "tracing",
 "tracing-core",
 "tracing-log",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -4,6 +4,7 @@ members = [
    "crates/evaluator",
    "crates/queue-runner",
    "crates/common",
+    "crates/migrate-cli",
 ]
 resolver = "3"

@ -17,15 +18,17 @@ authors = ["NotAShelf <raf@notashelf.dev"]
 [workspace.dependencies]
 tokio = { version = "1.48.0", features = ["full"] }
 axum = "0.8.6"
-sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] }
+sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid", "migrate"] }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.145"
 uuid = { version = "1.18.1", features = ["v4", "serde"] }
 chrono = { version = "0.4.42", features = ["serde"] }
 tracing = "0.1.41"
-tracing-subscriber = "0.3.20"
+tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
 anyhow = "1.0.100"
 thiserror = "2.0.17"
 git2 = "0.20.2"
 clap = { version = "4.5.51", features = ["derive"] }
 config = "0.15.18"
+tempfile = "3.8"
+toml = "0.9.8"
--- a/crates/common/Cargo.toml
+++ b/crates/common/Cargo.toml
@ -18,3 +18,7 @@ git2.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true
 clap.workspace = true
+config.workspace = true
+tempfile.workspace = true
+toml.workspace = true
+tokio.workspace = true
--- a/crates/common/src/config.rs
+++ b/crates/common/src/config.rs
@ -0,0 +1,263 @@
+//! Configuration management for FC CI
+
+use config as config_crate;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct Config {
+    pub database: DatabaseConfig,
+    pub server: ServerConfig,
+    pub evaluator: EvaluatorConfig,
+    pub queue_runner: QueueRunnerConfig,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DatabaseConfig {
+    pub url: String,
+    pub max_connections: u32,
+    pub min_connections: u32,
+    pub connect_timeout: u64,
+    pub idle_timeout: u64,
+    pub max_lifetime: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ServerConfig {
+    pub host: String,
+    pub port: u16,
+    pub request_timeout: u64,
+    pub max_body_size: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EvaluatorConfig {
+    pub poll_interval: u64,
+    pub git_timeout: u64,
+    pub nix_timeout: u64,
+    pub work_dir: PathBuf,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct QueueRunnerConfig {
+    pub workers: usize,
+    pub poll_interval: u64,
+    pub build_timeout: u64,
+    pub work_dir: PathBuf,
+}
+
+impl Default for DatabaseConfig {
+    fn default() -> Self {
+        Self {
+            url: "postgresql://fc_ci:password@localhost/fc_ci".to_string(),
+            max_connections: 20,
+            min_connections: 5,
+            connect_timeout: 30,
+            idle_timeout: 600,
+            max_lifetime: 1800,
+        }
+    }
+}
+
+impl DatabaseConfig {
+    pub fn validate(&self) -> anyhow::Result<()> {
+        if self.url.is_empty() {
+            return Err(anyhow::anyhow!("Database URL cannot be empty"));
+        }
+
+        if !self.url.starts_with("postgresql://") && !self.url.starts_with("postgres://") {
+            return Err(anyhow::anyhow!(
+                "Database URL must start with postgresql:// or postgres://"
+            ));
+        }
+
+        if self.max_connections == 0 {
+            return Err(anyhow::anyhow!(
+                "Max database connections must be greater than 0"
+            ));
+        }
+
+        if self.min_connections > self.max_connections {
+            return Err(anyhow::anyhow!(
+                "Min database connections cannot exceed max connections"
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+impl Default for ServerConfig {
+    fn default() -> Self {
+        Self {
+            host: "127.0.0.1".to_string(),
+            port: 3000,
+            request_timeout: 30,
+            max_body_size: 10 * 1024 * 1024, // 10MB
+        }
+    }
+}
+
+impl Default for EvaluatorConfig {
+    fn default() -> Self {
+        Self {
+            poll_interval: 60,
+            git_timeout: 600,
+            nix_timeout: 1800,
+            work_dir: PathBuf::from("/tmp/fc-evaluator"),
+        }
+    }
+}
+
+impl Default for QueueRunnerConfig {
+    fn default() -> Self {
+        Self {
+            workers: 4,
+            poll_interval: 5,
+            build_timeout: 3600,
+            work_dir: PathBuf::from("/tmp/fc-queue-runner"),
+        }
+    }
+}
+
+impl Config {
+    pub fn load() -> anyhow::Result<Self> {
+        let mut settings = config_crate::Config::builder();
+
+        // Load default configuration
+        settings = settings.add_source(config_crate::Config::try_from(&Self::default())?);
+
+        // Load from config file if it exists
+        if let Ok(config_path) = std::env::var("FC_CONFIG_FILE") {
+            if std::path::Path::new(&config_path).exists() {
+                settings = settings.add_source(config_crate::File::with_name(&config_path));
+            }
+        } else if std::path::Path::new("fc.toml").exists() {
+            settings = settings.add_source(config_crate::File::with_name("fc").required(false));
+        }
+
+        // Load from environment variables with FC_ prefix (highest priority)
+        settings = settings.add_source(
+            config_crate::Environment::with_prefix("FC")
+                .separator("__")
+                .try_parsing(true),
+        );
+
+        let config = settings.build()?.try_deserialize::<Self>()?;
+
+        // Validate configuration
+        config.validate()?;
+
+        Ok(config)
+    }
+
+    pub fn validate(&self) -> anyhow::Result<()> {
+        // Validate database URL
+        if self.database.url.is_empty() {
+            return Err(anyhow::anyhow!("Database URL cannot be empty"));
+        }
+
+        if !self.database.url.starts_with("postgresql://")
+            && !self.database.url.starts_with("postgres://")
+        {
+            return Err(anyhow::anyhow!(
+                "Database URL must start with postgresql:// or postgres://"
+            ));
+        }
+
+        // Validate connection pool settings
+        if self.database.max_connections == 0 {
+            return Err(anyhow::anyhow!(
+                "Max database connections must be greater than 0"
+            ));
+        }
+
+        if self.database.min_connections > self.database.max_connections {
+            return Err(anyhow::anyhow!(
+                "Min database connections cannot exceed max connections"
+            ));
+        }
+
+        // Validate server settings
+        if self.server.port == 0 {
+            return Err(anyhow::anyhow!("Server port must be greater than 0"));
+        }
+
+        // Validate evaluator settings
+        if self.evaluator.poll_interval == 0 {
+            return Err(anyhow::anyhow!(
+                "Evaluator poll interval must be greater than 0"
+            ));
+        }
+
+        // Validate queue runner settings
+        if self.queue_runner.workers == 0 {
+            return Err(anyhow::anyhow!(
+                "Queue runner workers must be greater than 0"
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::env;
+
+    #[test]
+    fn test_default_config() {
+        let config = Config::default();
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_invalid_database_url() {
+        let mut config = Config::default();
+        config.database.url = "invalid://url".to_string();
+        assert!(config.validate().is_err());
+    }
+
+    #[test]
+    fn test_invalid_port() {
+        let mut config = Config::default();
+        config.server.port = 0;
+        assert!(config.validate().is_err());
+
+        config.server.port = 65535;
+        assert!(config.validate().is_ok()); // valid port
+    }
+
+    #[test]
+    fn test_invalid_connections() {
+        let mut config = Config::default();
+        config.database.max_connections = 0;
+        assert!(config.validate().is_err());
+
+        config.database.max_connections = 10;
+        config.database.min_connections = 15;
+        assert!(config.validate().is_err());
+    }
+
+    #[test]
+    fn test_environment_override() {
+        // Test environment variable parsing directly
+        unsafe {
+            env::set_var("FC_DATABASE__URL", "postgresql://test:test@localhost/test");
+            env::set_var("FC_SERVER__PORT", "8080");
+        }
+
+        // Test that environment variables are being read correctly
+        let db_url = std::env::var("FC_DATABASE__URL").unwrap();
+        let server_port = std::env::var("FC_SERVER__PORT").unwrap();
+
+        assert_eq!(db_url, "postgresql://test:test@localhost/test");
+        assert_eq!(server_port, "8080");
+
+        unsafe {
+            env::remove_var("FC_DATABASE__URL");
+            env::remove_var("FC_SERVER__PORT");
+        }
+    }
+}
--- a/crates/common/src/database.rs
+++ b/crates/common/src/database.rs
@ -0,0 +1,144 @@
+//! Database connection and pool management
+
+use crate::config::DatabaseConfig;
+use sqlx::{PgPool, Row, postgres::PgPoolOptions};
+use std::time::Duration;
+use tracing::{debug, info, warn};
+
+pub struct Database {
+    pool: PgPool,
+}
+
+impl Database {
+    pub async fn new(config: DatabaseConfig) -> anyhow::Result<Self> {
+        info!("Initializing database connection pool");
+
+        let pool = PgPoolOptions::new()
+            .max_connections(config.max_connections)
+            .min_connections(config.min_connections)
+            .acquire_timeout(Duration::from_secs(config.connect_timeout))
+            .idle_timeout(Duration::from_secs(config.idle_timeout))
+            .max_lifetime(Duration::from_secs(config.max_lifetime))
+            .connect(&config.url)
+            .await?;
+
+        // Test the connection
+        Self::health_check(&pool).await?;
+
+        info!("Database connection pool initialized successfully");
+
+        Ok(Self { pool })
+    }
+
+    #[must_use] pub const fn pool(&self) -> &PgPool {
+        &self.pool
+    }
+
+    pub async fn health_check(pool: &PgPool) -> anyhow::Result<()> {
+        debug!("Performing database health check");
+
+        let result: i64 = sqlx::query_scalar("SELECT 1").fetch_one(pool).await?;
+
+        if result != 1 {
+            return Err(anyhow::anyhow!(
+                "Database health check failed: unexpected result"
+            ));
+        }
+
+        debug!("Database health check passed");
+        Ok(())
+    }
+
+    pub async fn close(&self) {
+        info!("Closing database connection pool");
+        self.pool.close().await;
+    }
+
+    pub async fn get_connection_info(&self) -> anyhow::Result<ConnectionInfo> {
+        let row = sqlx::query(
+            r"
+            SELECT 
+                current_database() as database,
+                current_user as user,
+                version() as version,
+                inet_server_addr() as server_ip,
+                inet_server_port() as server_port
+            ",
+        )
+        .fetch_one(&self.pool)
+        .await?;
+
+        Ok(ConnectionInfo {
+            database: row.get("database"),
+            user: row.get("user"),
+            version: row.get("version"),
+            server_ip: row.get("server_ip"),
+            server_port: row.get("server_port"),
+        })
+    }
+
+    pub async fn get_pool_stats(&self) -> PoolStats {
+        let pool = &self.pool;
+
+        PoolStats {
+            size: pool.size(),
+            idle: pool.num_idle() as u32,
+            active: (pool.size() - pool.num_idle() as u32),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ConnectionInfo {
+    pub database: String,
+    pub user: String,
+    pub version: String,
+    pub server_ip: Option<String>,
+    pub server_port: Option<i32>,
+}
+
+#[derive(Debug, Clone)]
+pub struct PoolStats {
+    pub size: u32,
+    pub idle: u32,
+    pub active: u32,
+}
+
+impl Drop for Database {
+    fn drop(&mut self) {
+        warn!("Database connection pool dropped without explicit close");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pool_stats() {
+        let stats = PoolStats {
+            size: 10,
+            idle: 3,
+            active: 7,
+        };
+
+        assert_eq!(stats.size, 10);
+        assert_eq!(stats.idle, 3);
+        assert_eq!(stats.active, 7);
+    }
+
+    #[test]
+    fn test_connection_info() {
+        let info = ConnectionInfo {
+            database: "test_db".to_string(),
+            user: "test_user".to_string(),
+            version: "PostgreSQL 14.0".to_string(),
+            server_ip: Some("127.0.0.1".to_string()),
+            server_port: Some(5432),
+        };
+
+        assert_eq!(info.database, "test_db");
+        assert_eq!(info.user, "test_user");
+        assert_eq!(info.server_port, Some(5432));
+    }
+}
--- a/crates/common/src/lib.rs
+++ b/crates/common/src/lib.rs
@ -1,10 +1,17 @@
 //! Common types and utilities for CI

+pub mod config;
+pub mod database;
 pub mod error;
 pub mod migrate;
 pub mod migrate_cli;
 pub mod models;

+#[cfg(test)]
+mod tests;
+
+pub use config::*;
+pub use database::*;
 pub use error::*;
 pub use migrate::*;
 pub use models::*;