From 0e6d249e0fd9c5852e339e871fc72c17b2a7ecf8 Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Sun, 2 Nov 2025 21:31:21 +0300 Subject: [PATCH] common: initial database handling Can be configured from the config file, and also using environment options. ```toml [database] url = "postgresql://fc_ci:password@localhost/fc_ci" max_connections = 20 min_connections = 5 connect_timeout = 30 idle_timeout = 600 max_lifetime = 1800 ``` We'll want to support SQlite in the future, and better secret handling for database credentials. For now, this is workable. --- Signed-off-by: NotAShelf Change-Id: I36b4c1306511052a2748ca9d5d3429366a6a6964 --- Cargo.lock | 110 +++++++++++++- Cargo.toml | 7 +- crates/common/Cargo.toml | 4 + crates/common/src/config.rs | 263 ++++++++++++++++++++++++++++++++++ crates/common/src/database.rs | 144 +++++++++++++++++++ crates/common/src/lib.rs | 7 + 6 files changed, 530 insertions(+), 5 deletions(-) create mode 100644 crates/common/src/config.rs create mode 100644 crates/common/src/database.rs diff --git a/Cargo.lock b/Cargo.lock index 2253e3e..72cb4e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -504,6 +513,16 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -526,20 +545,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fc-common" version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap", "git2", "serde", "serde_json", "sqlx", + "tempfile", "thiserror", - "tracing", - "tracing-subscriber", "uuid", ] @@ -563,6 +586,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "fc-migrate-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "fc-common", + "tokio", + "tracing-subscriber", +] + [[package]] name = "fc-queue-runner" version = "0.1.0" @@ -1175,6 +1209,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1196,6 +1236,15 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1543,6 +1592,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + [[package]] name = "ring" version = "0.17.14" @@ -1599,6 +1665,19 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.34" @@ -2093,6 +2172,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.17" @@ -2201,10 +2293,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ + "indexmap", "serde_core", "serde_spanned", "toml_datetime", "toml_parser", + "toml_writer", "winnow", ] @@ -2226,6 +2320,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_writer" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" + [[package]] name = "tower" version = "0.5.2" @@ -2304,10 +2404,14 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/Cargo.toml b/Cargo.toml index 638b16b..c0fb3b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/evaluator", "crates/queue-runner", "crates/common", + "crates/migrate-cli", ] resolver = "3" @@ -17,15 +18,17 @@ authors = ["NotAShelf Self { + Self { + url: "postgresql://fc_ci:password@localhost/fc_ci".to_string(), + max_connections: 20, + min_connections: 5, + connect_timeout: 30, + idle_timeout: 600, + max_lifetime: 1800, + } + } +} + +impl DatabaseConfig { + pub fn validate(&self) -> anyhow::Result<()> { + if self.url.is_empty() { + return Err(anyhow::anyhow!("Database URL cannot be empty")); + } + + if !self.url.starts_with("postgresql://") && !self.url.starts_with("postgres://") { + return Err(anyhow::anyhow!( + "Database URL must start with postgresql:// or postgres://" + )); + } + + if self.max_connections == 0 { + return Err(anyhow::anyhow!( + "Max database connections must be greater than 0" + )); + } + + if self.min_connections > self.max_connections { + return Err(anyhow::anyhow!( + "Min database connections cannot exceed max connections" + )); + } + + Ok(()) + } +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + host: "127.0.0.1".to_string(), + port: 3000, + request_timeout: 30, + max_body_size: 10 * 1024 * 1024, // 10MB + } + } +} + +impl Default for EvaluatorConfig { + fn default() -> Self { + Self { + poll_interval: 60, + git_timeout: 600, + nix_timeout: 1800, + work_dir: PathBuf::from("/tmp/fc-evaluator"), + } + } +} + +impl Default for QueueRunnerConfig { + fn default() -> Self { + Self { + workers: 4, + poll_interval: 5, + build_timeout: 3600, + work_dir: PathBuf::from("/tmp/fc-queue-runner"), + } + } +} + +impl Config { + pub fn load() -> anyhow::Result { + let mut settings = config_crate::Config::builder(); + + // Load default configuration + settings = settings.add_source(config_crate::Config::try_from(&Self::default())?); + + // Load from config file if it exists + if let Ok(config_path) = std::env::var("FC_CONFIG_FILE") { + if std::path::Path::new(&config_path).exists() { + settings = settings.add_source(config_crate::File::with_name(&config_path)); + } + } else if std::path::Path::new("fc.toml").exists() { + settings = settings.add_source(config_crate::File::with_name("fc").required(false)); + } + + // Load from environment variables with FC_ prefix (highest priority) + settings = settings.add_source( + config_crate::Environment::with_prefix("FC") + .separator("__") + .try_parsing(true), + ); + + let config = settings.build()?.try_deserialize::()?; + + // Validate configuration + config.validate()?; + + Ok(config) + } + + pub fn validate(&self) -> anyhow::Result<()> { + // Validate database URL + if self.database.url.is_empty() { + return Err(anyhow::anyhow!("Database URL cannot be empty")); + } + + if !self.database.url.starts_with("postgresql://") + && !self.database.url.starts_with("postgres://") + { + return Err(anyhow::anyhow!( + "Database URL must start with postgresql:// or postgres://" + )); + } + + // Validate connection pool settings + if self.database.max_connections == 0 { + return Err(anyhow::anyhow!( + "Max database connections must be greater than 0" + )); + } + + if self.database.min_connections > self.database.max_connections { + return Err(anyhow::anyhow!( + "Min database connections cannot exceed max connections" + )); + } + + // Validate server settings + if self.server.port == 0 { + return Err(anyhow::anyhow!("Server port must be greater than 0")); + } + + // Validate evaluator settings + if self.evaluator.poll_interval == 0 { + return Err(anyhow::anyhow!( + "Evaluator poll interval must be greater than 0" + )); + } + + // Validate queue runner settings + if self.queue_runner.workers == 0 { + return Err(anyhow::anyhow!( + "Queue runner workers must be greater than 0" + )); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + #[test] + fn test_default_config() { + let config = Config::default(); + assert!(config.validate().is_ok()); + } + + #[test] + fn test_invalid_database_url() { + let mut config = Config::default(); + config.database.url = "invalid://url".to_string(); + assert!(config.validate().is_err()); + } + + #[test] + fn test_invalid_port() { + let mut config = Config::default(); + config.server.port = 0; + assert!(config.validate().is_err()); + + config.server.port = 65535; + assert!(config.validate().is_ok()); // valid port + } + + #[test] + fn test_invalid_connections() { + let mut config = Config::default(); + config.database.max_connections = 0; + assert!(config.validate().is_err()); + + config.database.max_connections = 10; + config.database.min_connections = 15; + assert!(config.validate().is_err()); + } + + #[test] + fn test_environment_override() { + // Test environment variable parsing directly + unsafe { + env::set_var("FC_DATABASE__URL", "postgresql://test:test@localhost/test"); + env::set_var("FC_SERVER__PORT", "8080"); + } + + // Test that environment variables are being read correctly + let db_url = std::env::var("FC_DATABASE__URL").unwrap(); + let server_port = std::env::var("FC_SERVER__PORT").unwrap(); + + assert_eq!(db_url, "postgresql://test:test@localhost/test"); + assert_eq!(server_port, "8080"); + + unsafe { + env::remove_var("FC_DATABASE__URL"); + env::remove_var("FC_SERVER__PORT"); + } + } +} diff --git a/crates/common/src/database.rs b/crates/common/src/database.rs new file mode 100644 index 0000000..ded6452 --- /dev/null +++ b/crates/common/src/database.rs @@ -0,0 +1,144 @@ +//! Database connection and pool management + +use crate::config::DatabaseConfig; +use sqlx::{PgPool, Row, postgres::PgPoolOptions}; +use std::time::Duration; +use tracing::{debug, info, warn}; + +pub struct Database { + pool: PgPool, +} + +impl Database { + pub async fn new(config: DatabaseConfig) -> anyhow::Result { + info!("Initializing database connection pool"); + + let pool = PgPoolOptions::new() + .max_connections(config.max_connections) + .min_connections(config.min_connections) + .acquire_timeout(Duration::from_secs(config.connect_timeout)) + .idle_timeout(Duration::from_secs(config.idle_timeout)) + .max_lifetime(Duration::from_secs(config.max_lifetime)) + .connect(&config.url) + .await?; + + // Test the connection + Self::health_check(&pool).await?; + + info!("Database connection pool initialized successfully"); + + Ok(Self { pool }) + } + + #[must_use] pub const fn pool(&self) -> &PgPool { + &self.pool + } + + pub async fn health_check(pool: &PgPool) -> anyhow::Result<()> { + debug!("Performing database health check"); + + let result: i64 = sqlx::query_scalar("SELECT 1").fetch_one(pool).await?; + + if result != 1 { + return Err(anyhow::anyhow!( + "Database health check failed: unexpected result" + )); + } + + debug!("Database health check passed"); + Ok(()) + } + + pub async fn close(&self) { + info!("Closing database connection pool"); + self.pool.close().await; + } + + pub async fn get_connection_info(&self) -> anyhow::Result { + let row = sqlx::query( + r" + SELECT + current_database() as database, + current_user as user, + version() as version, + inet_server_addr() as server_ip, + inet_server_port() as server_port + ", + ) + .fetch_one(&self.pool) + .await?; + + Ok(ConnectionInfo { + database: row.get("database"), + user: row.get("user"), + version: row.get("version"), + server_ip: row.get("server_ip"), + server_port: row.get("server_port"), + }) + } + + pub async fn get_pool_stats(&self) -> PoolStats { + let pool = &self.pool; + + PoolStats { + size: pool.size(), + idle: pool.num_idle() as u32, + active: (pool.size() - pool.num_idle() as u32), + } + } +} + +#[derive(Debug, Clone)] +pub struct ConnectionInfo { + pub database: String, + pub user: String, + pub version: String, + pub server_ip: Option, + pub server_port: Option, +} + +#[derive(Debug, Clone)] +pub struct PoolStats { + pub size: u32, + pub idle: u32, + pub active: u32, +} + +impl Drop for Database { + fn drop(&mut self) { + warn!("Database connection pool dropped without explicit close"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pool_stats() { + let stats = PoolStats { + size: 10, + idle: 3, + active: 7, + }; + + assert_eq!(stats.size, 10); + assert_eq!(stats.idle, 3); + assert_eq!(stats.active, 7); + } + + #[test] + fn test_connection_info() { + let info = ConnectionInfo { + database: "test_db".to_string(), + user: "test_user".to_string(), + version: "PostgreSQL 14.0".to_string(), + server_ip: Some("127.0.0.1".to_string()), + server_port: Some(5432), + }; + + assert_eq!(info.database, "test_db"); + assert_eq!(info.user, "test_user"); + assert_eq!(info.server_port, Some(5432)); + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 35ff89a..86a1ec6 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,10 +1,17 @@ //! Common types and utilities for CI +pub mod config; +pub mod database; pub mod error; pub mod migrate; pub mod migrate_cli; pub mod models; +#[cfg(test)] +mod tests; + +pub use config::*; +pub use database::*; pub use error::*; pub use migrate::*; pub use models::*;