From a4c3cd15179c43e57a6c8b6a89625abad61a26b1 Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Sun, 2 Nov 2025 21:04:11 +0300 Subject: [PATCH] meta: add database migrations; set up migration CLI Signed-off-by: NotAShelf Change-Id: I0cbc9798243134d36f788099ecc3ee5a6a6a6964 --- Cargo.lock | 3 + crates/common/Cargo.toml | 3 + .../common/migrations/001_initial_schema.sql | 151 ++++++++++++++++++ crates/common/migrations/README.md | 26 +++ crates/common/src/lib.rs | 3 + crates/common/src/migrate.rs | 69 ++++++++ crates/common/src/migrate_cli.rs | 85 ++++++++++ crates/migrate-cli/Cargo.toml | 18 +++ crates/migrate-cli/src/main.rs | 8 + 9 files changed, 366 insertions(+) create mode 100644 crates/common/migrations/001_initial_schema.sql create mode 100644 crates/common/migrations/README.md create mode 100644 crates/common/src/migrate.rs create mode 100644 crates/common/src/migrate_cli.rs create mode 100644 crates/migrate-cli/Cargo.toml create mode 100644 crates/migrate-cli/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 6fb9de7..2253e3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -532,11 +532,14 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", + "clap", "git2", "serde", "serde_json", "sqlx", "thiserror", + "tracing", + "tracing-subscriber", "uuid", ] diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index b07c6b6..d76b663 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -15,3 +15,6 @@ chrono.workspace = true anyhow.workspace = true thiserror.workspace = true git2.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +clap.workspace = true diff --git a/crates/common/migrations/001_initial_schema.sql b/crates/common/migrations/001_initial_schema.sql new file mode 100644 index 0000000..f121c30 --- /dev/null +++ b/crates/common/migrations/001_initial_schema.sql @@ -0,0 +1,151 @@ +-- Initial schema for FC +-- Creates all core tables for the CI system + +-- Enable UUID extension for UUID generation +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Projects: stores repository configurations +CREATE TABLE projects ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + name VARCHAR(255) NOT NULL UNIQUE, + description TEXT, + repository_url TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Jobsets: Contains build configurations for each project +CREATE TABLE jobsets ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + nix_expression TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + UNIQUE(project_id, name) +); + +-- Evaluations: Tracks Nix evaluation results for each jobset +CREATE TABLE evaluations ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + jobset_id UUID NOT NULL REFERENCES jobsets(id) ON DELETE CASCADE, + commit_hash VARCHAR(40) NOT NULL, + evaluation_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'completed', 'failed')), + error_message TEXT, + UNIQUE(jobset_id, commit_hash) +); + +-- Builds: Individual build jobs with their status +CREATE TABLE builds ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + evaluation_id UUID NOT NULL REFERENCES evaluations(id) ON DELETE CASCADE, + job_name VARCHAR(255) NOT NULL, + drv_path TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'completed', 'failed', 'cancelled')), + started_at TIMESTAMP WITH TIME ZONE, + completed_at TIMESTAMP WITH TIME ZONE, + log_path TEXT, + build_output_path TEXT, + error_message TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + UNIQUE(evaluation_id, job_name) +); + +-- Build products: Stores output artifacts and metadata +CREATE TABLE build_products ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + build_id UUID NOT NULL REFERENCES builds(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + path TEXT NOT NULL, + sha256_hash VARCHAR(64), + file_size BIGINT, + content_type VARCHAR(100), + is_directory BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +-- Build steps: Detailed build execution logs and timing +CREATE TABLE build_steps ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + build_id UUID NOT NULL REFERENCES builds(id) ON DELETE CASCADE, + step_number INTEGER NOT NULL, + command TEXT NOT NULL, + output TEXT, + error_output TEXT, + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + completed_at TIMESTAMP WITH TIME ZONE, + exit_code INTEGER, + UNIQUE(build_id, step_number) +); + +-- Projects indexes +CREATE INDEX idx_projects_name ON projects(name); +CREATE INDEX idx_projects_created_at ON projects(created_at); + +-- Jobsets indexes +CREATE INDEX idx_jobsets_project_id ON jobsets(project_id); +CREATE INDEX idx_jobsets_enabled ON jobsets(enabled); +CREATE INDEX idx_jobsets_name ON jobsets(name); + +-- Evaluations indexes +CREATE INDEX idx_evaluations_jobset_id ON evaluations(jobset_id); +CREATE INDEX idx_evaluations_commit_hash ON evaluations(commit_hash); +CREATE INDEX idx_evaluations_status ON evaluations(status); +CREATE INDEX idx_evaluations_evaluation_time ON evaluations(evaluation_time); + +-- Builds indexes +CREATE INDEX idx_builds_evaluation_id ON builds(evaluation_id); +CREATE INDEX idx_builds_status ON builds(status); +CREATE INDEX idx_builds_job_name ON builds(job_name); +CREATE INDEX idx_builds_started_at ON builds(started_at); +CREATE INDEX idx_builds_completed_at ON builds(completed_at); + +-- Build products indexes +CREATE INDEX idx_build_products_build_id ON build_products(build_id); +CREATE INDEX idx_build_products_name ON build_products(name); + +-- Build steps indexes +CREATE INDEX idx_build_steps_build_id ON build_steps(build_id); +CREATE INDEX idx_build_steps_started_at ON build_steps(started_at); + +-- Create trigger functions for updated_at timestamps +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ language 'plpgsql'; + +-- Create triggers for automatic updated_at updates +CREATE TRIGGER update_projects_updated_at + BEFORE UPDATE ON projects + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_jobsets_updated_at + BEFORE UPDATE ON jobsets + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +-- Create view for active jobsets (jobsets that are enabled and belong to active projects) +CREATE VIEW active_jobsets AS +SELECT + j.*, + p.name as project_name, + p.repository_url +FROM jobsets j +JOIN projects p ON j.project_id = p.id +WHERE j.enabled = true; + +-- Create view for build statistics +CREATE VIEW build_stats AS +SELECT + COUNT(*) as total_builds, + COUNT(CASE WHEN status = 'completed' THEN 1 END) as completed_builds, + COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed_builds, + COUNT(CASE WHEN status = 'running' THEN 1 END) as running_builds, + COUNT(CASE WHEN status = 'pending' THEN 1 END) as pending_builds, + AVG(EXTRACT(EPOCH FROM (completed_at - started_at))) as avg_duration_seconds +FROM builds +WHERE started_at IS NOT NULL; diff --git a/crates/common/migrations/README.md b/crates/common/migrations/README.md new file mode 100644 index 0000000..8a7ffed --- /dev/null +++ b/crates/common/migrations/README.md @@ -0,0 +1,26 @@ +# Database Migrations + +This directory contains SQL migrations for the FC database. + +## Migration Files + +- `001_initial_schema.sql`: Creates the core database schema including projects, + jobsets, evaluations, builds, and related tables. + +## Running Migrations + +The easiest way to run migrations is to use the vendored CLI, `fc-migrate`. +Packagers should vendor this crate if possible. + +```bash +# Run all pending migrations +fc-migrate up postgresql://user:password@localhost/fc_ci + +# Validate current schema +fc-migrate validate postgresql://user:password@localhost/fc_ci + +# Create a new migration +fc-migrate create migration_name +``` + +TODO: add or generate schema overviews diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 978ee67..35ff89a 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,7 +1,10 @@ //! Common types and utilities for CI pub mod error; +pub mod migrate; +pub mod migrate_cli; pub mod models; pub use error::*; +pub use migrate::*; pub use models::*; diff --git a/crates/common/src/migrate.rs b/crates/common/src/migrate.rs new file mode 100644 index 0000000..733a49b --- /dev/null +++ b/crates/common/src/migrate.rs @@ -0,0 +1,69 @@ +//! Database migration utilities + +use sqlx::{PgPool, Postgres, migrate::MigrateDatabase}; +use tracing::{error, info, warn}; + +/// Runs database migrations and ensures the database exists +pub async fn run_migrations(database_url: &str) -> anyhow::Result<()> { + info!("Starting database migrations"); + + // Check if database exists, create if it doesn't + if !Postgres::database_exists(database_url).await? { + warn!("Database does not exist, creating it"); + Postgres::create_database(database_url).await?; + info!("Database created successfully"); + } + + // Set up connection pool with retry logic, then run migrations + let pool = create_connection_pool(database_url).await?; + match sqlx::migrate!("./migrations").run(&pool).await { + Ok(()) => { + info!("Database migrations completed successfully"); + Ok(()) + } + Err(e) => { + error!("Failed to run database migrations: {}", e); + Err(anyhow::anyhow!("Migration failed: {e}")) + } + } +} + +/// Creates a connection pool with proper configuration +async fn create_connection_pool(database_url: &str) -> anyhow::Result { + let pool = PgPool::connect(database_url).await?; + + // Test the connection + sqlx::query("SELECT 1").fetch_one(&pool).await?; + + Ok(pool) +} + +/// Validates that all required tables exist and have the expected structure +pub async fn validate_schema(pool: &PgPool) -> anyhow::Result<()> { + info!("Validating database schema"); + + let required_tables = vec![ + "projects", + "jobsets", + "evaluations", + "builds", + "build_products", + "build_steps", + ]; + + for table in required_tables { + let result = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = $1", + ) + .bind(table) + .fetch_one(pool) + .await?; + + if result == 0 { + return Err(anyhow::anyhow!("Required table '{table}' does not exist")); + } + } + + info!("Database schema validation passed"); + Ok(()) +} diff --git a/crates/common/src/migrate_cli.rs b/crates/common/src/migrate_cli.rs new file mode 100644 index 0000000..8c3d314 --- /dev/null +++ b/crates/common/src/migrate_cli.rs @@ -0,0 +1,85 @@ +//! CLI utility for database migrations + +use clap::{Parser, Subcommand}; +use tracing::info; +use tracing_subscriber::fmt::init; + +#[derive(Parser)] +#[command(name = "fc-migrate")] +#[command(about = "Database migration utility for FC CI")] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Run all pending migrations + Up { + /// Database connection URL + database_url: String, + }, + /// Validate the current schema + Validate { + /// Database connection URL + database_url: String, + }, + /// Create a new migration file + Create { + /// Migration name + #[arg(required = true)] + name: String, + }, +} + +pub async fn run() -> anyhow::Result<()> { + let cli = Cli::parse(); + + // Initialize logging + init(); + + match cli.command { + Commands::Up { database_url } => { + info!("Running database migrations"); + crate::run_migrations(&database_url).await?; + info!("Migrations completed successfully"); + } + Commands::Validate { database_url } => { + info!("Validating database schema"); + let pool = sqlx::PgPool::connect(&database_url).await?; + crate::validate_schema(&pool).await?; + info!("Schema validation passed"); + } + Commands::Create { name } => { + create_migration(&name)?; + } + } + + Ok(()) +} + +fn create_migration(name: &str) -> anyhow::Result<()> { + use chrono::Utc; + use std::fs; + + let timestamp = Utc::now().format("%Y%m%d_%H%M%S"); + let filename = format!("{timestamp}_{name}.sql"); + let filepath = format!("crates/common/migrations/{filename}"); + + let content = format!( + "-- Migration: {}\n\ + -- Created: {}\n\ + \n\ + -- Add your migration SQL here\n\ + \n\ + -- Uncomment below for rollback SQL\n\ + -- ROLLBACK;\n", + name, + Utc::now().to_rfc3339() + ); + + fs::write(&filepath, content)?; + println!("Created migration file: {filepath}"); + + Ok(()) +} diff --git a/crates/migrate-cli/Cargo.toml b/crates/migrate-cli/Cargo.toml new file mode 100644 index 0000000..2d4f42c --- /dev/null +++ b/crates/migrate-cli/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "fc-migrate-cli" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "fc-migrate" +path = "src/main.rs" + +[dependencies] +fc-common = { path = "../common" } +clap.workspace = true +anyhow.workspace = true +tracing-subscriber.workspace = true +tokio.workspace = true \ No newline at end of file diff --git a/crates/migrate-cli/src/main.rs b/crates/migrate-cli/src/main.rs new file mode 100644 index 0000000..99686e2 --- /dev/null +++ b/crates/migrate-cli/src/main.rs @@ -0,0 +1,8 @@ +//! Database migration CLI utility + +use fc_common::migrate_cli::run; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + run().await +}