NotAShelf/circus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nix: attempt to fix VM tests; general cleanup

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I65f6909ef02ab4599f5b0bbc0930367e6a6a6964
This commit is contained in:
raf 2026-02-14 13:55:07 +03:00
commit a2b638d4db
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
26 changed files with 2320 additions and 2939 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/common/src/models.rs
View file

@ -48,6 +48,7 @@ pub struct Evaluation {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)]
#[serde(rename_all = "lowercase")]
#[sqlx(type_name = "text", rename_all = "lowercase")]
pub enum EvaluationStatus {
Pending,
@ -121,6 +122,7 @@ pub struct Build {
}
#[derive(Debug, Clone, Serialize, Deserialize, sqlx::Type, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
#[sqlx(type_name = "text", rename_all = "lowercase")]
pub enum BuildStatus {
Pending,

17
crates/common/src/repo/evaluations.rs
View file

@ -170,3 +170,20 @@ pub async fn count(pool: &PgPool) -> Result<i64> {
.map_err(CiError::Database)?;
Ok(row.0)
}
/// Get an evaluation by jobset_id and commit_hash.
pub async fn get_by_jobset_and_commit(
pool: &PgPool,
jobset_id: Uuid,
commit_hash: &str,
) -> Result<Option<Evaluation>> {
sqlx::query_as::<_, Evaluation>(
"SELECT * FROM evaluations WHERE jobset_id = $1 AND commit_hash = $2 \
ORDER BY evaluation_time DESC LIMIT 1",
)
.bind(jobset_id)
.bind(commit_hash)
.fetch_optional(pool)
.await
.map_err(CiError::Database)
}

308
crates/evaluator/src/eval_loop.rs
View file

@ -1,9 +1,10 @@
use std::{collections::HashMap, time::Duration};
use anyhow::Context;
use chrono::Utc;
use fc_common::{
config::EvaluatorConfig,
error::check_disk_space,
error::{CiError, check_disk_space},
models::{
CreateBuild,
CreateEvaluation,
@ -15,6 +16,7 @@ use fc_common::{
};
use futures::stream::{self, StreamExt};
use sqlx::PgPool;
use tracing::info;
use uuid::Uuid;
pub async fn run(pool: PgPool, config: EvaluatorConfig) -> anyhow::Result<()> {
@ -172,8 +174,34 @@ async fn evaluate_jobset(
"Inputs unchanged (hash: {}), skipping evaluation",
&inputs_hash[..16],
);
// Create evaluation record even when skipped so system tracks this check
// Handle duplicate key conflict gracefully (another evaluator may have
// created it) - fall through to process existing evaluation instead of
// skipping
if let Err(e) = repo::evaluations::create(pool, CreateEvaluation {
jobset_id: jobset.id,
commit_hash: commit_hash.clone(),
pr_number: None,
pr_head_branch: None,
pr_base_branch: None,
pr_action: None,
})
.await
{
if !matches!(e, CiError::Conflict(_)) {
return Err(e.into());
}
tracing::info!(
jobset = %jobset.name,
commit = %commit_hash,
"Evaluation already exists (concurrent creation in inputs_hash path), will process"
);
} else {
// Successfully created new evaluation, can skip
repo::jobsets::update_last_checked(pool, jobset.id).await?;
return Ok(());
}
}
// Also skip if commit hasn't changed (backward compat)
if let Some(latest) = repo::evaluations::get_latest(pool, jobset.id).await?
@ -183,9 +211,114 @@ async fn evaluate_jobset(
tracing::debug!(
jobset = %jobset.name,
commit = %commit_hash,
"Already evaluated, skipping"
"Inputs unchanged (hash: {}), skipping evaluation",
&inputs_hash[..16],
);
// Create evaluation record even when skipped so system tracks this check
// Handle duplicate key conflict gracefully (another evaluator may have
// created it) - fall through to process existing evaluation instead of
// skipping
if let Err(e) = repo::evaluations::create(pool, CreateEvaluation {
jobset_id: jobset.id,
commit_hash: commit_hash.clone(),
pr_number: None,
pr_head_branch: None,
pr_base_branch: None,
pr_action: None,
})
.await
{
if !matches!(e, CiError::Conflict(_)) {
return Err(e.into());
}
tracing::info!(
jobset = %jobset.name,
commit = %commit_hash,
"Evaluation already exists (concurrent creation in commit path), will process"
);
let existing = repo::evaluations::get_by_jobset_and_commit(
pool,
jobset.id,
&commit_hash,
)
.await?
.ok_or_else(|| {
anyhow::anyhow!(
"Evaluation conflict but not found: {}/{}",
jobset.id,
commit_hash
)
})?;
if existing.status == EvaluationStatus::Completed {
// Check if we need to re-evaluate due to no builds
let builds =
repo::builds::list_for_evaluation(pool, existing.id).await?;
if builds.is_empty() {
info!(
"Evaluation completed with 0 builds, re-running nix evaluation \
jobset={} commit={}",
jobset.name, commit_hash
);
// Update existing evaluation status to Running
repo::evaluations::update_status(
pool,
existing.id,
EvaluationStatus::Running,
None,
)
.await?;
// Use existing evaluation instead of creating new one
let eval = existing;
// Run nix evaluation and create builds from the result
let eval_result = crate::nix::evaluate(
&repo_path,
&jobset.nix_expression,
jobset.flake_mode,
nix_timeout,
config,
&inputs,
)
.await?;
create_builds_from_eval(pool, eval.id, &eval_result).await?;
repo::evaluations::update_status(
pool,
eval.id,
EvaluationStatus::Completed,
None,
)
.await?;
repo::jobsets::update_last_checked(pool, jobset.id).await?;
return Ok(());
} else {
info!(
"Evaluation already completed with {} builds, skipping nix \
evaluation jobset={} commit={}",
builds.len(),
jobset.name,
commit_hash
);
repo::jobsets::update_last_checked(pool, jobset.id).await?;
return Ok(());
}
}
// Existing evaluation is pending or running, update status and continue
repo::evaluations::update_status(
pool,
existing.id,
EvaluationStatus::Running,
None,
)
.await?;
} else {
// Successfully created new evaluation, can skip
repo::jobsets::update_last_checked(pool, jobset.id).await?;
return Ok(());
}
}
tracing::info!(
@ -194,8 +327,9 @@ async fn evaluate_jobset(
"Starting evaluation"
);
// Create evaluation record
let eval = repo::evaluations::create(pool, CreateEvaluation {
// Create evaluation record. If it already exists (race condition), fetch the
// existing one and continue. Only update status if it's still pending.
let eval = match repo::evaluations::create(pool, CreateEvaluation {
jobset_id: jobset.id,
commit_hash: commit_hash.clone(),
pr_number: None,
@ -203,16 +337,72 @@ async fn evaluate_jobset(
pr_base_branch: None,
pr_action: None,
})
.await?;
.await
{
Ok(eval) => eval,
Err(CiError::Conflict(_)) => {
tracing::info!(
jobset = %jobset.name,
commit = %commit_hash,
"Evaluation already exists (conflict), fetching existing record"
);
let existing = repo::evaluations::get_by_jobset_and_commit(
pool,
jobset.id,
&commit_hash,
)
.await?
.ok_or_else(|| {
anyhow::anyhow!(
"Evaluation conflict but not found: {}/{}",
jobset.id,
commit_hash
)
})?;
// Mark as running and set inputs hash
if existing.status == EvaluationStatus::Pending {
repo::evaluations::update_status(
pool,
eval.id,
existing.id,
EvaluationStatus::Running,
None,
)
.await?;
} else if existing.status == EvaluationStatus::Completed {
let build_count = repo::builds::count_filtered(
pool,
Some(existing.id),
None,
None,
None,
)
.await?;
if build_count > 0 {
info!(
"Evaluation already completed with {} builds, skipping nix \
evaluation jobset={} commit={}",
build_count, jobset.name, commit_hash
);
return Ok(());
} else {
info!(
"Evaluation completed but has 0 builds, re-running nix evaluation \
jobset={} commit={}",
jobset.name, commit_hash
);
}
}
existing
},
Err(e) => {
return Err(anyhow::anyhow!(e)).with_context(|| {
format!("failed to create evaluation for jobset {}", jobset.name)
});
},
};
// Set inputs hash (only needed for new evaluations, not existing ones)
let _ = repo::evaluations::set_inputs_hash(pool, eval.id, &inputs_hash).await;
// Check for declarative config in repo
@ -230,6 +420,7 @@ async fn evaluate_jobset(
.await
{
Ok(eval_result) => {
tracing::debug!(jobset = %jobset.name, job_count = eval_result.jobs.len(), "Nix evaluation returned");
tracing::info!(
jobset = %jobset.name,
count = eval_result.jobs.len(),
@ -237,8 +428,61 @@ async fn evaluate_jobset(
"Evaluation discovered jobs"
);
// Create build records, tracking drv_path -> build_id for dependency
// resolution
create_builds_from_eval(pool, eval.id, &eval_result).await?;
repo::evaluations::update_status(
pool,
eval.id,
EvaluationStatus::Completed,
None,
)
.await?;
},
Err(e) => {
let msg = e.to_string();
tracing::error!(jobset = %jobset.name, "Evaluation failed: {msg}");
repo::evaluations::update_status(
pool,
eval.id,
EvaluationStatus::Failed,
Some(&msg),
)
.await?;
},
}
// Update last_checked_at timestamp for per-jobset interval tracking
if let Err(e) = repo::jobsets::update_last_checked(pool, jobset.id).await {
tracing::warn!(
jobset = %jobset.name,
"Failed to update last_checked_at: {e}"
);
}
// Mark one-shot jobsets as complete (disabled) after evaluation
if jobset.state == JobsetState::OneShot {
tracing::info!(
jobset = %jobset.name,
"One-shot evaluation complete, disabling jobset"
);
if let Err(e) = repo::jobsets::mark_one_shot_complete(pool, jobset.id).await
{
tracing::error!(
jobset = %jobset.name,
"Failed to mark one-shot complete: {e}"
);
}
}
Ok(())
}
/// Create build records from evaluation results, resolving dependencies.
async fn create_builds_from_eval(
pool: &PgPool,
eval_id: Uuid,
eval_result: &crate::nix::EvalResult,
) -> anyhow::Result<()> {
let mut drv_to_build: HashMap<String, Uuid> = HashMap::new();
let mut name_to_build: HashMap<String, Uuid> = HashMap::new();
@ -254,7 +498,7 @@ async fn evaluate_jobset(
let is_aggregate = job.constituents.is_some();
let build = repo::builds::create(pool, CreateBuild {
evaluation_id: eval.id,
evaluation_id: eval_id,
job_name: job.name.clone(),
drv_path: job.drv_path.clone(),
system: job.system.clone(),
@ -302,50 +546,6 @@ async fn evaluate_jobset(
}
}
repo::evaluations::update_status(
pool,
eval.id,
EvaluationStatus::Completed,
None,
)
.await?;
},
Err(e) => {
let msg = e.to_string();
tracing::error!(jobset = %jobset.name, "Evaluation failed: {msg}");
repo::evaluations::update_status(
pool,
eval.id,
EvaluationStatus::Failed,
Some(&msg),
)
.await?;
},
}
// Update last_checked_at timestamp for per-jobset interval tracking
if let Err(e) = repo::jobsets::update_last_checked(pool, jobset.id).await {
tracing::warn!(
jobset = %jobset.name,
"Failed to update last_checked_at: {e}"
);
}
// Mark one-shot jobsets as complete (disabled) after evaluation
if jobset.state == JobsetState::OneShot {
tracing::info!(
jobset = %jobset.name,
"One-shot evaluation complete, disabling jobset"
);
if let Err(e) = repo::jobsets::mark_one_shot_complete(pool, jobset.id).await
{
tracing::error!(
jobset = %jobset.name,
"Failed to mark one-shot complete: {e}"
);
}
}
Ok(())
}

40
crates/evaluator/src/git.rs
View file

@ -16,7 +16,9 @@ pub fn clone_or_fetch(
) -> Result<(PathBuf, String)> {
let repo_path = work_dir.join(project_name);
let repo = if repo_path.exists() {
let is_fetch = repo_path.exists();
let repo = if is_fetch {
let repo = Repository::open(&repo_path)?;
// Fetch origin — scope the borrow so `remote` is dropped before we move
// `repo`
@ -29,21 +31,35 @@ pub fn clone_or_fetch(
Repository::clone(url, &repo_path)?
};
// Resolve commit: use specific branch ref or fall back to HEAD
let hash = if let Some(branch_name) = branch {
let refname = format!("refs/remotes/origin/{branch_name}");
let reference = repo.find_reference(&refname).map_err(|e| {
// Resolve commit from remote refs (which are always up-to-date after fetch).
// When no branch is specified, detect the default branch from local HEAD's
// tracking target.
let branch_name = match branch {
Some(b) => b.to_string(),
None => {
let head = repo.head()?;
head.shorthand().unwrap_or("master").to_string()
},
};
let remote_ref = format!("refs/remotes/origin/{branch_name}");
let reference = repo.find_reference(&remote_ref).map_err(|e| {
fc_common::error::CiError::NotFound(format!(
"Branch '{branch_name}' not found ({refname}): {e}"
"Branch '{branch_name}' not found ({remote_ref}): {e}"
))
})?;
let commit = reference.peel_to_commit()?;
commit.id().to_string()
} else {
let head = repo.head()?;
let commit = head.peel_to_commit()?;
commit.id().to_string()
};
let hash = commit.id().to_string();
// After fetch, update the working tree so nix evaluation sees the latest
// files. Skip on fresh clone since the checkout is already current.
if is_fetch {
repo.checkout_tree(
commit.as_object(),
Some(git2::build::CheckoutBuilder::new().force()),
)?;
repo.set_head_detached(commit.id())?;
}
Ok((repo_path, hash))
}

61
crates/evaluator/src/nix.rs
View file

@ -8,22 +8,37 @@ use fc_common::{
};
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize)]
#[derive(Debug, Clone)]
pub struct NixJob {
pub name: String,
#[serde(alias = "drvPath")]
pub drv_path: String,
pub system: Option<String>,
pub outputs: Option<HashMap<String, String>>,
#[serde(alias = "inputDrvs")]
pub input_drvs: Option<HashMap<String, serde_json::Value>>,
pub constituents: Option<Vec<String>>,
}
/// Raw deserialization target for nix-eval-jobs output.
/// nix-eval-jobs emits both `attr` (attribute path) and `name` (derivation
/// name) in the same JSON object. We deserialize them separately and prefer
/// `attr` as the job identifier.
#[derive(Deserialize)]
struct RawNixJob {
name: Option<String>,
attr: Option<String>,
#[serde(alias = "drvPath")]
drv_path: Option<String>,
system: Option<String>,
outputs: Option<HashMap<String, String>>,
#[serde(alias = "inputDrvs")]
input_drvs: Option<HashMap<String, serde_json::Value>>,
constituents: Option<Vec<String>>,
}
/// An error reported by nix-eval-jobs for a single job.
#[derive(Debug, Clone, Deserialize)]
struct NixEvalError {
#[serde(alias = "attr")]
attr: Option<String>,
name: Option<String>,
error: String,
}
@ -49,7 +64,11 @@ pub fn parse_eval_output(stdout: &str) -> EvalResult {
&& parsed.get("error").is_some()
{
if let Ok(eval_err) = serde_json::from_str::<NixEvalError>(line) {
let name = eval_err.name.as_deref().unwrap_or("<unknown>");
let name = eval_err
.attr
.as_deref()
.or(eval_err.name.as_deref())
.unwrap_or("<unknown>");
tracing::warn!(
job = name,
"nix-eval-jobs reported error: {}",
@ -60,8 +79,20 @@ pub fn parse_eval_output(stdout: &str) -> EvalResult {
continue;
}
match serde_json::from_str::<NixJob>(line) {
Ok(job) => jobs.push(job),
match serde_json::from_str::<RawNixJob>(line) {
Ok(raw) => {
// drv_path is required for a valid job
if let Some(drv_path) = raw.drv_path {
jobs.push(NixJob {
name: raw.attr.or(raw.name).unwrap_or_default(),
drv_path,
system: raw.system,
outputs: raw.outputs,
input_drvs: raw.input_drvs,
constituents: raw.constituents,
});
}
},
Err(e) => {
tracing::warn!("Failed to parse nix-eval-jobs line: {e}");
},
@ -100,9 +131,11 @@ async fn evaluate_flake(
) -> Result<EvalResult> {
let flake_ref = format!("{}#{}", repo_path.display(), nix_expression);
tracing::debug!(flake_ref = %flake_ref, "Running nix-eval-jobs");
tokio::time::timeout(timeout, async {
let mut cmd = tokio::process::Command::new("nix-eval-jobs");
cmd.arg("--flake").arg(&flake_ref);
cmd.arg("--flake").arg(&flake_ref).arg("--force-recurse");
if config.restrict_eval {
cmd.args(["--option", "restrict-eval", "true"]);
@ -130,6 +163,16 @@ async fn evaluate_flake(
);
}
if result.jobs.is_empty() && result.error_count == 0 {
let stderr = String::from_utf8_lossy(&out.stderr);
if !stderr.trim().is_empty() {
tracing::warn!(
stderr = %stderr,
"nix-eval-jobs returned no jobs, stderr output present"
);
}
}
Ok(result)
},
_ => {
@ -163,7 +206,7 @@ async fn evaluate_legacy(
tokio::time::timeout(timeout, async {
// Try nix-eval-jobs without --flake for legacy expressions
let mut cmd = tokio::process::Command::new("nix-eval-jobs");
cmd.arg(&expr_path);
cmd.arg(&expr_path).arg("--force-recurse");
if config.restrict_eval {
cmd.args(["--option", "restrict-eval", "true"]);

25
crates/evaluator/tests/eval_tests.rs
View file

@ -87,6 +87,31 @@ fn test_parse_error_without_name() {
assert_eq!(result.error_count, 1);
}
#[test]
fn test_parse_nix_eval_jobs_attr_field() {
// nix-eval-jobs uses "attr" instead of "name" for the job identifier
let line = r#"{"attr":"x86_64-linux.hello","drvPath":"/nix/store/abc123-hello.drv","system":"x86_64-linux"}"#;
let result = fc_evaluator::nix::parse_eval_output(line);
assert_eq!(result.jobs.len(), 1);
assert_eq!(result.jobs[0].name, "x86_64-linux.hello");
assert_eq!(result.jobs[0].drv_path, "/nix/store/abc123-hello.drv");
}
#[test]
fn test_parse_nix_eval_jobs_both_attr_and_name() {
// nix-eval-jobs with --force-recurse outputs both "attr" and "name" fields.
// "attr" is the attribute path, "name" is the derivation name. We prefer
// "attr" as the job identifier.
let line = r#"{"attr":"x86_64-linux.hello","attrPath":["x86_64-linux","hello"],"drvPath":"/nix/store/abc123-hello.drv","name":"fc-test-hello","outputs":{"out":"/nix/store/abc123-hello"},"system":"x86_64-linux"}"#;
let result = fc_evaluator::nix::parse_eval_output(line);
assert_eq!(result.jobs.len(), 1);
assert_eq!(result.jobs[0].name, "x86_64-linux.hello");
assert_eq!(result.jobs[0].drv_path, "/nix/store/abc123-hello.drv");
assert_eq!(result.jobs[0].system.as_deref(), Some("x86_64-linux"));
let outputs = result.jobs[0].outputs.as_ref().unwrap();
assert_eq!(outputs.get("out").unwrap(), "/nix/store/abc123-hello");
}
// --- Inputs hash computation ---
#[test]

9
crates/queue-runner/src/worker.rs
View file

@ -27,6 +27,7 @@ use tokio::sync::Semaphore;
pub struct WorkerPool {
semaphore: Arc<Semaphore>,
worker_count: usize,
pool: PgPool,
work_dir: Arc<PathBuf>,
build_timeout: Duration,
@ -57,6 +58,7 @@ impl WorkerPool {
let alert_manager = alert_config.map(AlertManager::new);
Self {
semaphore: Arc::new(Semaphore::new(workers)),
worker_count: workers,
pool: db_pool,
work_dir: Arc::new(work_dir),
build_timeout,
@ -79,7 +81,7 @@ impl WorkerPool {
/// Wait until all in-flight builds complete (semaphore fully available).
pub async fn wait_for_drain(&self) {
// Acquire all permits = all workers idle
let workers = self.semaphore.available_permits() + 1; // at least 1
let workers = self.worker_count;
let _ = tokio::time::timeout(
Duration::from_secs(self.build_timeout.as_secs() + 60),
async {
@ -645,6 +647,11 @@ async fn run_build(
max = build.max_retries,
"Build failed, scheduling retry"
);
// Clean up old build steps before retry
sqlx::query("DELETE FROM build_steps WHERE build_id = $1")
.bind(build.id)
.execute(pool)
.await?;
sqlx::query(
"UPDATE builds SET status = 'pending', started_at = NULL, \
retry_count = retry_count + 1, completed_at = NULL WHERE id = $1",

28
crates/server/src/auth_middleware.rs
View file

@ -198,14 +198,36 @@ impl RequireRoles {
}
/// Session extraction middleware for dashboard routes.
/// Reads `fc_user_session` or `fc_session` cookie and inserts User/ApiKey into
/// extensions if valid.
/// Reads `fc_user_session` or `fc_session` cookie, or Bearer token (API key),
/// and inserts User/ApiKey into extensions if valid.
pub async fn extract_session(
State(state): State<AppState>,
mut request: Request,
next: Next,
) -> Response {
// Extract cookie header first, then clone to end the borrow
// Try Bearer token first (API key auth)
let auth_header = request
.headers()
.get("authorization")
.and_then(|v| v.to_str().ok())
.map(String::from);
if let Some(ref auth_header) = auth_header {
if let Some(token) = auth_header.strip_prefix("Bearer ") {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(token.as_bytes());
let key_hash = hex::encode(hasher.finalize());
if let Ok(Some(api_key)) =
fc_common::repo::api_keys::get_by_hash(&state.pool, &key_hash).await
{
request.extensions_mut().insert(api_key.clone());
}
}
}
// Extract cookie header next
let cookie_header = request
.headers()
.get("cookie")

123
crates/server/src/routes/cache.rs
View file

@ -10,6 +10,54 @@ use tokio::process::Command;
use crate::{error::ApiError, state::AppState};
/// Extract the first path info entry from `nix path-info --json` output,
/// handling both the old array format (`[{"path":...}]`) and the new
/// object-keyed format (`{"/nix/store/...": {...}}`).
fn first_path_info_entry(
parsed: &serde_json::Value,
) -> Option<(&serde_json::Value, Option<&str>)> {
if let Some(arr) = parsed.as_array() {
let entry = arr.first()?;
let path = entry.get("path").and_then(|v| v.as_str());
Some((entry, path))
} else if let Some(obj) = parsed.as_object() {
let (key, val) = obj.iter().next()?;
Some((val, Some(key.as_str())))
} else {
None
}
}
/// Look up a store path by its nix hash, checking both build_products and
/// builds tables.
async fn find_store_path(
pool: &sqlx::PgPool,
hash: &str,
) -> std::result::Result<Option<String>, ApiError> {
let like_pattern = format!("/nix/store/{hash}-%");
let path: Option<String> = sqlx::query_scalar(
"SELECT path FROM build_products WHERE path LIKE $1 LIMIT 1",
)
.bind(&like_pattern)
.fetch_optional(pool)
.await
.map_err(|e| ApiError(fc_common::CiError::Database(e)))?;
if path.is_some() {
return Ok(path);
}
sqlx::query_scalar(
"SELECT build_output_path FROM builds WHERE build_output_path LIKE $1 \
LIMIT 1",
)
.bind(&like_pattern)
.fetch_optional(pool)
.await
.map_err(|e| ApiError(fc_common::CiError::Database(e)))
}
/// Serve `NARInfo` for a store path hash.
/// GET /nix-cache/{hash}.narinfo
async fn narinfo(
@ -27,27 +75,14 @@ async fn narinfo(
return Ok(StatusCode::NOT_FOUND.into_response());
}
// Look up the store path from build_products by matching the hash prefix
let product = sqlx::query_as::<_, fc_common::models::BuildProduct>(
"SELECT * FROM build_products WHERE path LIKE $1 LIMIT 1",
)
.bind(format!("/nix/store/{hash}-%"))
.fetch_optional(&state.pool)
.await
.map_err(|e| ApiError(fc_common::CiError::Database(e)))?;
let product = match product {
Some(p) => p,
None => return Ok(StatusCode::NOT_FOUND.into_response()),
let store_path = match find_store_path(&state.pool, hash).await? {
Some(p) if fc_common::validate::is_valid_store_path(&p) => p,
_ => return Ok(StatusCode::NOT_FOUND.into_response()),
};
if !fc_common::validate::is_valid_store_path(&product.path) {
return Ok(StatusCode::NOT_FOUND.into_response());
}
// Get narinfo from nix path-info
let output = Command::new("nix")
.args(["path-info", "--json", &product.path])
.args(["path-info", "--json", &store_path])
.output()
.await;
@ -62,7 +97,7 @@ async fn narinfo(
Err(_) => return Ok(StatusCode::NOT_FOUND.into_response()),
};
let entry = match parsed.as_array().and_then(|a| a.first()) {
let (entry, path_from_info) = match first_path_info_entry(&parsed) {
Some(e) => e,
None => return Ok(StatusCode::NOT_FOUND.into_response()),
};
@ -72,10 +107,7 @@ async fn narinfo(
.get("narSize")
.and_then(serde_json::Value::as_u64)
.unwrap_or(0);
let store_path = entry
.get("path")
.and_then(|v| v.as_str())
.unwrap_or(&product.path);
let store_path = path_from_info.unwrap_or(&store_path);
let refs: Vec<&str> = entry
.get("references")
@ -174,11 +206,8 @@ async fn sign_narinfo(narinfo: &str, key_file: &std::path::Path) -> String {
if let Ok(o) = re_output
&& let Ok(parsed) =
serde_json::from_slice::<serde_json::Value>(&o.stdout)
&& let Some(sigs) = parsed
.as_array()
.and_then(|a| a.first())
.and_then(|e| e.get("signatures"))
.and_then(|v| v.as_array())
&& let Some((entry, _)) = first_path_info_entry(&parsed)
&& let Some(sigs) = entry.get("signatures").and_then(|v| v.as_array())
{
let sig_lines: Vec<String> = sigs
.iter()
@ -214,26 +243,14 @@ async fn serve_nar_zst(
return Ok(StatusCode::NOT_FOUND.into_response());
}
let product = sqlx::query_as::<_, fc_common::models::BuildProduct>(
"SELECT * FROM build_products WHERE path LIKE $1 LIMIT 1",
)
.bind(format!("/nix/store/{hash}-%"))
.fetch_optional(&state.pool)
.await
.map_err(|e| ApiError(fc_common::CiError::Database(e)))?;
let product = match product {
Some(p) => p,
None => return Ok(StatusCode::NOT_FOUND.into_response()),
let store_path = match find_store_path(&state.pool, hash).await? {
Some(p) if fc_common::validate::is_valid_store_path(&p) => p,
_ => return Ok(StatusCode::NOT_FOUND.into_response()),
};
if !fc_common::validate::is_valid_store_path(&product.path) {
return Ok(StatusCode::NOT_FOUND.into_response());
}
// Use two piped processes instead of sh -c to prevent command injection
let mut nix_child = std::process::Command::new("nix")
.args(["store", "dump-path", &product.path])
.args(["store", "dump-path", &store_path])
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.spawn()
@ -290,25 +307,13 @@ async fn serve_nar(
return Ok(StatusCode::NOT_FOUND.into_response());
}
let product = sqlx::query_as::<_, fc_common::models::BuildProduct>(
"SELECT * FROM build_products WHERE path LIKE $1 LIMIT 1",
)
.bind(format!("/nix/store/{hash}-%"))
.fetch_optional(&state.pool)
.await
.map_err(|e| ApiError(fc_common::CiError::Database(e)))?;
let product = match product {
Some(p) => p,
None => return Ok(StatusCode::NOT_FOUND.into_response()),
let store_path = match find_store_path(&state.pool, hash).await? {
Some(p) if fc_common::validate::is_valid_store_path(&p) => p,
_ => return Ok(StatusCode::NOT_FOUND.into_response()),
};
if !fc_common::validate::is_valid_store_path(&product.path) {
return Ok(StatusCode::NOT_FOUND.into_response());
}
let child = Command::new("nix")
.args(["store", "dump-path", &product.path])
.args(["store", "dump-path", &store_path])
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.spawn();

20
crates/server/src/routes/channels.rs
View file

@ -50,9 +50,29 @@ async fn create_channel(
input
.validate()
.map_err(|msg| ApiError(fc_common::CiError::Validation(msg)))?;
let jobset_id = input.jobset_id;
let channel = fc_common::repo::channels::create(&state.pool, input)
.await
.map_err(ApiError)?;
// Catch-up: if the jobset already has a completed evaluation, promote now
if let Ok(Some(eval)) =
fc_common::repo::evaluations::get_latest(&state.pool, jobset_id).await
{
if eval.status == fc_common::models::EvaluationStatus::Completed {
let _ = fc_common::repo::channels::auto_promote_if_complete(
&state.pool,
jobset_id,
eval.id,
)
.await;
}
}
// Re-fetch to include any promotion
let channel = fc_common::repo::channels::get(&state.pool, channel.id)
.await
.map_err(ApiError)?;
Ok(Json(channel))
}

7
crates/server/src/routes/dashboard.rs
View file

@ -3,7 +3,7 @@ use axum::{
Form,
Router,
extract::{Path, Query, State},
http::Extensions,
http::{Extensions, StatusCode},
response::{Html, IntoResponse, Redirect, Response},
routing::get,
};
@ -1271,10 +1271,13 @@ async fn login_action(
let tmpl = LoginTemplate {
error: Some("Invalid username or password".to_string()),
};
return Html(
return (
StatusCode::UNAUTHORIZED,
Html(
tmpl
.render()
.unwrap_or_else(|e| format!("Template error: {e}")),
),
)
.into_response();
}

15
crates/server/src/routes/metrics.rs
View file

@ -54,6 +54,15 @@ struct SystemDistributionResponse {
counts: Vec<i64>,
}
/// Escape a string for use as a Prometheus label value.
/// Per the exposition format, backslash, double-quote, and newline must be
/// escaped.
fn escape_prometheus_label(s: &str) -> String {
s.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
}
async fn prometheus_metrics(State(state): State<AppState>) -> Response {
let stats = match fc_common::repo::builds::get_stats(&state.pool).await {
Ok(s) => s,
@ -216,8 +225,9 @@ async fn prometheus_metrics(State(state): State<AppState>) -> Response {
);
output.push_str("# TYPE fc_project_builds_completed gauge\n");
for (name, completed, _) in &per_project {
let escaped = escape_prometheus_label(name);
output.push_str(&format!(
"fc_project_builds_completed{{project=\"{name}\"}} {completed}\n"
"fc_project_builds_completed{{project=\"{escaped}\"}} {completed}\n"
));
}
output.push_str(
@ -225,8 +235,9 @@ async fn prometheus_metrics(State(state): State<AppState>) -> Response {
);
output.push_str("# TYPE fc_project_builds_failed gauge\n");
for (name, _, failed) in &per_project {
let escaped = escape_prometheus_label(name);
output.push_str(&format!(
"fc_project_builds_failed{{project=\"{name}\"}} {failed}\n"
"fc_project_builds_failed{{project=\"{escaped}\"}} {failed}\n"
));
}
}

3
crates/server/src/routes/mod.rs
View file

@ -45,7 +45,6 @@ static STYLE_CSS: &str = include_str!("../../static/style.css");
struct RateLimitState {
requests: DashMap<IpAddr, Vec<Instant>>,
_rps: u64,
burst: u32,
last_cleanup: std::sync::atomic::AtomicU64,
}
@ -180,9 +179,9 @@ pub fn router(state: AppState, config: &ServerConfig) -> Router {
if let (Some(rps), Some(burst)) =
(config.rate_limit_rps, config.rate_limit_burst)
{
let _ = rps; // rate_limit_rps reserved for future use
let rl_state = Arc::new(RateLimitState {
requests: DashMap::new(),
_rps: rps,
burst,
last_cleanup: std::sync::atomic::AtomicU64::new(0),
});

624
crates/server/static/style.css
View file

File diff suppressed because it is too large Load diff

19
flake.nix
View file

@ -85,7 +85,7 @@
checks = forAllSystems (system: let
pkgs = nixpkgs.legacyPackages.${system};
in {
vmTests = {
# Split VM integration tests
service-startup = pkgs.callPackage ./nix/tests/startup.nix {inherit self;};
basic-api = pkgs.callPackage ./nix/tests/basic-api.nix {inherit self;};
@ -95,6 +95,13 @@
webhooks = pkgs.callPackage ./nix/tests/webhooks.nix {inherit self;};
e2e = pkgs.callPackage ./nix/tests/e2e.nix {inherit self;};
declarative = pkgs.callPackage ./nix/tests/declarative.nix {inherit self;};
};
in {
inherit (vmTests) service-startup basic-api auth-rbac api-crud features webhooks e2e declarative;
full = pkgs.symlinkJoin {
name = "vm-tests-full";
paths = builtins.attrValues vmTests;
};
});
devShells = forAllSystems (system: let
@ -125,10 +132,20 @@
runtimeInputs = [
pkgs.alejandra
pkgs.fd
pkgs.prettier
pkgs.deno
pkgs.taplo
];
text = ''
# Format Nix with Alejandra
fd "$@" -t f -e nix -x alejandra -q '{}'
# Format TOML with Taplo
fd "$@" -t f -e toml -x taplo fmt '{}'
# Format CSS with Prettier
fd "$@" -t f -e css -x prettier --write '{}'
'';
});
};

67
nix/demo-vm.nix
View file

@ -1,8 +1,14 @@
{
pkgs,
self,
pkgs,
lib,
}: let
fc-packages = self.packages.${pkgs.stdenv.hostPlatform.system};
# Demo password file to demonstrate passwordFile option
# Password must be at least 12 characters with at least one uppercase letter
demoPasswordFile = pkgs.writeText "demo-password" "DemoPassword123!";
nixos = pkgs.nixos ({
modulesPath,
pkgs,
@ -11,26 +17,12 @@
imports = [
self.nixosModules.fc-ci
(modulesPath + "/virtualisation/qemu-vm.nix")
./vm-common.nix
{config._module.args = {inherit self;};}
];
## VM hardware
virtualisation = {
memorySize = 2048;
cores = 2;
diskSize = 4096;
graphics = false;
# Forward guest:3000 -> host:3000 so the dashboard is reachable
forwardPorts = [
{
from = "host";
host.port = 3000;
guest.port = 3000;
}
];
};
services.fc = {
services.fc-ci = {
enable = true;
package = fc-packages.fc-server;
evaluatorPackage = fc-packages.fc-evaluator;
@ -49,9 +41,22 @@
signing.enabled = false;
server = {
# Bind to all interfaces so port forwarding works
host = "0.0.0.0";
host = lib.mkForce "0.0.0.0";
port = 3000;
cors_permissive = true;
cors_permissive = lib.mkForce true;
};
};
declarative.users = {
admin = {
email = "admin@localhost";
password = "AdminPassword123!";
role = "admin";
};
demo = {
email = "demo@localhost";
role = "read-only";
passwordFile = toString demoPasswordFile;
};
};
};
@ -89,18 +94,22 @@
psql -U fc -d fc -c "INSERT INTO api_keys (name, key_hash, role) VALUES ('demo-readonly', '$RO_HASH', 'read-only') ON CONFLICT DO NOTHING" 2>/dev/null || true
echo ""
echo "==========================================="
echo "====================================================="
echo ""
echo " Dashboard: http://localhost:3000"
echo " Health: http://localhost:3000/health"
echo " API base: http://localhost:3000/api/v1"
echo ""
echo " Admin key: fc_demo_admin_key"
echo " Web login: admin / AdminPassword123! (admin)"
echo " demo / DemoPassword123! (read-only)"
echo ""
echo " Admin API key: fc_demo_admin_key"
echo " Read-only key: fc_demo_readonly_key"
echo ""
echo " Login at http://localhost:3000/login"
echo " using the admin key above."
echo "==========================================="
echo " Login at http://localhost:3000/login using"
echo " the credentials or the API key provided above."
echo ""
echo "====================================================="
'';
};
@ -122,10 +131,12 @@
# Show a helpful MOTD
environment.etc."motd".text = ''
┌──────────────────────────────────────────────┐
┌─────────────────────────────────────────────────────────────┐
│ Dashboard: http://localhost:3000 │
│ API: http://localhost:3000/api/v1 │
│ │
│ Web login: admin / AdminPassword123! (admin) │
│ demo / DemoPassword123! (read-only) │
│ Admin API key: fc_demo_admin_key │
│ Read-only API key: fc_demo_readonly_key │
│ │
@ -136,7 +147,7 @@
│ $ curl -sf localhost:3000/metrics │
│ │
│ Press Ctrl-a x to quit QEMU. │
└──────────────────────────────────────────────┘
└─────────────────────────────────────────────────────────────┘
'';
system.stateVersion = "26.11";

407
nix/modules/nixos.nix
View file

@ -5,33 +5,70 @@
...
}: let
inherit (lib.modules) mkIf mkDefault;
inherit (lib.options) mkOption mkEnableOption;
inherit (lib.types) bool str int package listOf submodule nullOr;
inherit (lib.attrsets) recursiveUpdate optionalAttrs;
inherit (lib.options) mkOption mkEnableOption literalExpression;
inherit (lib.types) bool str int package listOf submodule nullOr enum attrsOf;
inherit (lib.attrsets) recursiveUpdate optionalAttrs mapAttrsToList filterAttrs;
inherit (lib.lists) optional map;
cfg = config.services.fc;
cfg = config.services.fc-ci;
settingsFormat = pkgs.formats.toml {};
settingsType = settingsFormat.type;
# Build the final settings by merging declarative config into settings
finalSettings = recursiveUpdate cfg.settings (optionalAttrs (cfg.declarative.projects != [] || cfg.declarative.apiKeys != []) {
finalSettings = recursiveUpdate cfg.settings (optionalAttrs (cfg.declarative.projects != [] || cfg.declarative.apiKeys != [] || cfg.declarative.users != {} || cfg.declarative.remoteBuilders != []) {
declarative = {
projects =
map (p: {
projects = map (p:
filterAttrs (_: v: v != null) {
name = p.name;
repository_url = p.repositoryUrl;
description = p.description or null;
jobsets =
map (j: {
description = p.description;
jobsets = map (j:
filterAttrs (_: v: v != null) {
name = j.name;
nix_expression = j.nixExpression;
enabled = j.enabled;
flake_mode = j.flakeMode;
check_interval = j.checkInterval;
state = j.state;
branch = j.branch;
scheduling_shares = j.schedulingShares;
inputs = map (i:
filterAttrs (_: v: v != null) {
name = i.name;
input_type = i.inputType;
value = i.value;
revision = i.revision;
})
j.inputs;
})
p.jobsets;
notifications =
map (n: {
notification_type = n.notificationType;
config = n.config;
enabled = n.enabled;
})
p.notifications;
webhooks = map (w:
filterAttrs (_: v: v != null) {
forge_type = w.forgeType;
secret_file = w.secretFile;
enabled = w.enabled;
})
p.webhooks;
channels =
map (c: {
name = c.name;
jobset_name = c.jobsetName;
})
p.channels;
members =
map (m: {
username = m.username;
role = m.role;
})
p.members;
})
cfg.declarative.projects;
@ -42,6 +79,38 @@
role = k.role;
})
cfg.declarative.apiKeys;
users = mapAttrsToList (username: u: let
hasInlinePassword = u.password != null;
_ =
if hasInlinePassword
then builtins.throw "User '${username}' has inline password set. Use passwordFile instead to avoid plaintext passwords in the Nix store."
else null;
in
filterAttrs (_: v: v != null) {
inherit username;
email = u.email;
full_name = u.fullName;
password_file = u.passwordFile;
role = u.role;
enabled = u.enabled;
})
cfg.declarative.users;
remote_builders = map (b:
filterAttrs (_: v: v != null) {
name = b.name;
ssh_uri = b.sshUri;
systems = b.systems;
max_jobs = b.maxJobs;
speed_factor = b.speedFactor;
supported_features = b.supportedFeatures;
mandatory_features = b.mandatoryFeatures;
ssh_key_file = b.sshKeyFile;
public_host_key = b.publicHostKey;
enabled = b.enabled;
})
cfg.declarative.remoteBuilders;
};
});
@ -52,7 +121,7 @@
enabled = mkOption {
type = bool;
default = true;
description = "Whether this jobset is enabled for evaluation.";
description = "Whether this jobset is enabled for evaluation. Deprecated: use `state` instead.";
};
name = mkOption {
@ -62,6 +131,8 @@
nixExpression = mkOption {
type = str;
default = "hydraJobs";
example = literalExpression "packages // checks";
description = "Nix expression to evaluate (e.g. 'packages', 'checks', 'hydraJobs').";
};
@ -76,6 +147,58 @@
default = 60;
description = "Seconds between evaluation checks.";
};
state = mkOption {
type = enum ["disabled" "enabled" "one_shot" "one_at_a_time"];
default = "enabled";
description = ''
Jobset scheduling state:
* `disabled`: Jobset will not be evaluated
* `enabled`: Normal operation, evaluated according to checkInterval
* `one_shot`: Evaluated once, then automatically set to disabled
* `one_at_a_time`: Only one build can run at a time for this jobset
'';
};
branch = mkOption {
type = nullOr str;
default = null;
description = "Git branch to track. Defaults to repository default branch.";
};
schedulingShares = mkOption {
type = int;
default = 100;
description = "Scheduling priority shares. Higher values = more priority.";
};
inputs = mkOption {
type = listOf (submodule {
options = {
name = mkOption {
type = str;
description = "Input name.";
};
inputType = mkOption {
type = str;
default = "git";
description = "Input type: git, string, boolean, path, or build.";
};
value = mkOption {
type = str;
description = "Input value.";
};
revision = mkOption {
type = nullOr str;
default = null;
description = "Git revision (for git inputs).";
};
};
});
default = [];
description = "Jobset inputs for parameterized evaluations.";
};
};
};
@ -102,6 +225,87 @@
default = [];
description = "Jobsets to create for this project.";
};
notifications = mkOption {
type = listOf (submodule {
options = {
notificationType = mkOption {
type = str;
description = "Notification type: github_status, email, gitlab_status, gitea_status, run_command.";
};
config = mkOption {
type = settingsType;
default = {};
description = "Type-specific configuration.";
};
enabled = mkOption {
type = bool;
default = true;
description = "Whether this notification is enabled.";
};
};
});
default = [];
description = "Notification configurations for this project.";
};
webhooks = mkOption {
type = listOf (submodule {
options = {
forgeType = mkOption {
type = enum ["github" "gitea" "gitlab"];
description = "Forge type for webhook.";
};
secretFile = mkOption {
type = nullOr str;
default = null;
description = "Path to file containing webhook secret.";
};
enabled = mkOption {
type = bool;
default = true;
description = "Whether this webhook is enabled.";
};
};
});
default = [];
description = "Webhook configurations for this project.";
};
channels = mkOption {
type = listOf (submodule {
options = {
name = mkOption {
type = str;
description = "Channel name.";
};
jobsetName = mkOption {
type = str;
description = "Name of the jobset this channel tracks.";
};
};
});
default = [];
description = "Release channels for this project.";
};
members = mkOption {
type = listOf (submodule {
options = {
username = mkOption {
type = str;
description = "Username of the member.";
};
role = mkOption {
type = enum ["member" "maintainer" "admin"];
default = "member";
description = "Project role for the member.";
};
};
});
default = [];
description = "Project members with their roles.";
};
};
};
@ -120,11 +324,13 @@
'';
};
# FIXME: should be a list, ideally
role = mkOption {
type = str;
default = "admin";
example = "eval-jobset";
description = ''
Role:
Role, one of:
* admin,
* read-only,
@ -137,10 +343,131 @@
};
};
};
userOpts = {
options = {
enabled = mkOption {
type = bool;
default = true;
description = "Whether this user is enabled.";
};
email = mkOption {
type = str;
description = "User's email address.";
};
fullName = mkOption {
type = nullOr str;
default = null;
description = "Optional full name for the user.";
};
password = mkOption {
type = nullOr str;
default = null;
description = ''
Password provided inline (for dev/testing only).
For production, use {option}`passwordFile` instead.
'';
};
passwordFile = mkOption {
type = nullOr str;
default = null;
description = ''
Path to a file containing the user's password.
Preferred for production deployments.
'';
};
role = mkOption {
type = str;
default = "read-only";
example = "eval-jobset";
description = ''
Role, one of:
* admin,
* read-only,
* create-projects,
* eval-jobset,
* cancel-build,
* restart-jobs,
* bump-to-front.
'';
};
};
};
remoteBuilderOpts = {
options = {
name = mkOption {
type = str;
description = "Unique name for this builder.";
};
sshUri = mkOption {
type = str;
example = "ssh://builder@builder.example.com";
description = "SSH URI for connecting to the builder.";
};
systems = mkOption {
type = listOf str;
default = ["x86_64-linux"];
description = "List of systems this builder supports.";
};
maxJobs = mkOption {
type = int;
default = 1;
description = "Maximum number of parallel jobs.";
};
speedFactor = mkOption {
type = int;
default = 1;
description = "Speed factor for scheduling (higher = faster builder).";
};
supportedFeatures = mkOption {
type = listOf str;
default = [];
description = "List of supported features.";
};
mandatoryFeatures = mkOption {
type = listOf str;
default = [];
description = "List of mandatory features.";
};
sshKeyFile = mkOption {
type = nullOr str;
default = null;
description = "Path to SSH private key file.";
};
publicHostKey = mkOption {
type = nullOr str;
default = null;
description = "SSH public host key for verification.";
};
enabled = mkOption {
type = bool;
default = true;
description = "Whether this builder is enabled.";
};
};
};
in {
options.services.fc = {
options.services.fc-ci = {
enable = mkEnableOption "FC CI system";
# TODO: could we use `mkPackageOption` here?
# Also for the options below
package = mkOption {
type = package;
description = "The FC server package.";
@ -221,6 +548,47 @@ in {
}
];
};
users = mkOption {
type = attrsOf (submodule userOpts);
default = {};
description = ''
Declarative user definitions. The attribute name is the username.
Users are upserted on every server startup.
Use {option}`passwordFile` with a secrets manager for production deployments.
'';
example = {
admin = {
email = "admin@example.com";
passwordFile = "/run/secrets/fc-admin-password";
role = "admin";
};
readonly = {
email = "readonly@example.com";
passwordFile = "/run/secrets/fc-readonly-password";
role = "read-only";
};
};
};
remoteBuilders = mkOption {
type = listOf (submodule remoteBuilderOpts);
default = [];
description = ''
Declarative remote builder definitions. Builders are upserted on every
server startup for distributed builds.
'';
example = [
{
name = "builder1";
sshUri = "ssh://builder@builder.example.com";
systems = ["x86_64-linux" "aarch64-linux"];
maxJobs = 4;
speedFactor = 2;
}
];
};
};
database = {
@ -236,7 +604,7 @@ in {
};
evaluator = {
enable = mkEnableOption "FC evaluator (git polling and nix evaluation)";
enable = mkEnableOption "FC evaluator (Git polling and nix evaluation)";
};
queueRunner = {
@ -245,6 +613,15 @@ in {
};
config = mkIf cfg.enable {
assertions =
mapAttrsToList (
username: user: {
assertion = user.password != null || user.passwordFile != null;
message = "User '${username}' must have either 'password' or 'passwordFile' set.";
}
)
cfg.declarative.users;
users.users.fc = {
isSystemUser = true;
group = "fc";
@ -265,7 +642,7 @@ in {
];
};
services.fc.settings = mkDefault {
services.fc-ci.settings = mkDefault {
database.url = "postgresql:///fc?host=/run/postgresql";
server.host = "127.0.0.1";
server.port = 3000;

4
nix/tests/api-crud.nix
View file

@ -10,7 +10,8 @@ pkgs.testers.nixosTest {
self.nixosModules.fc-ci
../vm-common.nix
];
_module.args.self = self;
config._module.args = {inherit self;};
};
# API CRUD tests: dashboard content, project/jobset/evaluation/build/channel/builder
@ -18,6 +19,7 @@ pkgs.testers.nixosTest {
testScript = ''
import hashlib
import json
import re
machine.start()
machine.wait_for_unit("postgresql.service")

10
nix/tests/auth-rbac.nix
View file

@ -133,7 +133,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
## 3C: API key lifecycle test
# API key lifecycle test
with subtest("API key lifecycle: create, use, delete, verify 401"):
# Create a new key via admin API
result = machine.succeed(
@ -173,7 +173,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "401", f"Expected 401 after key deletion, got {code.strip()}"
# ---- 3D: CRUD lifecycle test ----
# CRUD lifecycle test
with subtest("CRUD lifecycle: project -> jobset -> list -> delete -> 404"):
# Create project
result = machine.succeed(
@ -215,7 +215,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "404", f"Expected 404 after deletion, got {code.strip()}"
# ---- 3E: Edge case tests ----
# Edge case tests
with subtest("Duplicate project name returns 409"):
machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
@ -249,7 +249,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "400", f"Expected 400 for XSS name, got {code.strip()}"
# ---- 3F: Security fuzzing ----
# Security fuzzing
with subtest("SQL injection in search query returns 0 results"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/search?q=test%27%20OR%201%3D1%20--' | jq '.projects | length'"
@ -291,7 +291,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "400", f"Expected 400 for null bytes, got {code.strip()}"
# ---- 3G: Dashboard page smoke tests ----
# Dashboard page smoke tests
with subtest("All dashboard pages return 200"):
pages = ["/", "/projects", "/evaluations", "/builds", "/queue", "/channels", "/admin", "/login"]
for page in pages:

14
nix/tests/basic-api.nix
View file

@ -138,12 +138,14 @@ pkgs.testers.nixosTest {
with subtest("Builds list with combined filters returns 200"):
machine.succeed("curl -sf 'http://127.0.0.1:3000/api/v1/builds?system=x86_64-linux&status=pending&job_name=test' | jq '.items'")
# Metrics endpoint
with subtest("Metrics endpoint returns prometheus format"):
result = machine.succeed("curl -sf http://127.0.0.1:3000/metrics")
assert "fc_builds_total" in result, "Missing fc_builds_total in metrics"
assert "fc_projects_total" in result, "Missing fc_projects_total in metrics"
assert "fc_evaluations_total" in result, "Missing fc_evaluations_total in metrics"
# Prometheus endpoint
with subtest("Prometheus endpoint returns prometheus format"):
result = machine.succeed("curl -sf http://127.0.0.1:3000/prometheus")
machine.succeed(f"echo '{result[:1000]}' > /tmp/metrics.txt")
machine.succeed("echo 'PROMETHEUS OUTPUT:' && cat /tmp/metrics.txt")
assert "fc_builds_total" in result, f"Missing fc_builds_total. Got: {result[:300]}"
assert "fc_projects_total" in result, "Missing fc_projects_total in prometheus metrics"
assert "fc_evaluations_total" in result, "Missing fc_evaluations_total in prometheus metrics"
# CORS: default restrictive (no Access-Control-Allow-Origin for cross-origin)
with subtest("Default CORS does not allow arbitrary origins"):

471
nix/tests/declarative.nix Normal file
View file

@ -0,0 +1,471 @@
{
pkgs,
self,
}: let
fc-packages = self.packages.${pkgs.stdenv.hostPlatform.system};
# Password files for testing passwordFile option
# Passwords must be at least 12 characters with at least one uppercase letter
adminPasswordFile = pkgs.writeText "admin-password" "SecretAdmin123!";
userPasswordFile = pkgs.writeText "user-password" "SecretUser123!";
disabledPasswordFile = pkgs.writeText "disabled-password" "DisabledPass123!";
in
pkgs.testers.nixosTest {
name = "fc-declarative";
nodes.machine = {
imports = [self.nixosModules.fc-ci];
_module.args.self = self;
programs.git.enable = true;
security.sudo.enable = true;
environment.systemPackages = with pkgs; [nix nix-eval-jobs zstd curl jq openssl];
services.fc-ci = {
enable = true;
package = fc-packages.fc-server;
evaluatorPackage = fc-packages.fc-evaluator;
queueRunnerPackage = fc-packages.fc-queue-runner;
migratePackage = fc-packages.fc-migrate-cli;
server.enable = true;
evaluator.enable = true;
queueRunner.enable = true;
settings = {
database.url = "postgresql:///fc?host=/run/postgresql";
server = {
host = "127.0.0.1";
port = 3000;
cors_permissive = false;
};
gc.enabled = false;
logs.log_dir = "/var/lib/fc/logs";
cache.enabled = true;
signing.enabled = false;
};
# Declarative users
declarative.users = {
# Admin user with passwordFile
decl-admin = {
email = "admin@test.local";
passwordFile = toString adminPasswordFile;
role = "admin";
};
# Regular user with passwordFile
decl-user = {
email = "user@test.local";
passwordFile = toString userPasswordFile;
role = "read-only";
};
# User with passwordFile
decl-user2 = {
email = "user2@test.local";
passwordFile = toString userPasswordFile;
role = "read-only";
};
# Disabled user with passwordFile
decl-disabled = {
email = "disabled@test.local";
passwordFile = toString disabledPasswordFile;
role = "read-only";
enabled = false;
};
};
# Declarative API keys
declarative.apiKeys = [
{
name = "decl-admin-key";
key = "fc_decl_admin";
role = "admin";
}
{
name = "decl-readonly-key";
key = "fc_decl_readonly";
role = "read-only";
}
];
# Declarative projects with various jobset states
declarative.projects = [
{
name = "decl-project-1";
repositoryUrl = "https://github.com/test/decl1";
description = "First declarative project";
jobsets = [
{
name = "enabled-jobset";
nixExpression = "packages";
enabled = true;
flakeMode = true;
checkInterval = 300;
state = "enabled";
}
{
name = "disabled-jobset";
nixExpression = "disabled";
state = "disabled";
}
{
name = "oneshot-jobset";
nixExpression = "oneshot";
state = "one_shot";
}
{
name = "oneatatime-jobset";
nixExpression = "exclusive";
state = "one_at_a_time";
checkInterval = 60;
}
];
}
{
name = "decl-project-2";
repositoryUrl = "https://github.com/test/decl2";
jobsets = [
{
name = "main";
nixExpression = ".";
flakeMode = true;
}
];
}
];
};
};
testScript = ''
machine.start()
machine.wait_for_unit("postgresql.service")
machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30)
machine.wait_for_unit("fc-server.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30)
# DECLARATIVE USERS
with subtest("Declarative users are created in database"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM users WHERE username LIKE 'decl-%'\""
)
count = int(result.strip())
assert count == 4, f"Expected 4 declarative users, got {count}"
with subtest("Declarative admin user has admin role"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT role FROM users WHERE username = 'decl-admin'\""
)
assert result.strip() == "admin", f"Expected admin role, got '{result.strip()}'"
with subtest("Declarative regular users have read-only role"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT role FROM users WHERE username = 'decl-user'\""
)
assert result.strip() == "read-only", f"Expected read-only role, got '{result.strip()}'"
with subtest("Declarative disabled user is disabled"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT enabled FROM users WHERE username = 'decl-disabled'\""
)
assert result.strip() == "f", f"Expected disabled (f), got '{result.strip()}'"
with subtest("Declarative enabled users are enabled"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT enabled FROM users WHERE username = 'decl-admin'\""
)
assert result.strip() == "t", f"Expected enabled (t), got '{result.strip()}'"
with subtest("Declarative users have password hashes set"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-admin'\""
)
# Argon2 hashes start with $argon2
assert result.strip().startswith("$argon2"), f"Expected argon2 hash, got '{result.strip()[:20]}...'"
with subtest("User with passwordFile has correct password hash"):
# The password in the file is 'SecretAdmin123!'
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-admin'\""
)
assert len(result.strip()) > 50, "Password hash should be substantial length"
with subtest("User with inline password has correct password hash"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-user'\""
)
assert result.strip().startswith("$argon2"), f"Expected argon2 hash for inline password user, got '{result.strip()[:20]}...'"
# DECLARATIVE USER WEB LOGIN
with subtest("Web login with declarative admin user succeeds"):
# Login via POST to /login with username/password
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-admin&password=SecretAdmin123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Should redirect (302/303) on success
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with declarative user (passwordFile) succeeds"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-user&password=SecretUser123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with declarative user2 (passwordFile) succeeds"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-user2&password=SecretUser123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with wrong password fails"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-admin&password=wrongpassword'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Should return 401 for wrong password
assert code in ("401",), f"Expected 401 for wrong password, got {code}"
with subtest("Web login with disabled user fails"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-disabled&password=DisabledPass123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Disabled user should not be able to login (401 or 403)
assert code in ("401", "403"), f"Expected login failure for disabled user, got {code}"
# DECLARATIVE API KEYS
with subtest("Declarative API keys are created"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM api_keys WHERE name LIKE 'decl-%'\""
)
count = int(result.strip())
assert count == 2, f"Expected 2 declarative API keys, got {count}"
with subtest("Declarative admin API key works"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-H 'Authorization: Bearer fc_decl_admin' "
"http://127.0.0.1:3000/api/v1/projects"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative admin API key can create resources"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/projects "
"-H 'Authorization: Bearer fc_decl_admin' "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"api-created\", \"repository_url\": \"https://example.com/api\"}'"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative read-only API key works for GET"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-H 'Authorization: Bearer fc_decl_readonly' "
"http://127.0.0.1:3000/api/v1/projects"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative read-only API key cannot create resources"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/projects "
"-H 'Authorization: Bearer fc_decl_readonly' "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"should-fail\", \"repository_url\": \"https://example.com/fail\"}'"
)
assert code.strip() == "403", f"Expected 403, got {code.strip()}"
# DECLARATIVE PROJECTS
with subtest("Declarative projects are created"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq '.items | map(select(.name | startswith(\"decl-project\"))) | length'"
)
count = int(result.strip())
assert count == 2, f"Expected 2 declarative projects, got {count}"
with subtest("Declarative project has correct repository URL"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .repository_url'"
)
assert result.strip() == "https://github.com/test/decl1", f"Got '{result.strip()}'"
with subtest("Declarative project has description"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .description'"
)
assert result.strip() == "First declarative project", f"Got '{result.strip()}'"
# DECLARATIVE JOBSETS WITH STATES
with subtest("Declarative project has all jobsets"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq '.items | length'"
)
count = int(result.strip())
assert count == 4, f"Expected 4 jobsets, got {count}"
with subtest("Enabled jobset has state 'enabled'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"enabled-jobset\") | .state'"
)
assert result.strip() == "enabled", f"Expected 'enabled', got '{result.strip()}'"
with subtest("Disabled jobset has state 'disabled'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"disabled-jobset\") | .state'"
)
assert result.strip() == "disabled", f"Expected 'disabled', got '{result.strip()}'"
with subtest("One-shot jobset has state 'one_shot'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneshot-jobset\") | .state'"
)
assert result.strip() == "one_shot", f"Expected 'one_shot', got '{result.strip()}'"
with subtest("One-at-a-time jobset has state 'one_at_a_time'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneatatime-jobset\") | .state'"
)
assert result.strip() == "one_at_a_time", f"Expected 'one_at_a_time', got '{result.strip()}'"
with subtest("Disabled jobset is not in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'disabled-jobset'\""
)
count = int(result.strip())
assert count == 0, f"Disabled jobset should not be in active_jobsets, got {count}"
with subtest("Enabled jobsets are in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'enabled-jobset'\""
)
count = int(result.strip())
assert count == 1, f"Enabled jobset should be in active_jobsets, got {count}"
with subtest("One-shot jobset is in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'oneshot-jobset'\""
)
count = int(result.strip())
assert count == 1, f"One-shot jobset should be in active_jobsets, got {count}"
with subtest("One-at-a-time jobset is in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'oneatatime-jobset'\""
)
count = int(result.strip())
assert count == 1, f"One-at-a-time jobset should be in active_jobsets, got {count}"
with subtest("Jobset check_interval is correctly set"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneatatime-jobset\") | .check_interval'"
)
assert result.strip() == "60", f"Expected check_interval 60, got '{result.strip()}'"
# IDEMPOTENCY
with subtest("Bootstrap is idempotent - no duplicate users"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM users WHERE username = 'decl-admin'\""
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-admin user, got {count}"
with subtest("Bootstrap is idempotent - no duplicate projects"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq '.items | map(select(.name==\"decl-project-1\")) | length'"
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-project-1, got {count}"
with subtest("Bootstrap is idempotent - no duplicate API keys"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM api_keys WHERE name = 'decl-admin-key'\""
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-admin-key, got {count}"
with subtest("Bootstrap is idempotent - no duplicate jobsets"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq '.items | map(select(.name==\"enabled-jobset\")) | length'"
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 enabled-jobset, got {count}"
# USER MANAGEMENT UI (admin-only)
with subtest("Users page requires admin access"):
# Test HTML /users endpoint
htmlResp = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/users"
)
assert "User Management" in htmlResp or "Users" in htmlResp
# Non-admin should be denied access via API
machine.fail(
"curl -sf -H 'Authorization: Bearer fc_decl_readonly' http://127.0.0.1:3000/api/v1/users | grep 'decl-admin'"
)
# Admin should have access via API
adminApiResp = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/api/v1/users"
)
assert "decl-admin" in adminApiResp, "Expected decl-admin in API response"
assert "decl-user" in adminApiResp, "Expected decl-user in API response"
with subtest("Users API shows declarative users for admin"):
# Use the admin API key to list users instead of session-based auth
result = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/api/v1/users"
)
assert "decl-admin" in result, f"Users API should return decl-admin. Got: {result[:500]}"
assert "decl-user" in result, f"Users API should return decl-user. Got: {result[:500]}"
# STARRED JOBS PAGE
with subtest("Starred page exists and returns 200"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/starred"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Starred page shows login prompt when not logged in"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/starred")
assert "Login required" in body or "login" in body.lower(), "Starred page should prompt for login"
'';
}

259
nix/tests/e2e.nix
View file

@ -18,6 +18,8 @@ pkgs.testers.nixosTest {
testScript = ''
import hashlib
import json
import re
import time
machine.start()
machine.wait_for_unit("postgresql.service")
@ -52,30 +54,33 @@ pkgs.testers.nixosTest {
machine.succeed("mkdir -p /var/lib/fc/test-repos")
machine.succeed("git init --bare /var/lib/fc/test-repos/test-flake.git")
# Allow root to push to fc-owned repos (ownership changes after chown below)
machine.succeed("git config --global --add safe.directory /var/lib/fc/test-repos/test-flake.git")
# Create a working copy, write the flake, commit, push
machine.succeed("mkdir -p /tmp/test-flake-work")
machine.succeed("cd /tmp/test-flake-work && git init")
machine.succeed("cd /tmp/test-flake-work && git config user.email 'test@fc' && git config user.name 'FC Test'")
# Write a minimal flake.nix that builds a simple derivation
machine.succeed("""
cat > /tmp/test-flake-work/flake.nix << 'FLAKE'
{
description = "FC CI test flake";
outputs = { self, ... }: {
packages.x86_64-linux.hello = derivation {
name = "fc-test-hello";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo hello > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cat > /tmp/test-flake-work/flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.hello = derivation {\n'
' name = "fc-test-hello";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo hello > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'initial flake'")
machine.succeed("cd /tmp/test-flake-work && git remote add origin /var/lib/fc/test-repos/test-flake.git")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Set ownership for fc user
machine.succeed("chown -R fc:fc /var/lib/fc/test-repos")
@ -86,7 +91,7 @@ pkgs.testers.nixosTest {
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"e2e-test\", \"repository_url\": \"https://github.com/nixos/nixpkgs\"}' "
"-d '{\"name\": \"e2e-test\", \"repository_url\": \"file:///var/lib/fc/test-repos/test-flake.git\"}' "
"| jq -r .id"
)
e2e_project_id = result.strip()
@ -96,7 +101,7 @@ pkgs.testers.nixosTest {
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{e2e_project_id}/jobsets "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 5, \"branch\": null, \"scheduling_shares\": 100}' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 60}' "
"| jq -r .id"
)
e2e_jobset_id = result.strip()
@ -158,24 +163,24 @@ pkgs.testers.nixosTest {
).strip())
# Push a new commit
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake v2";
outputs = { self, ... }: {
packages.x86_64-linux.hello = derivation {
name = "fc-test-hello-v2";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo hello-v2 > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake v2";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.hello = derivation {\n'
' name = "fc-test-hello-v2";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo hello-v2 > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'v2 update'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluator to detect and create new evaluation
machine.wait_until_succeeds(
@ -385,43 +390,38 @@ pkgs.testers.nixosTest {
# Create a new simple build to trigger notification
# Push a trivial change to trigger a new evaluation
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake notify";
outputs = { self, ... }: {
packages.x86_64-linux.notify-test = derivation {
name = "fc-notify-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo notify-test > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger notification test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
# Wait for evaluator to create new evaluation
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={e2e_jobset_id}' "
"| jq '.items | length' | grep -v '^2$'",
timeout=60
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake notify";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.notify-test = derivation {\n'
' name = "fc-notify-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo notify-test > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger notification test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Get the new build ID
notify_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' | jq -r '.items[0].id'"
).strip()
# Wait for the build to complete
# Wait for the notify-test build to complete
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{notify_build_id} | jq -e 'select(.status==\"completed\")'",
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=120
)
# Get the build ID
notify_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' "
"| jq -r '.items[] | select(.status==\"completed\") | .id' | head -1"
).strip()
# Wait a bit for notification to dispatch
time.sleep(5)
@ -455,43 +455,38 @@ pkgs.testers.nixosTest {
with subtest("Signed builds have valid signatures"):
# Create a new build to test signing
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake signing";
outputs = { self, ... }: {
packages.x86_64-linux.sign-test = derivation {
name = "fc-sign-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo signed-build > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger signing test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
# Wait for evaluation
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={e2e_jobset_id}' "
"| jq '.items | length' | grep -v '^[23]$'",
timeout=60
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake signing";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.sign-test = derivation {\n'
' name = "fc-sign-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo signed-build > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger signing test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Get the sign-test build
sign_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' | jq -r '.items[0].id'"
).strip()
# Wait for build to complete
# Wait for the sign-test build to complete
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{sign_build_id} | jq -e 'select(.status==\"completed\")'",
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=120
)
# Get the sign-test build ID
sign_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' "
"| jq -r '.items[] | select(.status==\"completed\") | .id' | head -1"
).strip()
# Verify the build has signed=true
signed = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{sign_build_id} | jq -r .signed"
@ -529,24 +524,24 @@ pkgs.testers.nixosTest {
machine.succeed("chown -R fc:fc /nix/var/nix/gcroots/per-user/fc")
# Create a new build to test GC root creation
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake gc";
outputs = { self, ... }: {
packages.x86_64-linux.gc-test = derivation {
name = "fc-gc-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo gc-test > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake gc";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.gc-test = derivation {\n'
' name = "fc-gc-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo gc-test > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger gc test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluation and build
machine.wait_until_succeeds(
@ -561,24 +556,34 @@ pkgs.testers.nixosTest {
# Verify GC root symlink was created
# The symlink should be in /nix/var/nix/gcroots/per-user/fc/ and point to the build output
# Wait for GC root to be created (polling with timeout)
def wait_for_gc_root():
gc_roots = machine.succeed("find /nix/var/nix/gcroots/per-user/fc -type l 2>/dev/null || true").strip()
# Check if any symlink points to our build output
if gc_roots:
found_root = False
if not gc_roots:
return False
for root in gc_roots.split('\n'):
if root:
target = machine.succeed(f"readlink -f {root} 2>/dev/null || true").strip()
if target == gc_build_output:
found_root = True
break
return True
return False
# We might have GC roots, this is expected behavior
# The key thing is that the build output exists and is protected from GC
machine.succeed(f"test -e {gc_build_output}")
else:
# If no GC roots yet, at least verify the build output exists
# GC roots might be created asynchronously
# Poll for GC root creation (give queue-runner time to create it)
machine.wait_until_succeeds(
"test -e /nix/var/nix/gcroots/per-user/fc",
timeout=30
)
# Wait for a symlink pointing to our build output to appear
import time
found = False
for _ in range(10):
if wait_for_gc_root():
found = True
break
time.sleep(1)
# Verify build output exists and is protected from GC
machine.succeed(f"test -e {gc_build_output}")
with subtest("Declarative .fc.toml in repo auto-creates jobset"):
@ -594,7 +599,7 @@ pkgs.testers.nixosTest {
FCTOML
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'add declarative config'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluator to pick up the new commit and process declarative config
machine.wait_until_succeeds(

1
nix/tests/features.nix
View file

@ -16,6 +16,7 @@ pkgs.testers.nixosTest {
# Feature tests: logging, CSS, setup wizard, probe, metrics improvements
testScript = ''
import hashlib
import re
machine.start()
machine.wait_for_unit("postgresql.service")

206
nix/tests/s3-cache.nix Normal file
View file

@ -0,0 +1,206 @@
{
pkgs,
self,
}:
pkgs.testers.nixosTest {
name = "fc-s3-cache-upload";
nodes.machine = {
imports = [
self.nixosModules.fc-ci
../vm-common.nix
];
_module.args.self = self;
# Add MinIO for S3-compatible storage
services.minio = {
enable = true;
listenAddress = "127.0.0.1:9000";
rootCredentialsFile = pkgs.writeText "minio-root-credentials" ''
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin
'';
};
# Configure FC to upload to the local MinIO instance
services.fc-ci = {
settings = {
cache_upload = {
enabled = true;
store_uri = "s3://fc-cache?endpoint=http://127.0.0.1:9000&region=us-east-1";
s3 = {
region = "us-east-1";
access_key_id = "minioadmin";
secret_access_key = "minioadmin";
endpoint_url = "http://127.0.0.1:9000";
use_path_style = true;
};
};
};
};
};
testScript = ''
import hashlib
import json
import time
machine.start()
# Wait for PostgreSQL
machine.wait_for_unit("postgresql.service")
machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30)
# Wait for MinIO to be ready
machine.wait_for_unit("minio.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:9000/minio/health/live", timeout=30)
# Configure MinIO client and create bucket
machine.succeed("${pkgs.minio-client}/bin/mc alias set local http://127.0.0.1:9000 minioadmin minioadmin")
machine.succeed("${pkgs.minio-client}/bin/mc mb local/fc-cache")
machine.succeed("${pkgs.minio-client}/bin/mc policy set public local/fc-cache")
machine.wait_for_unit("fc-server.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30)
# Seed an API key for write operations
api_token = "fc_testkey123"
api_hash = hashlib.sha256(api_token.encode()).hexdigest()
machine.succeed(
f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('test', '{api_hash}', 'admin')\""
)
auth_header = f"-H 'Authorization: Bearer {api_token}'"
# Create a test flake inside the VM
with subtest("Create bare git repo with test flake"):
machine.succeed("mkdir -p /var/lib/fc/test-repos")
machine.succeed("git init --bare /var/lib/fc/test-repos/s3-test-flake.git")
# Create a working copy, write the flake, commit, push
machine.succeed("mkdir -p /tmp/s3-test-flake")
machine.succeed("cd /tmp/s3-test-flake && git init")
machine.succeed("cd /tmp/s3-test-flake && git config user.email 'test@fc' && git config user.name 'FC Test'")
# Write a minimal flake.nix that builds a simple derivation
machine.succeed("""
cat > /tmp/s3-test-flake/flake.nix << 'FLAKE'
{
description = "FC CI S3 cache test flake";
outputs = { self, ... }: {
packages.x86_64-linux.s3-test = derivation {
name = "fc-s3-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo s3-cache-test-content > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/s3-test-flake && git add -A && git commit -m 'initial flake'")
machine.succeed("cd /tmp/s3-test-flake && git remote add origin /var/lib/fc/test-repos/s3-test-flake.git")
machine.succeed("cd /tmp/s3-test-flake && git push origin HEAD:refs/heads/master")
machine.succeed("chown -R fc:fc /var/lib/fc/test-repos")
# Create project + jobset
with subtest("Create S3 test project and jobset"):
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"s3-test-project\", \"repository_url\": \"file:///var/lib/fc/test-repos/s3-test-flake.git\"}' "
"| jq -r .id"
)
project_id = result.strip()
assert len(project_id) == 36, f"Expected UUID, got '{project_id}'"
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 60}' "
"| jq -r .id"
)
jobset_id = result.strip()
assert len(jobset_id) == 36, f"Expected UUID for jobset, got '{jobset_id}'"
# Wait for evaluator to create evaluation and builds
with subtest("Evaluator discovers and evaluates the flake"):
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={jobset_id}' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=90
)
# Get the build ID
with subtest("Get build ID for s3-test job"):
build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=s3-test' | jq -r '.items[0].id'"
).strip()
assert len(build_id) == 36, f"Expected UUID for build, got '{build_id}'"
# Wait for queue runner to build it
with subtest("Queue runner builds pending derivation"):
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -e 'select(.status==\"completed\")'",
timeout=120
)
# Verify build completed successfully
with subtest("Build completed successfully"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .status"
).strip()
assert result == "completed", f"Expected completed status, got '{result}'"
output_path = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .build_output_path"
).strip()
assert output_path.startswith("/nix/store/"), f"Expected /nix/store/ output path, got '{output_path}'"
# Wait a bit for cache upload to complete (it's async after build)
with subtest("Wait for cache upload to complete"):
time.sleep(5)
# Verify the build output was uploaded to S3
with subtest("Build output was uploaded to S3 cache"):
# List objects in the S3 bucket
bucket_contents = machine.succeed("${pkgs.minio-client}/bin/mc ls --recursive local/fc-cache/")
# Should have the .narinfo file and the .nar file
assert ".narinfo" in bucket_contents, f"Expected .narinfo file in bucket, got: {bucket_contents}"
assert ".nar" in bucket_contents, f"Expected .nar file in bucket, got: {bucket_contents}"
# Verify we can download the narinfo from the S3 bucket
with subtest("Can download narinfo from S3 bucket"):
# Get the store hash from the output path
store_hash = output_path.split('/')[3].split('-')[0]
# Try to get the narinfo from S3
narinfo_content = machine.succeed(
f"curl -sf http://127.0.0.1:9000/fc-cache/{store_hash}.narinfo"
)
assert "StorePath:" in narinfo_content, f"Expected StorePath in narinfo: {narinfo_content}"
assert "NarHash:" in narinfo_content, f"Expected NarHash in narinfo: {narinfo_content}"
# Verify build log mentions cache upload
with subtest("Build log mentions cache upload"):
build_log = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id}/log"
)
# The nix copy output should appear in the log or the system log
# We'll check that the cache upload was attempted by looking at system logs
journal_log = machine.succeed("journalctl -u fc-queue-runner --since '5 minutes ago' --no-pager")
assert "Pushed to binary cache" in journal_log or "nix copy" in journal_log, \
f"Expected cache upload in logs: {journal_log}"
# Cleanup
with subtest("Delete S3 test project"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/projects/{project_id} "
f"{auth_header}"
)
assert code.strip() == "200", f"Expected 200 for project delete, got {code.strip()}"
'';
}

101
nix/vm-common.nix
View file

@ -1,25 +1,56 @@
# Common VM configuration for FC integration tests
{
self,
pkgs,
lib,
...
}: let
inherit (lib.modules) mkDefault;
fc-packages = self.packages.${pkgs.stdenv.hostPlatform.system};
in {
# Common machine configuration for all FC integration tests
config = {
## VM hardware
virtualisation = {
memorySize = 2048;
cores = 2;
diskSize = 10000;
graphics = false;
# Forward guest:3000 -> host:3000 so the dashboard is reachable
forwardPorts = [
{
from = "host";
host.port = 3000;
guest.port = 3000;
}
];
};
# Machine config
programs.git.enable = true;
security.sudo.enable = true;
# Ensure nix and zstd are available for cache endpoints
environment.systemPackages = with pkgs; [nix nix-eval-jobs zstd curl jq openssl];
services.fc = {
# Enable Nix flakes and nix-command experimental features required by evaluator
nix.settings.experimental-features = ["nix-command" "flakes"];
# VM tests have no network. We need to disable substituters to prevent
# Nix from trying to contact cache.nixos.org and timing out each time.
nix.settings.substituters = lib.mkForce [];
# Allow incoming requests on port 3000 to make the dashboard accessible from
# the host machine.
networking.firewall.allowedTCPPorts = [3000];
services.fc-ci = {
enable = true;
package = fc-packages.fc-server;
evaluatorPackage = fc-packages.fc-evaluator;
queueRunnerPackage = fc-packages.fc-queue-runner;
migratePackage = fc-packages.fc-migrate-cli;
package = mkDefault fc-packages.fc-server;
evaluatorPackage = mkDefault fc-packages.fc-evaluator;
queueRunnerPackage = mkDefault fc-packages.fc-queue-runner;
migratePackage = mkDefault fc-packages.fc-migrate-cli;
server.enable = true;
evaluator.enable = true;
@ -45,39 +76,47 @@ in {
show_timestamps = true;
};
evaluator.poll_interval = 5;
evaluator.work_dir = "/var/lib/fc/evaluator";
queue_runner.poll_interval = 3;
queue_runner.work_dir = "/var/lib/fc/queue-runner";
evaluator = {
poll_interval = 5;
work_dir = "/var/lib/fc/evaluator";
nix_timeout = 60;
};
# Declarative bootstrap: project + API key created on startup
declarative = {
projects = [
{
name = "declarative-project";
repository_url = "https://github.com/test/declarative";
description = "Bootstrap test project";
jobsets = [
{
name = "packages";
nix_expression = "packages";
enabled = true;
flake_mode = true;
check_interval = 300;
}
];
}
];
queue_runner = {
poll_interval = 3;
work_dir = "/var/lib/fc/queue-runner";
};
};
api_keys = [
# Declarative configuration for VM tests
# This is set outside of settings so the NixOS module can transform field names
declarative.apiKeys = [
{
name = "bootstrap-admin";
key = "fc_bootstrap_key";
role = "admin";
}
];
};
};
# Declarative project for tests that expect bootstrapped data
# Jobset is disabled so evaluator won't try to fetch from GitHub
declarative.projects = [
{
name = "declarative-project";
repositoryUrl = "https://github.com/test/declarative";
description = "Test declarative project";
jobsets = [
{
name = "packages";
nixExpression = "packages";
flakeMode = true;
enabled = true;
checkInterval = 3600;
state = "disabled"; # disabled: exists but won't be evaluated
}
];
}
];
};
};
}

2216
nix/vm-test.nix
View file

File diff suppressed because it is too large Load diff