NotAShelf/circus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nix: attempt to fix VM tests; general cleanup

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I65f6909ef02ab4599f5b0bbc0930367e6a6a6964
This commit is contained in:
raf 2026-02-14 13:55:07 +03:00
commit a2b638d4db
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
26 changed files with 2320 additions and 2939 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/tests/api-crud.nix
View file

@ -10,7 +10,8 @@ pkgs.testers.nixosTest {
self.nixosModules.fc-ci
../vm-common.nix
];
_module.args.self = self;
config._module.args = {inherit self;};
};
# API CRUD tests: dashboard content, project/jobset/evaluation/build/channel/builder
@ -18,6 +19,7 @@ pkgs.testers.nixosTest {
testScript = ''
import hashlib
import json
import re
machine.start()
machine.wait_for_unit("postgresql.service")

10
nix/tests/auth-rbac.nix
View file

@ -133,7 +133,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
## 3C: API key lifecycle test
# API key lifecycle test
with subtest("API key lifecycle: create, use, delete, verify 401"):
# Create a new key via admin API
result = machine.succeed(
@ -173,7 +173,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "401", f"Expected 401 after key deletion, got {code.strip()}"
# ---- 3D: CRUD lifecycle test ----
# CRUD lifecycle test
with subtest("CRUD lifecycle: project -> jobset -> list -> delete -> 404"):
# Create project
result = machine.succeed(
@ -215,7 +215,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "404", f"Expected 404 after deletion, got {code.strip()}"
# ---- 3E: Edge case tests ----
# Edge case tests
with subtest("Duplicate project name returns 409"):
machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
@ -249,7 +249,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "400", f"Expected 400 for XSS name, got {code.strip()}"
# ---- 3F: Security fuzzing ----
# Security fuzzing
with subtest("SQL injection in search query returns 0 results"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/search?q=test%27%20OR%201%3D1%20--' | jq '.projects | length'"
@ -291,7 +291,7 @@ pkgs.testers.nixosTest {
)
assert code.strip() == "400", f"Expected 400 for null bytes, got {code.strip()}"
# ---- 3G: Dashboard page smoke tests ----
# Dashboard page smoke tests
with subtest("All dashboard pages return 200"):
pages = ["/", "/projects", "/evaluations", "/builds", "/queue", "/channels", "/admin", "/login"]
for page in pages:

14
nix/tests/basic-api.nix
View file

@ -138,12 +138,14 @@ pkgs.testers.nixosTest {
with subtest("Builds list with combined filters returns 200"):
machine.succeed("curl -sf 'http://127.0.0.1:3000/api/v1/builds?system=x86_64-linux&status=pending&job_name=test' | jq '.items'")
# Metrics endpoint
with subtest("Metrics endpoint returns prometheus format"):
result = machine.succeed("curl -sf http://127.0.0.1:3000/metrics")
assert "fc_builds_total" in result, "Missing fc_builds_total in metrics"
assert "fc_projects_total" in result, "Missing fc_projects_total in metrics"
assert "fc_evaluations_total" in result, "Missing fc_evaluations_total in metrics"
# Prometheus endpoint
with subtest("Prometheus endpoint returns prometheus format"):
result = machine.succeed("curl -sf http://127.0.0.1:3000/prometheus")
machine.succeed(f"echo '{result[:1000]}' > /tmp/metrics.txt")
machine.succeed("echo 'PROMETHEUS OUTPUT:' && cat /tmp/metrics.txt")
assert "fc_builds_total" in result, f"Missing fc_builds_total. Got: {result[:300]}"
assert "fc_projects_total" in result, "Missing fc_projects_total in prometheus metrics"
assert "fc_evaluations_total" in result, "Missing fc_evaluations_total in prometheus metrics"
# CORS: default restrictive (no Access-Control-Allow-Origin for cross-origin)
with subtest("Default CORS does not allow arbitrary origins"):

471
nix/tests/declarative.nix Normal file
View file

@ -0,0 +1,471 @@
{
pkgs,
self,
}: let
fc-packages = self.packages.${pkgs.stdenv.hostPlatform.system};
# Password files for testing passwordFile option
# Passwords must be at least 12 characters with at least one uppercase letter
adminPasswordFile = pkgs.writeText "admin-password" "SecretAdmin123!";
userPasswordFile = pkgs.writeText "user-password" "SecretUser123!";
disabledPasswordFile = pkgs.writeText "disabled-password" "DisabledPass123!";
in
pkgs.testers.nixosTest {
name = "fc-declarative";
nodes.machine = {
imports = [self.nixosModules.fc-ci];
_module.args.self = self;
programs.git.enable = true;
security.sudo.enable = true;
environment.systemPackages = with pkgs; [nix nix-eval-jobs zstd curl jq openssl];
services.fc-ci = {
enable = true;
package = fc-packages.fc-server;
evaluatorPackage = fc-packages.fc-evaluator;
queueRunnerPackage = fc-packages.fc-queue-runner;
migratePackage = fc-packages.fc-migrate-cli;
server.enable = true;
evaluator.enable = true;
queueRunner.enable = true;
settings = {
database.url = "postgresql:///fc?host=/run/postgresql";
server = {
host = "127.0.0.1";
port = 3000;
cors_permissive = false;
};
gc.enabled = false;
logs.log_dir = "/var/lib/fc/logs";
cache.enabled = true;
signing.enabled = false;
};
# Declarative users
declarative.users = {
# Admin user with passwordFile
decl-admin = {
email = "admin@test.local";
passwordFile = toString adminPasswordFile;
role = "admin";
};
# Regular user with passwordFile
decl-user = {
email = "user@test.local";
passwordFile = toString userPasswordFile;
role = "read-only";
};
# User with passwordFile
decl-user2 = {
email = "user2@test.local";
passwordFile = toString userPasswordFile;
role = "read-only";
};
# Disabled user with passwordFile
decl-disabled = {
email = "disabled@test.local";
passwordFile = toString disabledPasswordFile;
role = "read-only";
enabled = false;
};
};
# Declarative API keys
declarative.apiKeys = [
{
name = "decl-admin-key";
key = "fc_decl_admin";
role = "admin";
}
{
name = "decl-readonly-key";
key = "fc_decl_readonly";
role = "read-only";
}
];
# Declarative projects with various jobset states
declarative.projects = [
{
name = "decl-project-1";
repositoryUrl = "https://github.com/test/decl1";
description = "First declarative project";
jobsets = [
{
name = "enabled-jobset";
nixExpression = "packages";
enabled = true;
flakeMode = true;
checkInterval = 300;
state = "enabled";
}
{
name = "disabled-jobset";
nixExpression = "disabled";
state = "disabled";
}
{
name = "oneshot-jobset";
nixExpression = "oneshot";
state = "one_shot";
}
{
name = "oneatatime-jobset";
nixExpression = "exclusive";
state = "one_at_a_time";
checkInterval = 60;
}
];
}
{
name = "decl-project-2";
repositoryUrl = "https://github.com/test/decl2";
jobsets = [
{
name = "main";
nixExpression = ".";
flakeMode = true;
}
];
}
];
};
};
testScript = ''
machine.start()
machine.wait_for_unit("postgresql.service")
machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30)
machine.wait_for_unit("fc-server.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30)
# DECLARATIVE USERS
with subtest("Declarative users are created in database"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM users WHERE username LIKE 'decl-%'\""
)
count = int(result.strip())
assert count == 4, f"Expected 4 declarative users, got {count}"
with subtest("Declarative admin user has admin role"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT role FROM users WHERE username = 'decl-admin'\""
)
assert result.strip() == "admin", f"Expected admin role, got '{result.strip()}'"
with subtest("Declarative regular users have read-only role"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT role FROM users WHERE username = 'decl-user'\""
)
assert result.strip() == "read-only", f"Expected read-only role, got '{result.strip()}'"
with subtest("Declarative disabled user is disabled"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT enabled FROM users WHERE username = 'decl-disabled'\""
)
assert result.strip() == "f", f"Expected disabled (f), got '{result.strip()}'"
with subtest("Declarative enabled users are enabled"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT enabled FROM users WHERE username = 'decl-admin'\""
)
assert result.strip() == "t", f"Expected enabled (t), got '{result.strip()}'"
with subtest("Declarative users have password hashes set"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-admin'\""
)
# Argon2 hashes start with $argon2
assert result.strip().startswith("$argon2"), f"Expected argon2 hash, got '{result.strip()[:20]}...'"
with subtest("User with passwordFile has correct password hash"):
# The password in the file is 'SecretAdmin123!'
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-admin'\""
)
assert len(result.strip()) > 50, "Password hash should be substantial length"
with subtest("User with inline password has correct password hash"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT password_hash FROM users WHERE username = 'decl-user'\""
)
assert result.strip().startswith("$argon2"), f"Expected argon2 hash for inline password user, got '{result.strip()[:20]}...'"
# DECLARATIVE USER WEB LOGIN
with subtest("Web login with declarative admin user succeeds"):
# Login via POST to /login with username/password
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-admin&password=SecretAdmin123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Should redirect (302/303) on success
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with declarative user (passwordFile) succeeds"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-user&password=SecretUser123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with declarative user2 (passwordFile) succeeds"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-user2&password=SecretUser123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
assert code in ("200", "302", "303"), f"Expected redirect on login, got {code}"
with subtest("Web login with wrong password fails"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-admin&password=wrongpassword'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Should return 401 for wrong password
assert code in ("401",), f"Expected 401 for wrong password, got {code}"
with subtest("Web login with disabled user fails"):
result = machine.succeed(
"curl -s -w '\\n%{http_code}' "
"-X POST http://127.0.0.1:3000/login "
"-d 'username=decl-disabled&password=DisabledPass123!'"
)
lines = result.strip().split('\n')
code = lines[-1]
# Disabled user should not be able to login (401 or 403)
assert code in ("401", "403"), f"Expected login failure for disabled user, got {code}"
# DECLARATIVE API KEYS
with subtest("Declarative API keys are created"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM api_keys WHERE name LIKE 'decl-%'\""
)
count = int(result.strip())
assert count == 2, f"Expected 2 declarative API keys, got {count}"
with subtest("Declarative admin API key works"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-H 'Authorization: Bearer fc_decl_admin' "
"http://127.0.0.1:3000/api/v1/projects"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative admin API key can create resources"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/projects "
"-H 'Authorization: Bearer fc_decl_admin' "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"api-created\", \"repository_url\": \"https://example.com/api\"}'"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative read-only API key works for GET"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-H 'Authorization: Bearer fc_decl_readonly' "
"http://127.0.0.1:3000/api/v1/projects"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Declarative read-only API key cannot create resources"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/projects "
"-H 'Authorization: Bearer fc_decl_readonly' "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"should-fail\", \"repository_url\": \"https://example.com/fail\"}'"
)
assert code.strip() == "403", f"Expected 403, got {code.strip()}"
# DECLARATIVE PROJECTS
with subtest("Declarative projects are created"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq '.items | map(select(.name | startswith(\"decl-project\"))) | length'"
)
count = int(result.strip())
assert count == 2, f"Expected 2 declarative projects, got {count}"
with subtest("Declarative project has correct repository URL"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .repository_url'"
)
assert result.strip() == "https://github.com/test/decl1", f"Got '{result.strip()}'"
with subtest("Declarative project has description"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .description'"
)
assert result.strip() == "First declarative project", f"Got '{result.strip()}'"
# DECLARATIVE JOBSETS WITH STATES
with subtest("Declarative project has all jobsets"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq '.items | length'"
)
count = int(result.strip())
assert count == 4, f"Expected 4 jobsets, got {count}"
with subtest("Enabled jobset has state 'enabled'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"enabled-jobset\") | .state'"
)
assert result.strip() == "enabled", f"Expected 'enabled', got '{result.strip()}'"
with subtest("Disabled jobset has state 'disabled'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"disabled-jobset\") | .state'"
)
assert result.strip() == "disabled", f"Expected 'disabled', got '{result.strip()}'"
with subtest("One-shot jobset has state 'one_shot'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneshot-jobset\") | .state'"
)
assert result.strip() == "one_shot", f"Expected 'one_shot', got '{result.strip()}'"
with subtest("One-at-a-time jobset has state 'one_at_a_time'"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneatatime-jobset\") | .state'"
)
assert result.strip() == "one_at_a_time", f"Expected 'one_at_a_time', got '{result.strip()}'"
with subtest("Disabled jobset is not in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'disabled-jobset'\""
)
count = int(result.strip())
assert count == 0, f"Disabled jobset should not be in active_jobsets, got {count}"
with subtest("Enabled jobsets are in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'enabled-jobset'\""
)
count = int(result.strip())
assert count == 1, f"Enabled jobset should be in active_jobsets, got {count}"
with subtest("One-shot jobset is in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'oneshot-jobset'\""
)
count = int(result.strip())
assert count == 1, f"One-shot jobset should be in active_jobsets, got {count}"
with subtest("One-at-a-time jobset is in active_jobsets view"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM active_jobsets WHERE name = 'oneatatime-jobset'\""
)
count = int(result.strip())
assert count == 1, f"One-at-a-time jobset should be in active_jobsets, got {count}"
with subtest("Jobset check_interval is correctly set"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq -r '.items[] | select(.name==\"oneatatime-jobset\") | .check_interval'"
)
assert result.strip() == "60", f"Expected check_interval 60, got '{result.strip()}'"
# IDEMPOTENCY
with subtest("Bootstrap is idempotent - no duplicate users"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM users WHERE username = 'decl-admin'\""
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-admin user, got {count}"
with subtest("Bootstrap is idempotent - no duplicate projects"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq '.items | map(select(.name==\"decl-project-1\")) | length'"
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-project-1, got {count}"
with subtest("Bootstrap is idempotent - no duplicate API keys"):
result = machine.succeed(
"sudo -u fc psql -U fc -d fc -t -c \"SELECT COUNT(*) FROM api_keys WHERE name = 'decl-admin-key'\""
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 decl-admin-key, got {count}"
with subtest("Bootstrap is idempotent - no duplicate jobsets"):
project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"decl-project-1\") | .id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq '.items | map(select(.name==\"enabled-jobset\")) | length'"
)
count = int(result.strip())
assert count == 1, f"Expected exactly 1 enabled-jobset, got {count}"
# USER MANAGEMENT UI (admin-only)
with subtest("Users page requires admin access"):
# Test HTML /users endpoint
htmlResp = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/users"
)
assert "User Management" in htmlResp or "Users" in htmlResp
# Non-admin should be denied access via API
machine.fail(
"curl -sf -H 'Authorization: Bearer fc_decl_readonly' http://127.0.0.1:3000/api/v1/users | grep 'decl-admin'"
)
# Admin should have access via API
adminApiResp = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/api/v1/users"
)
assert "decl-admin" in adminApiResp, "Expected decl-admin in API response"
assert "decl-user" in adminApiResp, "Expected decl-user in API response"
with subtest("Users API shows declarative users for admin"):
# Use the admin API key to list users instead of session-based auth
result = machine.succeed(
"curl -sf -H 'Authorization: Bearer fc_decl_admin' http://127.0.0.1:3000/api/v1/users"
)
assert "decl-admin" in result, f"Users API should return decl-admin. Got: {result[:500]}"
assert "decl-user" in result, f"Users API should return decl-user. Got: {result[:500]}"
# STARRED JOBS PAGE
with subtest("Starred page exists and returns 200"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/starred"
)
assert code.strip() == "200", f"Expected 200, got {code.strip()}"
with subtest("Starred page shows login prompt when not logged in"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/starred")
assert "Login required" in body or "login" in body.lower(), "Starred page should prompt for login"
'';
}

263
nix/tests/e2e.nix
View file

@ -18,6 +18,8 @@ pkgs.testers.nixosTest {
testScript = ''
import hashlib
import json
import re
import time
machine.start()
machine.wait_for_unit("postgresql.service")
@ -52,30 +54,33 @@ pkgs.testers.nixosTest {
machine.succeed("mkdir -p /var/lib/fc/test-repos")
machine.succeed("git init --bare /var/lib/fc/test-repos/test-flake.git")
# Allow root to push to fc-owned repos (ownership changes after chown below)
machine.succeed("git config --global --add safe.directory /var/lib/fc/test-repos/test-flake.git")
# Create a working copy, write the flake, commit, push
machine.succeed("mkdir -p /tmp/test-flake-work")
machine.succeed("cd /tmp/test-flake-work && git init")
machine.succeed("cd /tmp/test-flake-work && git config user.email 'test@fc' && git config user.name 'FC Test'")
# Write a minimal flake.nix that builds a simple derivation
machine.succeed("""
cat > /tmp/test-flake-work/flake.nix << 'FLAKE'
{
description = "FC CI test flake";
outputs = { self, ... }: {
packages.x86_64-linux.hello = derivation {
name = "fc-test-hello";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo hello > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cat > /tmp/test-flake-work/flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.hello = derivation {\n'
' name = "fc-test-hello";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo hello > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'initial flake'")
machine.succeed("cd /tmp/test-flake-work && git remote add origin /var/lib/fc/test-repos/test-flake.git")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Set ownership for fc user
machine.succeed("chown -R fc:fc /var/lib/fc/test-repos")
@ -86,7 +91,7 @@ pkgs.testers.nixosTest {
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"e2e-test\", \"repository_url\": \"https://github.com/nixos/nixpkgs\"}' "
"-d '{\"name\": \"e2e-test\", \"repository_url\": \"file:///var/lib/fc/test-repos/test-flake.git\"}' "
"| jq -r .id"
)
e2e_project_id = result.strip()
@ -96,7 +101,7 @@ pkgs.testers.nixosTest {
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{e2e_project_id}/jobsets "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 5, \"branch\": null, \"scheduling_shares\": 100}' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 60}' "
"| jq -r .id"
)
e2e_jobset_id = result.strip()
@ -158,24 +163,24 @@ pkgs.testers.nixosTest {
).strip())
# Push a new commit
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake v2";
outputs = { self, ... }: {
packages.x86_64-linux.hello = derivation {
name = "fc-test-hello-v2";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo hello-v2 > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake v2";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.hello = derivation {\n'
' name = "fc-test-hello-v2";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo hello-v2 > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'v2 update'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluator to detect and create new evaluation
machine.wait_until_succeeds(
@ -385,43 +390,38 @@ pkgs.testers.nixosTest {
# Create a new simple build to trigger notification
# Push a trivial change to trigger a new evaluation
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake notify";
outputs = { self, ... }: {
packages.x86_64-linux.notify-test = derivation {
name = "fc-notify-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo notify-test > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger notification test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
# Wait for evaluator to create new evaluation
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={e2e_jobset_id}' "
"| jq '.items | length' | grep -v '^2$'",
timeout=60
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake notify";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.notify-test = derivation {\n'
' name = "fc-notify-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo notify-test > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger notification test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Get the new build ID
notify_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' | jq -r '.items[0].id'"
).strip()
# Wait for the build to complete
# Wait for the notify-test build to complete
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{notify_build_id} | jq -e 'select(.status==\"completed\")'",
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=120
)
# Get the build ID
notify_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=notify-test' "
"| jq -r '.items[] | select(.status==\"completed\") | .id' | head -1"
).strip()
# Wait a bit for notification to dispatch
time.sleep(5)
@ -455,43 +455,38 @@ pkgs.testers.nixosTest {
with subtest("Signed builds have valid signatures"):
# Create a new build to test signing
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake signing";
outputs = { self, ... }: {
packages.x86_64-linux.sign-test = derivation {
name = "fc-sign-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo signed-build > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger signing test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
# Wait for evaluation
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={e2e_jobset_id}' "
"| jq '.items | length' | grep -v '^[23]$'",
timeout=60
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake signing";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.sign-test = derivation {\n'
' name = "fc-sign-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo signed-build > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger signing test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Get the sign-test build
sign_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' | jq -r '.items[0].id'"
).strip()
# Wait for build to complete
# Wait for the sign-test build to complete
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{sign_build_id} | jq -e 'select(.status==\"completed\")'",
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=120
)
# Get the sign-test build ID
sign_build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=sign-test' "
"| jq -r '.items[] | select(.status==\"completed\") | .id' | head -1"
).strip()
# Verify the build has signed=true
signed = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{sign_build_id} | jq -r .signed"
@ -529,24 +524,24 @@ pkgs.testers.nixosTest {
machine.succeed("chown -R fc:fc /nix/var/nix/gcroots/per-user/fc")
# Create a new build to test GC root creation
machine.succeed("""
cd /tmp/test-flake-work && \
cat > flake.nix << 'FLAKE'
{
description = "FC CI test flake gc";
outputs = { self, ... }: {
packages.x86_64-linux.gc-test = derivation {
name = "fc-gc-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo gc-test > $out" ];
};
};
}
FLAKE
""")
machine.succeed(
"cd /tmp/test-flake-work && \\\n"
"cat > flake.nix << 'FLAKE'\n"
"{\n"
' description = "FC CI test flake gc";\n'
' outputs = { self, ... }: {\n'
' packages.x86_64-linux.gc-test = derivation {\n'
' name = "fc-gc-test";\n'
' system = "x86_64-linux";\n'
' builder = "/bin/sh";\n'
' args = [ "-c" "echo gc-test > $out" ];\n'
" };\n"
" };\n"
"}\n"
"FLAKE\n"
)
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'trigger gc test'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluation and build
machine.wait_until_succeeds(
@ -561,25 +556,35 @@ pkgs.testers.nixosTest {
# Verify GC root symlink was created
# The symlink should be in /nix/var/nix/gcroots/per-user/fc/ and point to the build output
gc_roots = machine.succeed("find /nix/var/nix/gcroots/per-user/fc -type l 2>/dev/null || true").strip()
# Check if any symlink points to our build output
if gc_roots:
found_root = False
# Wait for GC root to be created (polling with timeout)
def wait_for_gc_root():
gc_roots = machine.succeed("find /nix/var/nix/gcroots/per-user/fc -type l 2>/dev/null || true").strip()
if not gc_roots:
return False
for root in gc_roots.split('\n'):
if root:
target = machine.succeed(f"readlink -f {root} 2>/dev/null || true").strip()
if target == gc_build_output:
found_root = True
break
return True
return False
# We might have GC roots, this is expected behavior
# The key thing is that the build output exists and is protected from GC
machine.succeed(f"test -e {gc_build_output}")
else:
# If no GC roots yet, at least verify the build output exists
# GC roots might be created asynchronously
machine.succeed(f"test -e {gc_build_output}")
# Poll for GC root creation (give queue-runner time to create it)
machine.wait_until_succeeds(
"test -e /nix/var/nix/gcroots/per-user/fc",
timeout=30
)
# Wait for a symlink pointing to our build output to appear
import time
found = False
for _ in range(10):
if wait_for_gc_root():
found = True
break
time.sleep(1)
# Verify build output exists and is protected from GC
machine.succeed(f"test -e {gc_build_output}")
with subtest("Declarative .fc.toml in repo auto-creates jobset"):
# Add .fc.toml to the test repo with a new jobset definition
@ -594,7 +599,7 @@ pkgs.testers.nixosTest {
FCTOML
""")
machine.succeed("cd /tmp/test-flake-work && git add -A && git commit -m 'add declarative config'")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/main")
machine.succeed("cd /tmp/test-flake-work && git push origin HEAD:refs/heads/master")
# Wait for evaluator to pick up the new commit and process declarative config
machine.wait_until_succeeds(

1
nix/tests/features.nix
View file

@ -16,6 +16,7 @@ pkgs.testers.nixosTest {
# Feature tests: logging, CSS, setup wizard, probe, metrics improvements
testScript = ''
import hashlib
import re
machine.start()
machine.wait_for_unit("postgresql.service")

206
nix/tests/s3-cache.nix Normal file
View file

@ -0,0 +1,206 @@
{
pkgs,
self,
}:
pkgs.testers.nixosTest {
name = "fc-s3-cache-upload";
nodes.machine = {
imports = [
self.nixosModules.fc-ci
../vm-common.nix
];
_module.args.self = self;
# Add MinIO for S3-compatible storage
services.minio = {
enable = true;
listenAddress = "127.0.0.1:9000";
rootCredentialsFile = pkgs.writeText "minio-root-credentials" ''
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin
'';
};
# Configure FC to upload to the local MinIO instance
services.fc-ci = {
settings = {
cache_upload = {
enabled = true;
store_uri = "s3://fc-cache?endpoint=http://127.0.0.1:9000&region=us-east-1";
s3 = {
region = "us-east-1";
access_key_id = "minioadmin";
secret_access_key = "minioadmin";
endpoint_url = "http://127.0.0.1:9000";
use_path_style = true;
};
};
};
};
};
testScript = ''
import hashlib
import json
import time
machine.start()
# Wait for PostgreSQL
machine.wait_for_unit("postgresql.service")
machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30)
# Wait for MinIO to be ready
machine.wait_for_unit("minio.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:9000/minio/health/live", timeout=30)
# Configure MinIO client and create bucket
machine.succeed("${pkgs.minio-client}/bin/mc alias set local http://127.0.0.1:9000 minioadmin minioadmin")
machine.succeed("${pkgs.minio-client}/bin/mc mb local/fc-cache")
machine.succeed("${pkgs.minio-client}/bin/mc policy set public local/fc-cache")
machine.wait_for_unit("fc-server.service")
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30)
# Seed an API key for write operations
api_token = "fc_testkey123"
api_hash = hashlib.sha256(api_token.encode()).hexdigest()
machine.succeed(
f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('test', '{api_hash}', 'admin')\""
)
auth_header = f"-H 'Authorization: Bearer {api_token}'"
# Create a test flake inside the VM
with subtest("Create bare git repo with test flake"):
machine.succeed("mkdir -p /var/lib/fc/test-repos")
machine.succeed("git init --bare /var/lib/fc/test-repos/s3-test-flake.git")
# Create a working copy, write the flake, commit, push
machine.succeed("mkdir -p /tmp/s3-test-flake")
machine.succeed("cd /tmp/s3-test-flake && git init")
machine.succeed("cd /tmp/s3-test-flake && git config user.email 'test@fc' && git config user.name 'FC Test'")
# Write a minimal flake.nix that builds a simple derivation
machine.succeed("""
cat > /tmp/s3-test-flake/flake.nix << 'FLAKE'
{
description = "FC CI S3 cache test flake";
outputs = { self, ... }: {
packages.x86_64-linux.s3-test = derivation {
name = "fc-s3-test";
system = "x86_64-linux";
builder = "/bin/sh";
args = [ "-c" "echo s3-cache-test-content > $out" ];
};
};
}
FLAKE
""")
machine.succeed("cd /tmp/s3-test-flake && git add -A && git commit -m 'initial flake'")
machine.succeed("cd /tmp/s3-test-flake && git remote add origin /var/lib/fc/test-repos/s3-test-flake.git")
machine.succeed("cd /tmp/s3-test-flake && git push origin HEAD:refs/heads/master")
machine.succeed("chown -R fc:fc /var/lib/fc/test-repos")
# Create project + jobset
with subtest("Create S3 test project and jobset"):
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"s3-test-project\", \"repository_url\": \"file:///var/lib/fc/test-repos/s3-test-flake.git\"}' "
"| jq -r .id"
)
project_id = result.strip()
assert len(project_id) == 36, f"Expected UUID, got '{project_id}'"
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 60}' "
"| jq -r .id"
)
jobset_id = result.strip()
assert len(jobset_id) == 36, f"Expected UUID for jobset, got '{jobset_id}'"
# Wait for evaluator to create evaluation and builds
with subtest("Evaluator discovers and evaluates the flake"):
machine.wait_until_succeeds(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={jobset_id}' "
"| jq -e '.items[] | select(.status==\"completed\")'",
timeout=90
)
# Get the build ID
with subtest("Get build ID for s3-test job"):
build_id = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=s3-test' | jq -r '.items[0].id'"
).strip()
assert len(build_id) == 36, f"Expected UUID for build, got '{build_id}'"
# Wait for queue runner to build it
with subtest("Queue runner builds pending derivation"):
machine.wait_until_succeeds(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -e 'select(.status==\"completed\")'",
timeout=120
)
# Verify build completed successfully
with subtest("Build completed successfully"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .status"
).strip()
assert result == "completed", f"Expected completed status, got '{result}'"
output_path = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .build_output_path"
).strip()
assert output_path.startswith("/nix/store/"), f"Expected /nix/store/ output path, got '{output_path}'"
# Wait a bit for cache upload to complete (it's async after build)
with subtest("Wait for cache upload to complete"):
time.sleep(5)
# Verify the build output was uploaded to S3
with subtest("Build output was uploaded to S3 cache"):
# List objects in the S3 bucket
bucket_contents = machine.succeed("${pkgs.minio-client}/bin/mc ls --recursive local/fc-cache/")
# Should have the .narinfo file and the .nar file
assert ".narinfo" in bucket_contents, f"Expected .narinfo file in bucket, got: {bucket_contents}"
assert ".nar" in bucket_contents, f"Expected .nar file in bucket, got: {bucket_contents}"
# Verify we can download the narinfo from the S3 bucket
with subtest("Can download narinfo from S3 bucket"):
# Get the store hash from the output path
store_hash = output_path.split('/')[3].split('-')[0]
# Try to get the narinfo from S3
narinfo_content = machine.succeed(
f"curl -sf http://127.0.0.1:9000/fc-cache/{store_hash}.narinfo"
)
assert "StorePath:" in narinfo_content, f"Expected StorePath in narinfo: {narinfo_content}"
assert "NarHash:" in narinfo_content, f"Expected NarHash in narinfo: {narinfo_content}"
# Verify build log mentions cache upload
with subtest("Build log mentions cache upload"):
build_log = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id}/log"
)
# The nix copy output should appear in the log or the system log
# We'll check that the cache upload was attempted by looking at system logs
journal_log = machine.succeed("journalctl -u fc-queue-runner --since '5 minutes ago' --no-pager")
assert "Pushed to binary cache" in journal_log or "nix copy" in journal_log, \
f"Expected cache upload in logs: {journal_log}"
# Cleanup
with subtest("Delete S3 test project"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/projects/{project_id} "
f"{auth_header}"
)
assert code.strip() == "200", f"Expected 200 for project delete, got {code.strip()}"
'';
}