NotAShelf/circus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
circus/nix/tests/api-crud.nix
NotAShelf 7dae114783
 nix: split off monolithic VM test 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: Ifc0a92ed8b6c7622ae345a21880fd0296a6a6964
2026-02-05 22:45:07 +03:00

768 lines
35 KiB
Nix
Raw Blame History

# API CRUD tests: dashboard content, project/jobset/evaluation/build/channel/builder CRUD, admin endpoints, pagination, search
{
pkgs,
fc-packages,
nixosModule,
}:
pkgs.testers.nixosTest {
name = "fc-api-crud";
nodes.machine = import ./common.nix {inherit pkgs fc-packages nixosModule;};
testScript = ''
import hashlib
import json
machine.start()
machine.wait_for_unit("postgresql.service")
# Ensure PostgreSQL is actually ready to accept connections before fc-server starts
machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30)
machine.wait_for_unit("fc-server.service")
# Wait for the server to start listening
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30)
# ---- Seed an API key for write operations ----
# Token: fc_testkey123 -> SHA-256 hash inserted into api_keys table
api_token = "fc_testkey123"
api_hash = hashlib.sha256(api_token.encode()).hexdigest()
machine.succeed(
f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('test', '{api_hash}', 'admin')\""
)
auth_header = f"-H 'Authorization: Bearer {api_token}'"
# Seed a read-only key
ro_token = "fc_readonly_key"
ro_hash = hashlib.sha256(ro_token.encode()).hexdigest()
machine.succeed(
f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('readonly', '{ro_hash}', 'read-only')\""
)
ro_header = f"-H 'Authorization: Bearer {ro_token}'"
# Create initial project for tests
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"test-project\", \"repository_url\": \"https://github.com/test/repo\"}' "
"| jq -r .id"
)
project_id = result.strip()
# ========================================================================
# Phase 4: Dashboard Content & Deep Functional Tests
# ========================================================================
# ---- 4A: Dashboard content verification ----
with subtest("Home page contains Dashboard heading"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/")
assert "Dashboard" in body, "Home page missing 'Dashboard' heading"
with subtest("Home page contains stats grid"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/")
assert "stat-card" in body, "Home page missing stats grid"
assert "Completed" in body, "Home page missing 'Completed' stat"
with subtest("Home page shows project overview table"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/")
# We created projects earlier, they should appear
assert "test-project" in body, "Home page should list test-project in overview"
with subtest("Projects page contains created projects"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/projects")
assert "test-project" in body, "Projects page should list test-project"
with subtest("Projects page returns HTML content type"):
ct = machine.succeed(
"curl -s -D - -o /dev/null http://127.0.0.1:3000/projects | grep -i content-type"
)
assert "text/html" in ct.lower(), f"Expected text/html, got: {ct}"
with subtest("Admin page shows system status"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/admin")
assert "Administration" in body, "Admin page missing heading"
assert "System Status" in body, "Admin page missing system status section"
assert "Remote Builders" in body, "Admin page missing remote builders section"
with subtest("Queue page renders"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/queue")
assert "Queue" in body or "Pending" in body or "Running" in body, \
"Queue page missing expected content"
with subtest("Channels page renders"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/channels")
# Page should render even if empty
assert "Channel" in body or "channel" in body, "Channels page missing expected content"
with subtest("Builds page renders with filter params"):
body = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/builds?status=pending&system=x86_64-linux'"
)
assert "Build" in body or "build" in body, "Builds page missing expected content"
with subtest("Evaluations page renders"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/evaluations")
assert "Evaluation" in body or "evaluation" in body, "Evaluations page missing expected content"
with subtest("Login page contains form"):
body = machine.succeed("curl -sf http://127.0.0.1:3000/login")
assert "api_key" in body or "API" in body, "Login page missing API key input"
assert "<form" in body.lower(), "Login page missing form element"
# ---- 4B: Dashboard page for specific entities ----
with subtest("Project detail page renders for existing project"):
body = machine.succeed(
f"curl -sf http://127.0.0.1:3000/project/{project_id}"
)
assert "test-project" in body, "Project detail page should show project name"
with subtest("Project detail page with invalid UUID returns graceful error"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/project/00000000-0000-0000-0000-000000000000"
)
# Should return 200 with "not found" message or similar, not crash
assert code.strip() == "200", f"Expected 200 for missing project detail, got {code.strip()}"
# ---- 4C: Project update via PUT ----
with subtest("Update project description via PUT"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/projects/{project_id} "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"description\": \"Updated description\"}'"
)
assert code.strip() == "200", f"Expected 200 for PUT project, got {code.strip()}"
with subtest("Updated project reflects new description"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id} | jq -r .description"
)
assert result.strip() == "Updated description", f"Expected updated description, got: {result.strip()}"
with subtest("Update project with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/projects/{project_id} "
f"{ro_header} "
"-H 'Content-Type: application/json' "
"-d '{\"description\": \"Hacked\"}'"
)
assert code.strip() == "403", f"Expected 403 for read-only PUT, got {code.strip()}"
# ---- 4D: Jobset CRUD ----
with subtest("Create jobset for test-project"):
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"main\", \"nix_expression\": \"packages\"}' "
"| jq -r .id"
)
test_jobset_id = result.strip()
assert len(test_jobset_id) == 36, f"Expected UUID for jobset, got '{test_jobset_id}'"
with subtest("List jobsets for project includes new jobset"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets | jq '.items | length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 jobset, got {result.strip()}"
with subtest("Jobset detail page renders"):
body = machine.succeed(
f"curl -sf http://127.0.0.1:3000/jobset/{test_jobset_id}"
)
assert "main" in body, "Jobset detail page should show jobset name"
# ---- 4E: Evaluation trigger and lifecycle ----
with subtest("Trigger evaluation via API"):
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/evaluations/trigger "
f"{auth_header} "
"-H 'Content-Type: application/json' "
f"-d '{{\"jobset_id\": \"{test_jobset_id}\", \"commit_hash\": \"abcdef1234567890abcdef1234567890abcdef12\"}}' "
"| jq -r .id"
)
test_eval_id = result.strip()
assert len(test_eval_id) == 36, f"Expected UUID for evaluation, got '{test_eval_id}'"
with subtest("Get evaluation by ID"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/evaluations/{test_eval_id} | jq -r .status"
)
assert result.strip().lower() == "pending", f"Expected pending status, got: {result.strip()}"
with subtest("List evaluations includes triggered one"):
result = machine.succeed(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={test_jobset_id}' | jq '.items | length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 evaluation, got {result.strip()}"
with subtest("Evaluation detail dashboard page renders"):
body = machine.succeed(
f"curl -sf http://127.0.0.1:3000/evaluation/{test_eval_id}"
)
assert "abcdef123456" in body, "Evaluation page should show commit hash prefix"
with subtest("Trigger evaluation with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/evaluations/trigger "
f"{ro_header} "
"-H 'Content-Type: application/json' "
f"-d '{{\"jobset_id\": \"{test_jobset_id}\", \"commit_hash\": \"0000000000000000000000000000000000000000\"}}'"
)
assert code.strip() == "403", f"Expected 403 for read-only eval trigger, got {code.strip()}"
# ---- 4E2: Build lifecycle (restart, bump) ----
# Create a build via SQL since builds are normally created by the evaluator
with subtest("Create test build via SQL"):
machine.succeed(
"sudo -u fc psql -U fc -d fc -c \""
"INSERT INTO builds (id, evaluation_id, job_name, drv_path, status, system, priority, created_at) "
f"VALUES ('aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', '{test_eval_id}', 'hello', '/nix/store/fake.drv', 'failed', 'x86_64-linux', 5, NOW())"
"\""
)
test_build_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
with subtest("Get build by ID"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/builds/{test_build_id} | jq -r .status"
)
assert result.strip().lower() == "failed", f"Expected failed, got: {result.strip()}"
with subtest("Restart failed build"):
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/builds/{test_build_id}/restart "
f"{auth_header} "
"| jq -r .status"
)
assert result.strip().lower() == "pending", f"Expected pending status for restarted build, got: {result.strip()}"
with subtest("Restart with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X POST http://127.0.0.1:3000/api/v1/builds/{test_build_id}/restart "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only restart, got {code.strip()}"
# Create a pending build to test bump
with subtest("Create pending build for bump test"):
machine.succeed(
"sudo -u fc psql -U fc -d fc -c \""
"INSERT INTO builds (id, evaluation_id, job_name, drv_path, status, system, priority, created_at) "
f"VALUES ('bbbbbbbb-cccc-dddd-eeee-ffffffffffff', '{test_eval_id}', 'world', '/nix/store/fake2.drv', 'pending', 'x86_64-linux', 5, NOW())"
"\""
)
bump_build_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff"
with subtest("Bump build priority"):
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/builds/{bump_build_id}/bump "
f"{auth_header} "
"| jq -r .priority"
)
assert int(result.strip()) == 15, f"Expected priority 15 (5+10), got: {result.strip()}"
with subtest("Bump with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X POST http://127.0.0.1:3000/api/v1/builds/{bump_build_id}/bump "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only bump, got {code.strip()}"
with subtest("Cancel build"):
result = machine.succeed(
f"curl -sf -X POST http://127.0.0.1:3000/api/v1/builds/{bump_build_id}/cancel "
f"{auth_header} "
"| jq '.[0].status'"
)
assert "cancelled" in result.strip().lower(), f"Expected cancelled, got: {result.strip()}"
# ---- 4E3: Evaluation comparison ----
with subtest("Trigger second evaluation for comparison"):
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/evaluations/trigger "
f"{auth_header} "
"-H 'Content-Type: application/json' "
f"-d '{{\"jobset_id\": \"{test_jobset_id}\", \"commit_hash\": \"deadbeef1234567890abcdef1234567890abcdef\"}}' "
"| jq -r .id"
)
second_eval_id = result.strip()
# Add a build to the second evaluation
machine.succeed(
"sudo -u fc psql -U fc -d fc -c \""
"INSERT INTO builds (id, evaluation_id, job_name, drv_path, status, system, priority, created_at) "
f"VALUES ('cccccccc-dddd-eeee-ffff-aaaaaaaaaaaa', '{second_eval_id}', 'hello', '/nix/store/changed.drv', 'pending', 'x86_64-linux', 5, NOW())"
"\""
)
machine.succeed(
"sudo -u fc psql -U fc -d fc -c \""
"INSERT INTO builds (id, evaluation_id, job_name, drv_path, status, system, priority, created_at) "
f"VALUES ('dddddddd-eeee-ffff-aaaa-bbbbbbbbbbbb', '{second_eval_id}', 'new-pkg', '/nix/store/new.drv', 'pending', 'x86_64-linux', 5, NOW())"
"\""
)
with subtest("Compare evaluations shows diff"):
result = machine.succeed(
f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations/{test_eval_id}/compare?to={second_eval_id}'"
)
data = json.loads(result)
# hello changed derivation, world was removed, new-pkg was added
assert len(data["changed_jobs"]) >= 1, f"Expected at least 1 changed job, got {data['changed_jobs']}"
assert len(data["new_jobs"]) >= 1, f"Expected at least 1 new job, got {data['new_jobs']}"
assert any(j["job_name"] == "new-pkg" for j in data["new_jobs"]), "new-pkg should be in new_jobs"
# ---- 4F: Channel CRUD lifecycle ----
with subtest("Create channel via API"):
result = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/channels "
f"{auth_header} "
"-H 'Content-Type: application/json' "
f"-d '{{\"project_id\": \"{project_id}\", \"name\": \"stable\", \"jobset_id\": \"{test_jobset_id}\"}}' "
"| jq -r .id"
)
test_channel_id = result.strip()
assert len(test_channel_id) == 36, f"Expected UUID for channel, got '{test_channel_id}'"
with subtest("List channels includes new channel"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/channels | jq 'length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 channel, got {result.strip()}"
with subtest("Get channel by ID"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/channels/{test_channel_id} | jq -r .name"
)
assert result.strip() == "stable", f"Expected 'stable', got: {result.strip()}"
with subtest("List project channels"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/projects/{project_id}/channels | jq 'length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 project channel, got {result.strip()}"
with subtest("Promote channel to evaluation"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X POST http://127.0.0.1:3000/api/v1/channels/{test_channel_id}/promote/{test_eval_id} "
f"{auth_header}"
)
assert code.strip() == "200", f"Expected 200 for channel promote, got {code.strip()}"
with subtest("Channel promote with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X POST http://127.0.0.1:3000/api/v1/channels/{test_channel_id}/promote/{test_eval_id} "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only promote, got {code.strip()}"
with subtest("Create channel with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/channels "
f"{ro_header} "
"-H 'Content-Type: application/json' "
f"-d '{{\"project_id\": \"{project_id}\", \"name\": \"nightly\", \"jobset_id\": \"{test_jobset_id}\"}}'"
)
assert code.strip() == "403", f"Expected 403 for read-only channel create, got {code.strip()}"
with subtest("Delete channel"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/channels/{test_channel_id} "
f"{auth_header}"
)
assert code.strip() == "200", f"Expected 200 for channel delete, got {code.strip()}"
# ---- 4G: Remote builder CRUD lifecycle ----
with subtest("List remote builders"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/admin/builders | jq 'length'"
)
# We created one earlier in auth tests
assert int(result.strip()) >= 0, f"Expected >= 0 builders, got {result.strip()}"
# Create a builder for testing
machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/api/v1/admin/builders "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"test-builder\", \"ssh_uri\": \"ssh://nix@builder\", \"systems\": [\"x86_64-linux\"], \"max_jobs\": 2}'"
)
with subtest("Get remote builder by ID"):
# Get the first builder's ID
builder_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/admin/builders | jq -r '.[0].id'"
).strip()
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} | jq -r .name"
)
assert result.strip() == "test-builder", f"Expected 'test-builder', got: {result.strip()}"
with subtest("Update remote builder (disable)"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} "
f"{auth_header} "
"-H 'Content-Type: application/json' "
"-d '{\"enabled\": false}'"
)
assert code.strip() == "200", f"Expected 200 for builder update, got {code.strip()}"
with subtest("Updated builder is disabled"):
result = machine.succeed(
f"curl -sf http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} | jq -r .enabled"
)
assert result.strip() == "false", f"Expected false, got: {result.strip()}"
with subtest("Update builder with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} "
f"{ro_header} "
"-H 'Content-Type: application/json' "
"-d '{\"enabled\": true}'"
)
assert code.strip() == "403", f"Expected 403 for read-only builder update, got {code.strip()}"
with subtest("Delete remote builder with read-only key returns 403"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only builder delete, got {code.strip()}"
with subtest("Delete remote builder with admin key"):
# First clear the builder_id from builds that reference it
machine.succeed(
"sudo -u fc psql -U fc -d fc -c "
f"\"UPDATE builds SET builder_id = NULL WHERE builder_id = '{builder_id}'\""
)
# Now delete the builder
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/admin/builders/{builder_id} "
f"{auth_header}"
)
assert code.strip() == "200", f"Expected 200 for builder delete, got {code.strip()}"
# ---- 4H: Admin system status endpoint ----
with subtest("System status endpoint requires admin"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/admin/system "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only system status, got {code.strip()}"
with subtest("System status endpoint returns data with admin key"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/admin/system "
f"{auth_header} "
"| jq .projects_count"
)
assert int(result.strip()) >= 1, f"Expected at least 1 project in system status, got {result.strip()}"
# ---- 4I: API key listing ----
with subtest("List API keys requires admin"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/api-keys "
f"{ro_header}"
)
assert code.strip() == "403", f"Expected 403 for read-only API key list, got {code.strip()}"
with subtest("List API keys returns array with admin key"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/api-keys "
f"{auth_header} "
"| jq 'length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 API key, got {result.strip()}"
# ---- 4J: Badge endpoints ----
with subtest("Badge endpoint returns SVG for unknown project"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/job/nonexistent/main/hello/shield"
)
# Should return 404 or error since project doesn't exist
assert code.strip() in ("404", "500"), f"Expected 404/500 for unknown badge, got {code.strip()}"
with subtest("Badge endpoint returns SVG for existing project"):
# Create a badge-compatible project name lookup
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/job/test-project/main/hello/shield"
)
# Should return 200 with SVG (even if no builds, shows "not found" badge)
assert code.strip() == "200", f"Expected 200 for badge, got {code.strip()}"
with subtest("Badge returns SVG content type"):
ct = machine.succeed(
"curl -s -D - -o /dev/null "
"http://127.0.0.1:3000/api/v1/job/test-project/main/hello/shield "
"| grep -i content-type"
)
assert "image/svg+xml" in ct.lower(), f"Expected SVG content type, got: {ct}"
with subtest("Latest build endpoint for unknown project returns 404"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/job/nonexistent/main/hello/latest"
)
assert code.strip() in ("404", "500"), f"Expected 404/500 for latest build, got {code.strip()}"
# ---- 4K: Pagination tests ----
# Re-verify server is healthy before pagination tests
machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=15)
with subtest("Projects pagination with limit and offset"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/projects?limit=1&offset=0' | jq '.items | length'"
)
assert int(result.strip()) == 1, f"Expected 1 project with limit=1, got {result.strip()}"
with subtest("Projects pagination returns total count"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/projects?limit=1&offset=0' | jq '.total'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 total projects, got {result.strip()}"
with subtest("Builds pagination with limit"):
result = machine.succeed(
"curl -s 'http://127.0.0.1:3000/api/v1/builds?limit=5'"
)
data = json.loads(result)
assert "limit" in data, f"Expected paginated response with 'limit' field, got: {result[:300]}"
assert data["limit"] == 5, f"Expected limit=5, got {data['limit']}"
with subtest("Evaluations pagination with limit"):
result = machine.succeed(
"curl -s 'http://127.0.0.1:3000/api/v1/evaluations?limit=2'"
)
data = json.loads(result)
assert "limit" in data, f"Expected paginated response with 'limit' field, got: {result[:300]}"
assert data["limit"] == 2, f"Expected limit=2, got {data['limit']}"
# ---- 4L: Build sub-resources ----
with subtest("Build steps endpoint returns empty array for nonexistent build"):
result = machine.succeed(
"curl -sf "
"http://127.0.0.1:3000/api/v1/builds/00000000-0000-0000-0000-000000000000/steps"
" | jq 'length'"
)
assert int(result.strip()) == 0, f"Expected empty steps array, got {result.strip()}"
with subtest("Build products endpoint returns empty array for nonexistent build"):
result = machine.succeed(
"curl -sf "
"http://127.0.0.1:3000/api/v1/builds/00000000-0000-0000-0000-000000000000/products"
" | jq 'length'"
)
assert int(result.strip()) == 0, f"Expected empty products array, got {result.strip()}"
with subtest("Build log endpoint for nonexistent build returns 404"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/builds/00000000-0000-0000-0000-000000000000/log"
)
assert code.strip() == "404", f"Expected 404 for nonexistent build log, got {code.strip()}"
# ---- 4M: Search functionality ----
with subtest("Search returns matching projects"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/search?q=test-project' | jq '.projects | length'"
)
assert int(result.strip()) >= 1, f"Expected at least 1 matching project, got {result.strip()}"
with subtest("Search returns empty for nonsense query"):
result = machine.succeed(
"curl -sf 'http://127.0.0.1:3000/api/v1/search?q=zzzznonexistent99999' | jq '.projects | length'"
)
assert result.strip() == "0", f"Expected 0, got {result.strip()}"
# ---- 4N: Content-Type verification for API endpoints ----
with subtest("API endpoints return application/json"):
ct = machine.succeed(
"curl -s -D - -o /dev/null http://127.0.0.1:3000/api/v1/projects | grep -i content-type"
)
assert "application/json" in ct.lower(), f"Expected application/json, got: {ct}"
with subtest("Health endpoint returns application/json"):
ct = machine.succeed(
"curl -s -D - -o /dev/null http://127.0.0.1:3000/health | grep -i content-type"
)
assert "application/json" in ct.lower(), f"Expected application/json, got: {ct}"
with subtest("Metrics endpoint returns text/plain"):
ct = machine.succeed(
"curl -s -D - -o /dev/null http://127.0.0.1:3000/metrics | grep -i content-type"
)
assert "text/plain" in ct.lower() or "text/" in ct.lower(), f"Expected text content type for metrics, got: {ct}"
# ---- 4O: Session/Cookie auth for dashboard ----
with subtest("Login with valid API key sets session cookie"):
result = machine.succeed(
"curl -s -D - -o /dev/null "
"-X POST http://127.0.0.1:3000/login "
f"-d 'api_key={api_token}'"
)
assert "fc_session=" in result, f"Expected fc_session cookie in response: {result}"
assert "HttpOnly" in result, "Expected HttpOnly flag on session cookie"
with subtest("Login with invalid API key shows error"):
body = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/login "
"-d 'api_key=fc_invalid_key'"
)
assert "Invalid" in body or "invalid" in body or "error" in body.lower(), \
f"Expected error message for invalid login: {body[:200]}"
with subtest("Login with empty API key shows error"):
body = machine.succeed(
"curl -sf -X POST http://127.0.0.1:3000/login "
"-d 'api_key='"
)
assert "required" in body.lower() or "error" in body.lower() or "Invalid" in body, \
f"Expected error message for empty login: {body[:200]}"
with subtest("Session cookie grants admin access on dashboard"):
# Login and capture cookie
cookie = machine.succeed(
"curl -s -D - -o /dev/null "
"-X POST http://127.0.0.1:3000/login "
f"-d 'api_key={api_token}' "
"| grep -i set-cookie | head -1"
)
match = re.search(r'fc_session=([^;]+)', cookie)
if match:
session_val = match.group(1)
body = machine.succeed(
f"curl -sf -H 'Cookie: fc_session={session_val}' http://127.0.0.1:3000/admin"
)
# Admin page with session should show API Keys section and admin controls
assert "API Keys" in body, "Admin page with session should show API Keys section"
with subtest("Logout clears session cookie"):
result = machine.succeed(
"curl -s -D - -o /dev/null -X POST http://127.0.0.1:3000/logout"
)
assert "Max-Age=0" in result or "max-age=0" in result.lower(), \
"Logout should set Max-Age=0 to clear cookie"
# ---- 4P: RBAC with create-projects role ----
cp_token = "fc_createprojects_key"
cp_hash = hashlib.sha256(cp_token.encode()).hexdigest()
machine.succeed(
f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('creator', '{cp_hash}', 'create-projects')\""
)
cp_header = f"-H 'Authorization: Bearer {cp_token}'"
with subtest("create-projects role can create project"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/projects "
f"{cp_header} "
"-H 'Content-Type: application/json' "
"-d '{\"name\": \"creator-project\", \"repository_url\": \"https://example.com/creator\"}'"
)
assert code.strip() == "200", f"Expected 200 for create-projects role, got {code.strip()}"
with subtest("create-projects role cannot delete project"):
# Get the new project ID
cp_project_id = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects | jq -r '.items[] | select(.name==\"creator-project\") | .id'"
).strip()
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/projects/{cp_project_id} "
f"{cp_header}"
)
assert code.strip() == "403", f"Expected 403 for create-projects role DELETE, got {code.strip()}"
with subtest("create-projects role cannot update project"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/projects/{cp_project_id} "
f"{cp_header} "
"-H 'Content-Type: application/json' "
"-d '{\"description\": \"hacked\"}'"
)
assert code.strip() == "403", f"Expected 403 for create-projects PUT, got {code.strip()}"
with subtest("create-projects role cannot access admin endpoints"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/admin/system "
f"{cp_header}"
)
assert code.strip() == "403", f"Expected 403 for create-projects system status, got {code.strip()}"
# ---- 4Q: Additional security tests ----
with subtest("DELETE project without auth returns 401"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X DELETE http://127.0.0.1:3000/api/v1/projects/{project_id}"
)
assert code.strip() == "401", f"Expected 401 for unauthenticated DELETE, got {code.strip()}"
with subtest("PUT project without auth returns 401"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
f"-X PUT http://127.0.0.1:3000/api/v1/projects/{project_id} "
"-H 'Content-Type: application/json' "
"-d '{\"description\": \"hacked\"}'"
)
assert code.strip() == "401", f"Expected 401 for unauthenticated PUT, got {code.strip()}"
with subtest("POST channel without auth returns 401"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X POST http://127.0.0.1:3000/api/v1/channels "
"-H 'Content-Type: application/json' "
"-d '{\"project_id\": \"00000000-0000-0000-0000-000000000000\", \"name\": \"x\", \"jobset_id\": \"00000000-0000-0000-0000-000000000000\"}'"
)
assert code.strip() == "401", f"Expected 401 for unauthenticated channel create, got {code.strip()}"
with subtest("API returns JSON error body for 404"):
result = machine.succeed(
"curl -sf http://127.0.0.1:3000/api/v1/projects/00000000-0000-0000-0000-000000000001 2>&1 || "
"curl -s http://127.0.0.1:3000/api/v1/projects/00000000-0000-0000-0000-000000000001"
)
parsed = json.loads(result)
assert "error" in parsed or "error_code" in parsed, f"Expected JSON error body, got: {result}"
with subtest("Nonexistent API route returns 404"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"http://127.0.0.1:3000/api/v1/nonexistent"
)
# Axum returns 404 for unmatched routes
assert code.strip() in ("404", "405"), f"Expected 404/405 for nonexistent route, got {code.strip()}"
with subtest("HEAD request to health returns 200"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' -I http://127.0.0.1:3000/health"
)
assert code.strip() == "200", f"Expected 200 for HEAD /health, got {code.strip()}"
with subtest("OPTIONS request returns valid response"):
code = machine.succeed(
"curl -s -o /dev/null -w '%{http_code}' "
"-X OPTIONS http://127.0.0.1:3000/api/v1/projects"
)
# Axum may return 200, 204, or 405 depending on CORS configuration
assert code.strip() in ("200", "204", "405"), f"Expected 200/204/405 for OPTIONS, got {code.strip()}"
'';
}
Reference in a new issue View git blame Copy permalink