{ pkgs, self, }: pkgs.testers.nixosTest { name = "fc-s3-cache-upload"; nodes.machine = { imports = [ self.nixosModules.fc-ci ../vm-common.nix ]; _module.args.self = self; # Add MinIO for S3-compatible storage services.minio = { enable = true; listenAddress = "127.0.0.1:9000"; rootCredentialsFile = pkgs.writeText "minio-root-credentials" '' MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ''; }; # Configure FC to upload to the local MinIO instance services.fc-ci = { settings = { cache_upload = { enabled = true; store_uri = "s3://fc-cache?endpoint=http://127.0.0.1:9000®ion=us-east-1"; s3 = { region = "us-east-1"; access_key_id = "minioadmin"; secret_access_key = "minioadmin"; endpoint_url = "http://127.0.0.1:9000"; use_path_style = true; }; }; }; }; }; testScript = '' import hashlib import json import time machine.start() # Wait for PostgreSQL machine.wait_for_unit("postgresql.service") machine.wait_until_succeeds("sudo -u fc psql -U fc -d fc -c 'SELECT 1'", timeout=30) # Wait for MinIO to be ready machine.wait_for_unit("minio.service") machine.wait_until_succeeds("curl -sf http://127.0.0.1:9000/minio/health/live", timeout=30) # Configure MinIO client and create bucket machine.succeed("${pkgs.minio-client}/bin/mc alias set local http://127.0.0.1:9000 minioadmin minioadmin") machine.succeed("${pkgs.minio-client}/bin/mc mb local/fc-cache") machine.succeed("${pkgs.minio-client}/bin/mc policy set public local/fc-cache") machine.wait_for_unit("fc-server.service") machine.wait_until_succeeds("curl -sf http://127.0.0.1:3000/health", timeout=30) # Seed an API key for write operations api_token = "fc_testkey123" api_hash = hashlib.sha256(api_token.encode()).hexdigest() machine.succeed( f"sudo -u fc psql -U fc -d fc -c \"INSERT INTO api_keys (name, key_hash, role) VALUES ('test', '{api_hash}', 'admin')\"" ) auth_header = f"-H 'Authorization: Bearer {api_token}'" # Create a test flake inside the VM with subtest("Create bare git repo with test flake"): machine.succeed("mkdir -p /var/lib/fc/test-repos") machine.succeed("git init --bare /var/lib/fc/test-repos/s3-test-flake.git") # Create a working copy, write the flake, commit, push machine.succeed("mkdir -p /tmp/s3-test-flake") machine.succeed("cd /tmp/s3-test-flake && git init") machine.succeed("cd /tmp/s3-test-flake && git config user.email 'test@fc' && git config user.name 'FC Test'") # Write a minimal flake.nix that builds a simple derivation machine.succeed(""" cat > /tmp/s3-test-flake/flake.nix << 'FLAKE' { description = "FC CI S3 cache test flake"; outputs = { self, ... }: { packages.x86_64-linux.s3-test = derivation { name = "fc-s3-test"; system = "x86_64-linux"; builder = "/bin/sh"; args = [ "-c" "echo s3-cache-test-content > $out" ]; }; }; } FLAKE """) machine.succeed("cd /tmp/s3-test-flake && git add -A && git commit -m 'initial flake'") machine.succeed("cd /tmp/s3-test-flake && git remote add origin /var/lib/fc/test-repos/s3-test-flake.git") machine.succeed("cd /tmp/s3-test-flake && git push origin HEAD:refs/heads/master") machine.succeed("chown -R fc:fc /var/lib/fc/test-repos") # Create project + jobset with subtest("Create S3 test project and jobset"): result = machine.succeed( "curl -sf -X POST http://127.0.0.1:3000/api/v1/projects " f"{auth_header} " "-H 'Content-Type: application/json' " "-d '{\"name\": \"s3-test-project\", \"repository_url\": \"file:///var/lib/fc/test-repos/s3-test-flake.git\"}' " "| jq -r .id" ) project_id = result.strip() assert len(project_id) == 36, f"Expected UUID, got '{project_id}'" result = machine.succeed( f"curl -sf -X POST http://127.0.0.1:3000/api/v1/projects/{project_id}/jobsets " f"{auth_header} " "-H 'Content-Type: application/json' " "-d '{\"name\": \"packages\", \"nix_expression\": \"packages\", \"flake_mode\": true, \"enabled\": true, \"check_interval\": 60}' " "| jq -r .id" ) jobset_id = result.strip() assert len(jobset_id) == 36, f"Expected UUID for jobset, got '{jobset_id}'" # Wait for evaluator to create evaluation and builds with subtest("Evaluator discovers and evaluates the flake"): machine.wait_until_succeeds( f"curl -sf 'http://127.0.0.1:3000/api/v1/evaluations?jobset_id={jobset_id}' " "| jq -e '.items[] | select(.status==\"completed\")'", timeout=90 ) # Get the build ID with subtest("Get build ID for s3-test job"): build_id = machine.succeed( "curl -sf 'http://127.0.0.1:3000/api/v1/builds?job_name=s3-test' | jq -r '.items[0].id'" ).strip() assert len(build_id) == 36, f"Expected UUID for build, got '{build_id}'" # Wait for queue runner to build it with subtest("Queue runner builds pending derivation"): machine.wait_until_succeeds( f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -e 'select(.status==\"completed\")'", timeout=120 ) # Verify build completed successfully with subtest("Build completed successfully"): result = machine.succeed( f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .status" ).strip() assert result == "completed", f"Expected completed status, got '{result}'" output_path = machine.succeed( f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id} | jq -r .build_output_path" ).strip() assert output_path.startswith("/nix/store/"), f"Expected /nix/store/ output path, got '{output_path}'" # Wait a bit for cache upload to complete (it's async after build) with subtest("Wait for cache upload to complete"): time.sleep(5) # Verify the build output was uploaded to S3 with subtest("Build output was uploaded to S3 cache"): # List objects in the S3 bucket bucket_contents = machine.succeed("${pkgs.minio-client}/bin/mc ls --recursive local/fc-cache/") # Should have the .narinfo file and the .nar file assert ".narinfo" in bucket_contents, f"Expected .narinfo file in bucket, got: {bucket_contents}" assert ".nar" in bucket_contents, f"Expected .nar file in bucket, got: {bucket_contents}" # Verify we can download the narinfo from the S3 bucket with subtest("Can download narinfo from S3 bucket"): # Get the store hash from the output path store_hash = output_path.split('/')[3].split('-')[0] # Try to get the narinfo from S3 narinfo_content = machine.succeed( f"curl -sf http://127.0.0.1:9000/fc-cache/{store_hash}.narinfo" ) assert "StorePath:" in narinfo_content, f"Expected StorePath in narinfo: {narinfo_content}" assert "NarHash:" in narinfo_content, f"Expected NarHash in narinfo: {narinfo_content}" # Verify build log mentions cache upload with subtest("Build log mentions cache upload"): build_log = machine.succeed( f"curl -sf http://127.0.0.1:3000/api/v1/builds/{build_id}/log" ) # The nix copy output should appear in the log or the system log # We'll check that the cache upload was attempted by looking at system logs journal_log = machine.succeed("journalctl -u fc-queue-runner --since '5 minutes ago' --no-pager") assert "Pushed to binary cache" in journal_log or "nix copy" in journal_log, \ f"Expected cache upload in logs: {journal_log}" # Cleanup with subtest("Delete S3 test project"): code = machine.succeed( "curl -s -o /dev/null -w '%{http_code}' " f"-X DELETE http://127.0.0.1:3000/api/v1/projects/{project_id} " f"{auth_header}" ) assert code.strip() == "200", f"Expected 200 for project delete, got {code.strip()}" ''; }