From 133d392df7b7a29ba488d039bba279bc9a766fcc Mon Sep 17 00:00:00 2001 From: NotAShelf Date: Tue, 17 Mar 2026 21:37:37 +0300 Subject: [PATCH] tests: initial integration test setup Signed-off-by: NotAShelf Change-Id: I082de335ed2b2bf53d687d81db0901ef6a6a6964 --- Cargo.lock | 51 ++++++++++ Cargo.toml | 13 ++- tests/backup.rs | 56 +++++++++++ tests/common/mod.rs | 230 ++++++++++++++++++++++++++++++++++++++++++++ tests/filter.rs | 138 ++++++++++++++++++++++++++ tests/manifest.rs | 117 ++++++++++++++++++++++ tests/retry.rs | 121 +++++++++++++++++++++++ 7 files changed, 721 insertions(+), 5 deletions(-) create mode 100644 tests/backup.rs create mode 100644 tests/common/mod.rs create mode 100644 tests/filter.rs create mode 100644 tests/manifest.rs create mode 100644 tests/retry.rs diff --git a/Cargo.lock b/Cargo.lock index 3b9de22..7c2460b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1127,12 +1127,33 @@ dependencies = [ "subtle", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -1832,6 +1853,7 @@ dependencies = [ "chrono", "clap", "color-eyre", + "flate2", "futures", "hex", "regex", @@ -1839,6 +1861,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "tar", "tempfile", "thiserror", "tokio", @@ -1970,6 +1993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2737,6 +2761,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "slab" version = "0.4.12" @@ -3063,6 +3093,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -4025,6 +4066,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 05730c6..132af30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ reqwest = { version = "0.13.2", features = [ "json", "stream" ], defa serde = { version = "1.0.228", features = [ "derive" ] } serde_json = "1.0.149" sqlx = { version = "0.8.6", features = [ "runtime-tokio", "sqlite", "migrate", "uuid", "chrono" ] } +tar = "0.4.44" thiserror = "2.0.18" tokio = { version = "1.50.0", features = [ "full" ] } tokio-util = { version = "0.7.18", features = [ "io", "compat" ] } @@ -28,6 +29,13 @@ tracing-subscriber = { version = "0.3.23", features = [ "json", "env-filter" ] } urlencoding = "2.1.3" uuid = { version = "1.22.0", features = [ "v4", "serde" ] } + +[dev-dependencies] +flate2 = "1.1.0" +tempfile = "3.27.0" +tokio-test = "0.4.5" +wiremock = "0.6.5" + # See: # [lints.clippy] @@ -102,8 +110,3 @@ lto = true opt-level = "z" panic = "abort" strip = "symbols" - -[dev-dependencies] -tempfile = "3.27.0" -tokio-test = "0.4.5" -wiremock = "0.6.5" diff --git a/tests/backup.rs b/tests/backup.rs new file mode 100644 index 0000000..9adbf46 --- /dev/null +++ b/tests/backup.rs @@ -0,0 +1,56 @@ +//! End-to-end backup flow integration test. +#![expect(clippy::expect_used)] +use konservejo::{ + core::{pipeline::Pipeline, types::JobStatus}, + storage::Storage, +}; + +mod common; +use common::{ + MockRepo, + TestContext, + compute_hash, + create_test_archive, + load_config, + setup_mock_server, +}; + +#[tokio::test] +async fn test_backup_single_repository() { + let archive = create_test_archive("test-repo"); + let expected_hash = compute_hash(&archive); + + let repos = vec![MockRepo::new("test-org", "test-repo", archive)]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!(jobs.len(), 1, "Should have exactly 1 job"); + assert_eq!( + jobs[0].status, + JobStatus::Committed, + "Job should be committed" + ); + assert_eq!( + jobs[0].artifact_hash.as_ref(), + expected_hash, + "Hash should match" + ); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..4b12dd5 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,230 @@ +//! Common test utilities for integration tests. + +#![expect(clippy::expect_used)] + +use std::{collections::HashMap, path::Path}; + +use tempfile::{NamedTempFile, TempDir}; +use wiremock::{ + Mock, + MockServer, + ResponseTemplate, + matchers::{header, method, path_regex}, +}; + +/// Configuration for a mock repository. +#[derive(Debug, Clone)] +pub struct MockRepo { + pub owner: String, + pub name: String, + pub archive_bytes: Vec, +} + +impl MockRepo { + pub fn new(owner: &str, name: &str, archive_bytes: Vec) -> Self { + Self { + owner: owner.to_string(), + name: name.to_string(), + archive_bytes, + } + } +} + +/// Sets up a mock Forgejo server with the given repositories. +pub async fn setup_mock_server(repos: Vec) -> (MockServer, String) { + let server = MockServer::start().await; + + // Group repos by organization + let mut org_repos: HashMap> = HashMap::new(); + for repo in repos { + org_repos.entry(repo.owner.clone()).or_default().push(repo); + } + + // Set up organization repo listing endpoints + for (org, repos) in org_repos { + let repos_clone = repos.clone(); + + Mock::given(method("GET")) + .and(path_regex(format!(r"/orgs/{}/repos", regex::escape(&org)))) + .and(header("authorization", "token test-token")) + .and(header("accept", "application/json")) + .respond_with(move |_req: &wiremock::Request| { + let api_repos: Vec = repos_clone + .iter() + .map(|r| { + serde_json::json!({ + "name": r.name, + "default_branch": "main", + "owner": {"login": r.owner} + }) + }) + .collect(); + + ResponseTemplate::new(200).set_body_json(api_repos) + }) + .mount(&server) + .await; + + // Set up archive download endpoints for each repo + for repo in repos { + let archive_bytes = repo.archive_bytes.clone(); + Mock::given(method("GET")) + .and(path_regex(format!( + r"/repos/{}/{}/archive/.*", + regex::escape(&repo.owner), + regex::escape(&repo.name) + ))) + .and(header("authorization", "token test-token")) + .respond_with(move |_req: &wiremock::Request| { + ResponseTemplate::new(200) + .set_body_bytes(archive_bytes.clone()) + .insert_header("content-type", "application/gzip") + }) + .mount(&server) + .await; + } + } + + let url = server.uri(); + (server, url) +} + +/// Test context holding all temporary resources. +pub struct TestContext { + temp_dir: TempDir, + sink_dir: TempDir, + db_file: NamedTempFile, + config_file: NamedTempFile, + mock_url: String, +} + +impl TestContext { + /// Creates a new test context with the given mock server URL. + pub fn new(mock_url: String) -> Self { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let sink_dir = TempDir::new().expect("Failed to create sink dir"); + let db_file = NamedTempFile::new().expect("Failed to create temp db file"); + let config_file = + NamedTempFile::new().expect("Failed to create temp config file"); + + Self { + temp_dir, + sink_dir, + db_file, + config_file, + mock_url, + } + } + + /// Writes a TOML configuration file for the test. + pub fn write_config( + &self, + orgs: &[String], + excludes: &[String], + concurrency: usize, + max_retries: u32, + backoff_ms: u64, + ) { + let orgs_list = orgs + .iter() + .map(|o| format!("\"{o}\"")) + .collect::>() + .join(", "); + let excludes_str = excludes + .iter() + .map(|e| format!("\"{e}\"")) + .collect::>() + .join(", "); + + let config = format!( + r#"[service] +name = "test" +temp_dir = "{}" +state_db_path = "{}" +concurrency_limit = {concurrency} + +[service.retry] +max_retries = {max_retries} +initial_backoff_ms = {backoff_ms} + +[[source]] +type = "forgejo" +id = "test-source" +api_url = "{}" +token = "test-token" + +[source.scope] +organizations = [{orgs_list}] +exclude_repos = [{excludes_str}] + +[[sink]] +type = "filesystem" +id = "test-sink" +path = "{}" +verify_on_write = true +"#, + self.temp_dir.path().display(), + self.db_file.path().display(), + self.mock_url, + self.sink_dir.path().display() + ); + + std::fs::write(self.config_file.path(), config) + .expect("Failed to write config file"); + } + + /// Returns the path to the configuration file. + pub fn config_path(&self) -> &Path { + self.config_file.path() + } + + /// Returns the path to the database file. + pub fn db_path(&self) -> &Path { + self.db_file.path() + } +} + +/// Creates a minimal valid tar.gz archive containing a single file. +pub fn create_test_archive(seed: &str) -> Vec { + use std::io::Write; + + let mut tar_builder = tar::Builder::new(Vec::new()); + + let content = + format!("Test content for seed: {seed}\nThis is deterministic."); + let mut header = tar::Header::new_gnu(); + header + .set_path(format!("{seed}/test.txt")) + .expect("Failed to set tar path"); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + + tar_builder + .append(&header, content.as_bytes()) + .expect("Failed to append to tar"); + + let tar_data = tar_builder + .into_inner() + .expect("Failed to finish tar archive"); + + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&tar_data) + .expect("Failed to compress tar"); + encoder.finish().expect("Failed to finish gzip") +} + +/// Computes the blake3 hash of the given data. +#[allow(dead_code)] +pub fn compute_hash(data: &[u8]) -> String { + blake3::hash(data).to_hex().to_string() +} + +/// Loads the configuration from the test context. +pub fn load_config( + ctx: &TestContext, +) -> anyhow::Result { + konservejo::config::Config::from_file(ctx.config_path()) +} diff --git a/tests/filter.rs b/tests/filter.rs new file mode 100644 index 0000000..d91f63f --- /dev/null +++ b/tests/filter.rs @@ -0,0 +1,138 @@ +//! Repository filtering tests. +#![expect(clippy::expect_used)] + +use konservejo::{core::pipeline::Pipeline, storage::Storage}; + +mod common; +use common::{ + MockRepo, + TestContext, + create_test_archive, + load_config, + setup_mock_server, +}; + +#[tokio::test] +async fn test_exclude_exact_match() { + let archive1 = create_test_archive("include-repo"); + let archive2 = create_test_archive("exclude-repo"); + + let repos = vec![ + MockRepo::new("test-org", "include-repo", archive1), + MockRepo::new("test-org", "exclude-repo", archive2), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config( + &["test-org".to_string()], + &["test-org/exclude-repo".to_string()], + 4, + 1, + 10, + ); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!( + jobs.len(), + 1, + "Should have 1 job (excluded repo should not be backed up)" + ); +} + +#[tokio::test] +async fn test_exclude_wildcard_pattern() { + let archive1 = create_test_archive("org1-repo1"); + let archive2 = create_test_archive("org1-excluded"); + let archive3 = create_test_archive("org2-excluded"); + let archive4 = create_test_archive("org2-kept"); + + let repos = vec![ + MockRepo::new("org1", "repo1", archive1), + MockRepo::new("org1", "excluded", archive2), + MockRepo::new("org2", "excluded", archive3), + MockRepo::new("org2", "kept", archive4), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config( + &["org1".to_string(), "org2".to_string()], + &["*/excluded".to_string()], + 4, + 1, + 10, + ); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!( + jobs.len(), + 2, + "Should have 2 jobs (wildcard exclude should match both orgs)" + ); +} + +#[tokio::test] +async fn test_no_excludes_backs_up_all() { + let archive1 = create_test_archive("repo1"); + let archive2 = create_test_archive("repo2"); + let archive3 = create_test_archive("repo3"); + + let repos = vec![ + MockRepo::new("test-org", "repo1", archive1), + MockRepo::new("test-org", "repo2", archive2), + MockRepo::new("test-org", "repo3", archive3), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!(jobs.len(), 3, "Should have all 3 jobs when no excludes"); +} diff --git a/tests/manifest.rs b/tests/manifest.rs new file mode 100644 index 0000000..e355bb0 --- /dev/null +++ b/tests/manifest.rs @@ -0,0 +1,117 @@ +//! Manifest verification integration test. +#![expect(clippy::expect_used)] + +use konservejo::{ + core::pipeline::Pipeline, + crypto::manifest::{ + compute_merkle_root, + get_inclusion_proof, + verify_inclusion_proof, + }, + storage::Storage, +}; + +mod common; +use common::{ + MockRepo, + TestContext, + create_test_archive, + load_config, + setup_mock_server, +}; + +#[tokio::test] +async fn test_manifest_root_computed_correctly() { + let archive1 = create_test_archive("repo1"); + let archive2 = create_test_archive("repo2"); + + let repos = vec![ + MockRepo::new("test-org", "repo1", archive1), + MockRepo::new("test-org", "repo2", archive2), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!(jobs.len(), 2); + + let leaf_hashes: Vec<_> = + jobs.iter().map(|j| j.artifact_hash.clone()).collect(); + let computed_root = + compute_merkle_root(&leaf_hashes).expect("Failed to compute root"); + + let stored_root = storage + .get_manifest_root(&run_id) + .await + .expect("Failed to get manifest root") + .expect("Should have stored root"); + + assert_eq!( + stored_root.as_ref(), + computed_root.as_ref(), + "Stored root should match computed root" + ); +} + +#[tokio::test] +async fn test_inclusion_proofs_valid() { + let archive1 = create_test_archive("proof-repo1"); + let archive2 = create_test_archive("proof-repo2"); + + let repos = vec![ + MockRepo::new("test-org", "repo1", archive1), + MockRepo::new("test-org", "repo2", archive2), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + let leaf_hashes: Vec<_> = + jobs.iter().map(|j| j.artifact_hash.clone()).collect(); + let computed_root = + compute_merkle_root(&leaf_hashes).expect("Failed to compute root"); + + for (i, job) in jobs.iter().enumerate() { + let proof = + get_inclusion_proof(&leaf_hashes, i).expect("Failed to generate proof"); + + let valid = + verify_inclusion_proof(&computed_root, &job.artifact_hash, &proof); + + assert!(valid, "Proof for job {i} should be valid"); + } +} diff --git a/tests/retry.rs b/tests/retry.rs new file mode 100644 index 0000000..4392807 --- /dev/null +++ b/tests/retry.rs @@ -0,0 +1,121 @@ +//! Retry logic integration tests. +#![expect(clippy::expect_used)] + +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use konservejo::core::pipeline::Pipeline; +use wiremock::{ + Mock, + MockServer, + ResponseTemplate, + matchers::{header, method, path_regex}, +}; + +mod common; +use common::{ + MockRepo, + TestContext, + create_test_archive, + load_config, + setup_mock_server, +}; + +#[tokio::test] +async fn test_no_retry_on_not_found() { + let request_count = Arc::new(AtomicUsize::new(0)); + + let server = MockServer::start().await; + + let count = Arc::clone(&request_count); + Mock::given(method("GET")) + .and(path_regex("/orgs/test-org/repos.*")) + .and(header("authorization", "token test-token")) + .respond_with(move |_req: &wiremock::Request| { + count.fetch_add(1, Ordering::SeqCst); + ResponseTemplate::new(404) + }) + .mount(&server) + .await; + + let ctx = TestContext::new(server.uri()); + ctx.write_config(&["test-org".to_string()], &[], 4, 5, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let result = pipeline.run().await; + + assert!(result.is_err(), "Should fail with 404"); + assert_eq!( + request_count.load(Ordering::SeqCst), + 1, + "Should not retry on 404 (only makes 1 request)" + ); +} + +#[tokio::test] +async fn test_backup_with_multiple_repos() { + let archive1 = create_test_archive("repo1"); + let archive2 = create_test_archive("repo2"); + + let repos = vec![ + MockRepo::new("test-org", "repo1", archive1), + MockRepo::new("test-org", "repo2", archive2), + ]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = konservejo::storage::Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!(jobs.len(), 2, "Should have 2 jobs"); +} + +#[tokio::test] +async fn test_backup_single_repo() { + let archive = create_test_archive("single-repo"); + + let repos = vec![MockRepo::new("test-org", "repo1", archive)]; + let (_server, mock_url) = setup_mock_server(repos).await; + + let ctx = TestContext::new(mock_url); + ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10); + + let config = load_config(&ctx).expect("Failed to load config"); + let pipeline = Pipeline::new(config) + .await + .expect("Failed to create pipeline"); + + let run_id = pipeline.run().await.expect("Backup failed"); + + let db_url = format!("sqlite://{}", ctx.db_path().display()); + let storage = konservejo::storage::Storage::new(&db_url) + .await + .expect("Failed to connect to DB"); + let jobs = storage + .list_jobs_by_run(&run_id) + .await + .expect("Failed to list jobs"); + + assert_eq!(jobs.len(), 1, "Should have 1 job"); +}