NotAShelf/konservejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests: initial integration test setup

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I082de335ed2b2bf53d687d81db0901ef6a6a6964
This commit is contained in:
raf 2026-03-17 21:37:37 +03:00
commit 133d392df7
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
7 changed files with 721 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Cargo.lock generated
View file

@ -1127,12 +1127,33 @@ dependencies = [
"subtle",
]
[[package]]
name = "filetime"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db"
dependencies = [
"cfg-if",
"libc",
"libredox",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "flume"
version = "0.11.1"
@ -1832,6 +1853,7 @@ dependencies = [
"chrono",
"clap",
"color-eyre",
"flate2",
"futures",
"hex",
"regex",
@ -1839,6 +1861,7 @@ dependencies = [
"serde",
"serde_json",
"sqlx",
"tar",
"tempfile",
"thiserror",
"tokio",
@ -1970,6 +1993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
@ -2737,6 +2761,12 @@ dependencies = [
"rand_core",
]
[[package]]
name = "simd-adler32"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "slab"
version = "0.4.12"
@ -3063,6 +3093,17 @@ dependencies = [
"syn",
]
[[package]]
name = "tar"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
dependencies = [
"filetime",
"libc",
"xattr",
]
[[package]]
name = "tempfile"
version = "3.27.0"
@ -4025,6 +4066,16 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xattr"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [
"libc",
"rustix",
]
[[package]]
name = "xmlparser"
version = "0.13.6"

13
Cargo.toml
View file

@ -19,6 +19,7 @@ reqwest = { version = "0.13.2", features = [ "json", "stream" ], defa
serde = { version = "1.0.228", features = [ "derive" ] }
serde_json = "1.0.149"
sqlx = { version = "0.8.6", features = [ "runtime-tokio", "sqlite", "migrate", "uuid", "chrono" ] }
tar = "0.4.44"
thiserror = "2.0.18"
tokio = { version = "1.50.0", features = [ "full" ] }
tokio-util = { version = "0.7.18", features = [ "io", "compat" ] }
@ -28,6 +29,13 @@ tracing-subscriber = { version = "0.3.23", features = [ "json", "env-filter" ] }
urlencoding = "2.1.3"
uuid = { version = "1.22.0", features = [ "v4", "serde" ] }
[dev-dependencies]
flate2 = "1.1.0"
tempfile = "3.27.0"
tokio-test = "0.4.5"
wiremock = "0.6.5"
# See:
# <https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html>
[lints.clippy]
@ -102,8 +110,3 @@ lto = true
opt-level = "z"
panic = "abort"
strip = "symbols"
[dev-dependencies]
tempfile = "3.27.0"
tokio-test = "0.4.5"
wiremock = "0.6.5"

56
tests/backup.rs Normal file
View file

@ -0,0 +1,56 @@
//! End-to-end backup flow integration test.
#![expect(clippy::expect_used)]
use konservejo::{
core::{pipeline::Pipeline, types::JobStatus},
storage::Storage,
};
mod common;
use common::{
MockRepo,
TestContext,
compute_hash,
create_test_archive,
load_config,
setup_mock_server,
};
#[tokio::test]
async fn test_backup_single_repository() {
let archive = create_test_archive("test-repo");
let expected_hash = compute_hash(&archive);
let repos = vec![MockRepo::new("test-org", "test-repo", archive)];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(jobs.len(), 1, "Should have exactly 1 job");
assert_eq!(
jobs[0].status,
JobStatus::Committed,
"Job should be committed"
);
assert_eq!(
jobs[0].artifact_hash.as_ref(),
expected_hash,
"Hash should match"
);
}

230
tests/common/mod.rs Normal file
View file

@ -0,0 +1,230 @@
//! Common test utilities for integration tests.
#![expect(clippy::expect_used)]
use std::{collections::HashMap, path::Path};
use tempfile::{NamedTempFile, TempDir};
use wiremock::{
Mock,
MockServer,
ResponseTemplate,
matchers::{header, method, path_regex},
};
/// Configuration for a mock repository.
#[derive(Debug, Clone)]
pub struct MockRepo {
pub owner: String,
pub name: String,
pub archive_bytes: Vec<u8>,
}
impl MockRepo {
pub fn new(owner: &str, name: &str, archive_bytes: Vec<u8>) -> Self {
Self {
owner: owner.to_string(),
name: name.to_string(),
archive_bytes,
}
}
}
/// Sets up a mock Forgejo server with the given repositories.
pub async fn setup_mock_server(repos: Vec<MockRepo>) -> (MockServer, String) {
let server = MockServer::start().await;
// Group repos by organization
let mut org_repos: HashMap<String, Vec<MockRepo>> = HashMap::new();
for repo in repos {
org_repos.entry(repo.owner.clone()).or_default().push(repo);
}
// Set up organization repo listing endpoints
for (org, repos) in org_repos {
let repos_clone = repos.clone();
Mock::given(method("GET"))
.and(path_regex(format!(r"/orgs/{}/repos", regex::escape(&org))))
.and(header("authorization", "token test-token"))
.and(header("accept", "application/json"))
.respond_with(move |_req: &wiremock::Request| {
let api_repos: Vec<serde_json::Value> = repos_clone
.iter()
.map(|r| {
serde_json::json!({
"name": r.name,
"default_branch": "main",
"owner": {"login": r.owner}
})
})
.collect();
ResponseTemplate::new(200).set_body_json(api_repos)
})
.mount(&server)
.await;
// Set up archive download endpoints for each repo
for repo in repos {
let archive_bytes = repo.archive_bytes.clone();
Mock::given(method("GET"))
.and(path_regex(format!(
r"/repos/{}/{}/archive/.*",
regex::escape(&repo.owner),
regex::escape(&repo.name)
)))
.and(header("authorization", "token test-token"))
.respond_with(move |_req: &wiremock::Request| {
ResponseTemplate::new(200)
.set_body_bytes(archive_bytes.clone())
.insert_header("content-type", "application/gzip")
})
.mount(&server)
.await;
}
}
let url = server.uri();
(server, url)
}
/// Test context holding all temporary resources.
pub struct TestContext {
temp_dir: TempDir,
sink_dir: TempDir,
db_file: NamedTempFile,
config_file: NamedTempFile,
mock_url: String,
}
impl TestContext {
/// Creates a new test context with the given mock server URL.
pub fn new(mock_url: String) -> Self {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let sink_dir = TempDir::new().expect("Failed to create sink dir");
let db_file = NamedTempFile::new().expect("Failed to create temp db file");
let config_file =
NamedTempFile::new().expect("Failed to create temp config file");
Self {
temp_dir,
sink_dir,
db_file,
config_file,
mock_url,
}
}
/// Writes a TOML configuration file for the test.
pub fn write_config(
&self,
orgs: &[String],
excludes: &[String],
concurrency: usize,
max_retries: u32,
backoff_ms: u64,
) {
let orgs_list = orgs
.iter()
.map(|o| format!("\"{o}\""))
.collect::<Vec<_>>()
.join(", ");
let excludes_str = excludes
.iter()
.map(|e| format!("\"{e}\""))
.collect::<Vec<_>>()
.join(", ");
let config = format!(
r#"[service]
name = "test"
temp_dir = "{}"
state_db_path = "{}"
concurrency_limit = {concurrency}
[service.retry]
max_retries = {max_retries}
initial_backoff_ms = {backoff_ms}
[[source]]
type = "forgejo"
id = "test-source"
api_url = "{}"
token = "test-token"
[source.scope]
organizations = [{orgs_list}]
exclude_repos = [{excludes_str}]
[[sink]]
type = "filesystem"
id = "test-sink"
path = "{}"
verify_on_write = true
"#,
self.temp_dir.path().display(),
self.db_file.path().display(),
self.mock_url,
self.sink_dir.path().display()
);
std::fs::write(self.config_file.path(), config)
.expect("Failed to write config file");
}
/// Returns the path to the configuration file.
pub fn config_path(&self) -> &Path {
self.config_file.path()
}
/// Returns the path to the database file.
pub fn db_path(&self) -> &Path {
self.db_file.path()
}
}
/// Creates a minimal valid tar.gz archive containing a single file.
pub fn create_test_archive(seed: &str) -> Vec<u8> {
use std::io::Write;
let mut tar_builder = tar::Builder::new(Vec::new());
let content =
format!("Test content for seed: {seed}\nThis is deterministic.");
let mut header = tar::Header::new_gnu();
header
.set_path(format!("{seed}/test.txt"))
.expect("Failed to set tar path");
header.set_size(content.len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar_builder
.append(&header, content.as_bytes())
.expect("Failed to append to tar");
let tar_data = tar_builder
.into_inner()
.expect("Failed to finish tar archive");
let mut encoder =
flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder
.write_all(&tar_data)
.expect("Failed to compress tar");
encoder.finish().expect("Failed to finish gzip")
}
/// Computes the blake3 hash of the given data.
#[allow(dead_code)]
pub fn compute_hash(data: &[u8]) -> String {
blake3::hash(data).to_hex().to_string()
}
/// Loads the configuration from the test context.
pub fn load_config(
ctx: &TestContext,
) -> anyhow::Result<konservejo::config::Config> {
konservejo::config::Config::from_file(ctx.config_path())
}

138
tests/filter.rs Normal file
View file

@ -0,0 +1,138 @@
//! Repository filtering tests.
#![expect(clippy::expect_used)]
use konservejo::{core::pipeline::Pipeline, storage::Storage};
mod common;
use common::{
MockRepo,
TestContext,
create_test_archive,
load_config,
setup_mock_server,
};
#[tokio::test]
async fn test_exclude_exact_match() {
let archive1 = create_test_archive("include-repo");
let archive2 = create_test_archive("exclude-repo");
let repos = vec![
MockRepo::new("test-org", "include-repo", archive1),
MockRepo::new("test-org", "exclude-repo", archive2),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(
&["test-org".to_string()],
&["test-org/exclude-repo".to_string()],
4,
1,
10,
);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(
jobs.len(),
1,
"Should have 1 job (excluded repo should not be backed up)"
);
}
#[tokio::test]
async fn test_exclude_wildcard_pattern() {
let archive1 = create_test_archive("org1-repo1");
let archive2 = create_test_archive("org1-excluded");
let archive3 = create_test_archive("org2-excluded");
let archive4 = create_test_archive("org2-kept");
let repos = vec![
MockRepo::new("org1", "repo1", archive1),
MockRepo::new("org1", "excluded", archive2),
MockRepo::new("org2", "excluded", archive3),
MockRepo::new("org2", "kept", archive4),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(
&["org1".to_string(), "org2".to_string()],
&["*/excluded".to_string()],
4,
1,
10,
);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(
jobs.len(),
2,
"Should have 2 jobs (wildcard exclude should match both orgs)"
);
}
#[tokio::test]
async fn test_no_excludes_backs_up_all() {
let archive1 = create_test_archive("repo1");
let archive2 = create_test_archive("repo2");
let archive3 = create_test_archive("repo3");
let repos = vec![
MockRepo::new("test-org", "repo1", archive1),
MockRepo::new("test-org", "repo2", archive2),
MockRepo::new("test-org", "repo3", archive3),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(jobs.len(), 3, "Should have all 3 jobs when no excludes");
}

117
tests/manifest.rs Normal file
View file

@ -0,0 +1,117 @@
//! Manifest verification integration test.
#![expect(clippy::expect_used)]
use konservejo::{
core::pipeline::Pipeline,
crypto::manifest::{
compute_merkle_root,
get_inclusion_proof,
verify_inclusion_proof,
},
storage::Storage,
};
mod common;
use common::{
MockRepo,
TestContext,
create_test_archive,
load_config,
setup_mock_server,
};
#[tokio::test]
async fn test_manifest_root_computed_correctly() {
let archive1 = create_test_archive("repo1");
let archive2 = create_test_archive("repo2");
let repos = vec![
MockRepo::new("test-org", "repo1", archive1),
MockRepo::new("test-org", "repo2", archive2),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(jobs.len(), 2);
let leaf_hashes: Vec<_> =
jobs.iter().map(|j| j.artifact_hash.clone()).collect();
let computed_root =
compute_merkle_root(&leaf_hashes).expect("Failed to compute root");
let stored_root = storage
.get_manifest_root(&run_id)
.await
.expect("Failed to get manifest root")
.expect("Should have stored root");
assert_eq!(
stored_root.as_ref(),
computed_root.as_ref(),
"Stored root should match computed root"
);
}
#[tokio::test]
async fn test_inclusion_proofs_valid() {
let archive1 = create_test_archive("proof-repo1");
let archive2 = create_test_archive("proof-repo2");
let repos = vec![
MockRepo::new("test-org", "repo1", archive1),
MockRepo::new("test-org", "repo2", archive2),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
let leaf_hashes: Vec<_> =
jobs.iter().map(|j| j.artifact_hash.clone()).collect();
let computed_root =
compute_merkle_root(&leaf_hashes).expect("Failed to compute root");
for (i, job) in jobs.iter().enumerate() {
let proof =
get_inclusion_proof(&leaf_hashes, i).expect("Failed to generate proof");
let valid =
verify_inclusion_proof(&computed_root, &job.artifact_hash, &proof);
assert!(valid, "Proof for job {i} should be valid");
}
}

121
tests/retry.rs Normal file
View file

@ -0,0 +1,121 @@
//! Retry logic integration tests.
#![expect(clippy::expect_used)]
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use konservejo::core::pipeline::Pipeline;
use wiremock::{
Mock,
MockServer,
ResponseTemplate,
matchers::{header, method, path_regex},
};
mod common;
use common::{
MockRepo,
TestContext,
create_test_archive,
load_config,
setup_mock_server,
};
#[tokio::test]
async fn test_no_retry_on_not_found() {
let request_count = Arc::new(AtomicUsize::new(0));
let server = MockServer::start().await;
let count = Arc::clone(&request_count);
Mock::given(method("GET"))
.and(path_regex("/orgs/test-org/repos.*"))
.and(header("authorization", "token test-token"))
.respond_with(move |_req: &wiremock::Request| {
count.fetch_add(1, Ordering::SeqCst);
ResponseTemplate::new(404)
})
.mount(&server)
.await;
let ctx = TestContext::new(server.uri());
ctx.write_config(&["test-org".to_string()], &[], 4, 5, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let result = pipeline.run().await;
assert!(result.is_err(), "Should fail with 404");
assert_eq!(
request_count.load(Ordering::SeqCst),
1,
"Should not retry on 404 (only makes 1 request)"
);
}
#[tokio::test]
async fn test_backup_with_multiple_repos() {
let archive1 = create_test_archive("repo1");
let archive2 = create_test_archive("repo2");
let repos = vec![
MockRepo::new("test-org", "repo1", archive1),
MockRepo::new("test-org", "repo2", archive2),
];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = konservejo::storage::Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(jobs.len(), 2, "Should have 2 jobs");
}
#[tokio::test]
async fn test_backup_single_repo() {
let archive = create_test_archive("single-repo");
let repos = vec![MockRepo::new("test-org", "repo1", archive)];
let (_server, mock_url) = setup_mock_server(repos).await;
let ctx = TestContext::new(mock_url);
ctx.write_config(&["test-org".to_string()], &[], 4, 1, 10);
let config = load_config(&ctx).expect("Failed to load config");
let pipeline = Pipeline::new(config)
.await
.expect("Failed to create pipeline");
let run_id = pipeline.run().await.expect("Backup failed");
let db_url = format!("sqlite://{}", ctx.db_path().display());
let storage = konservejo::storage::Storage::new(&db_url)
.await
.expect("Failed to connect to DB");
let jobs = storage
.list_jobs_by_run(&run_id)
.await
.expect("Failed to list jobs");
assert_eq!(jobs.len(), 1, "Should have 1 job");
}