NotAShelf/eh
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

eh: make tests more robust & remove flaky tests

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: Ibf43701f2c79b6be86df72e4298de7206a6a6964
This commit is contained in:
raf 2025-11-14 21:25:04 +03:00
commit 261e834ec4
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
3 changed files with 291 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

94
Cargo.lock generated
View file

@ -17,6 +17,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
[[package]]
name = "bitflags"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
[[package]]
name = "cfg-if"
version = "1.0.1"
@ -76,6 +82,7 @@ version = "0.1.2"
dependencies = [
"clap",
"regex",
"tempfile",
"thiserror",
"tracing",
"tracing-subscriber",
@ -83,6 +90,34 @@ dependencies = [
"yansi",
]
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
]
[[package]]
name = "heck"
version = "0.5.0"
@ -95,6 +130,18 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
version = "0.2.177"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
[[package]]
name = "linux-raw-sys"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
[[package]]
name = "log"
version = "0.4.27"
@ -146,6 +193,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "regex"
version = "1.12.2"
@ -175,6 +228,19 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustix"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "same-file"
version = "1.0.6"
@ -210,6 +276,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
dependencies = [
"fastrand",
"getrandom",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "2.0.17"
@ -318,6 +397,15 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasip2"
version = "1.0.1+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "winapi-util"
version = "0.1.11"
@ -342,6 +430,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "wit-bindgen"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
[[package]]
name = "xtask"
version = "0.1.2"

3
eh/Cargo.toml
View file

@ -18,3 +18,6 @@ tracing.workspace = true
tracing-subscriber.workspace = true
walkdir.workspace = true
yansi.workspace = true
[dev-dependencies]
tempfile = "3.0"

337
eh/tests/basic.rs
View file

@ -1,178 +1,219 @@
//! I hate writing tests, and I hate writing integration tests. This is the best
//! that you are getting, deal with it.
use std::process::{Command, Stdio};
use eh::util::{
DefaultNixErrorClassifier, DefaultNixFileFixer, HashExtractor, NixErrorClassifier,
NixFileFixer, RegexHashExtractor,
};
use std::fs;
use std::process::Command;
use tempfile::TempDir;
#[test]
fn nix_eval_validation() {
// Test that invalid expressions are caught early for all commands
let commands = ["build", "run", "shell"];
fn test_hash_extraction_from_real_nix_errors() {
// Test hash extraction from actual Nix error messages
let extractor = RegexHashExtractor;
for cmd in &commands {
let output = Command::new("timeout")
.args([
"10",
"cargo",
"run",
"--bin",
"eh",
"--",
cmd,
"invalid-flake-ref",
])
.output()
.expect("Failed to execute command");
let test_cases = [
(
r#"error: hash mismatch in fixed-output derivation '/nix/store/xxx-foo.drv':
specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
got: sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="#,
Some("sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=".to_string()),
),
(
"actual: sha256-abc123def456",
Some("sha256-abc123def456".to_string()),
),
("have: sha256-xyz789", Some("sha256-xyz789".to_string())),
("no hash here", None),
];
// Should fail fast with eval error
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("Error: Expression evaluation failed") || !output.status.success());
for (input, expected) in test_cases {
assert_eq!(extractor.extract_hash(input), expected);
}
}
#[test]
fn unfree_package_handling() {
// Test that unfree packages are detected and handled correctly
fn test_error_classification_for_retry_logic() {
// Test that the classifier correctly identifies errors that should be retried
let classifier = DefaultNixErrorClassifier;
// These should trigger retries
let retry_cases = [
"Package 'discord-1.0.0' has an unfree license ('unfree'), refusing to evaluate.",
"Package 'openssl-1.1.1' has been marked as insecure, refusing to evaluate.",
"Package 'broken-1.0' has been marked as broken, refusing to evaluate.",
"hash mismatch in fixed-output derivation\ngot: sha256-newhash",
];
for error in retry_cases {
assert!(classifier.should_retry(error), "Should retry: {}", error);
}
// These should NOT trigger retries
let no_retry_cases = [
"build failed",
"random error",
"permission denied",
"network error",
];
for error in no_retry_cases {
assert!(
!classifier.should_retry(error),
"Should not retry: {}",
error
);
}
}
#[test]
fn test_hash_fixing_in_nix_files() {
// Test that hash fixing actually works on real Nix files
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let fixer = DefaultNixFileFixer;
// Create a mock Nix file with various hash formats
let nix_content = r#"
stdenv.mkDerivation {
name = "test-package";
src = fetchurl {
url = "https://example.com.tar.gz";
hash = "sha256-oldhash123";
};
buildInputs = [ fetchurl {
url = "https://deps.com.tar.gz";
sha256 = "sha256-oldhash456";
}];
outputHash = "sha256-oldhash789";
}
"#;
let file_path = temp_dir.path().join("test.nix");
fs::write(&file_path, nix_content).expect("Failed to write test file");
// Test hash replacement
let new_hash = "sha256-newhashabc";
let was_fixed = fixer
.fix_hash_in_file(&file_path, new_hash)
.expect("Failed to fix hash");
assert!(was_fixed, "File should have been modified");
let updated_content = fs::read_to_string(&file_path).expect("Failed to read updated file");
// All hash formats should be updated
assert!(updated_content.contains(&format!(r#"hash = "{}""#, new_hash)));
assert!(updated_content.contains(&format!(r#"sha256 = "{}""#, new_hash)));
assert!(updated_content.contains(&format!(r#"outputHash = "{}""#, new_hash)));
// Old hashes should be gone
assert!(!updated_content.contains("oldhash123"));
assert!(!updated_content.contains("oldhash456"));
assert!(!updated_content.contains("oldhash789"));
}
#[test]
fn test_multicall_binary_dispatch() {
// Test that multicall binaries work without needing actual Nix evaluation
let commands = [("nb", "build"), ("nr", "run"), ("ns", "shell")];
for (binary_name, _expected_command) in &commands {
// Test that the binary starts and handles invalid arguments gracefully
let output = Command::new("timeout")
.args(["5", "cargo", "run", "--bin", "eh", "--"])
.env("CARGO_BIN_NAME", binary_name)
.arg("invalid-package-ref")
.output()
.expect("Failed to execute command");
// Should fail gracefully (not panic or hang)
assert!(
output.status.code().is_some(),
"{} should exit with a code",
binary_name
);
// Should show an error message, not crash
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
stderr.contains("Error:") || stderr.contains("error:") || stderr.contains("failed"),
"{} should show error for invalid package",
binary_name
);
}
}
#[test]
fn test_invalid_expression_handling() {
// Test that invalid Nix expressions fail fast with proper error messages
let invalid_refs = [
"invalid-flake-ref",
"nonexistent-package",
"file:///nonexistent/path",
];
for invalid_ref in invalid_refs {
let output = Command::new("timeout")
.args([
"30",
"10",
"cargo",
"run",
"--bin",
"eh",
"--",
"build",
"nixpkgs#discord",
invalid_ref,
])
.output()
.expect("Failed to execute command");
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
let combined = format!("{}{}", stdout, stderr);
// Should detect unfree package and show appropriate message
// Should fail with a proper error, not hang or crash
assert!(
combined.contains("has an unfree license")
|| combined.contains("NIXPKGS_ALLOW_UNFREE")
|| combined.contains("⚠ Unfree package detected")
!output.status.success(),
"Invalid ref '{}' should fail",
invalid_ref
);
}
#[test]
fn insecure_package_handling() {
// Test that error classification works for insecure packages
use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
let classifier = DefaultNixErrorClassifier;
let stderr_insecure =
"Package 'example-1.0' has been marked as insecure, refusing to evaluate.";
assert!(classifier.should_retry(stderr_insecure));
}
#[test]
fn broken_package_handling() {
// Test that error classification works for broken packages
use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
let classifier = DefaultNixErrorClassifier;
let stderr_broken = "Package 'example-1.0' has been marked as broken, refusing to evaluate.";
assert!(classifier.should_retry(stderr_broken));
}
#[test]
fn multicall_binary_dispatch() {
// Test that nb/nr/ns dispatch correctly based on binary name
let commands = [("nb", "build"), ("nr", "run"), ("ns", "shell")];
for (binary_name, _expected_cmd) in &commands {
let output = Command::new("timeout")
.args(["10", "cargo", "run", "--bin", "eh"])
.env("CARGO_BIN_NAME", binary_name)
.arg("nixpkgs#hello")
.arg("--help") // Use help to avoid actually building
.output()
.expect("Failed to execute command");
// Should execute without panicking (status code may vary)
assert!(output.status.code().is_some());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
stderr.contains("Error:") || stderr.contains("error:") || stderr.contains("failed"),
"Should show error message for invalid ref '{}': {}",
invalid_ref,
stderr
);
}
}
#[test]
fn interactive_mode_inheritance() {
// Test that run commands inherit stdio properly
let mut child = Command::new("timeout")
.args([
"10",
"cargo",
"run",
"--bin",
"eh",
"--",
"run",
"nixpkgs#echo",
"test",
])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("Failed to spawn command");
fn test_nix_file_discovery() {
// Test that the fixer can find Nix files in a directory structure
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let fixer = DefaultNixFileFixer;
let status = child.wait().expect("Failed to wait for child");
// Create directory structure with Nix files
fs::create_dir_all(temp_dir.path().join("subdir")).expect("Failed to create subdir");
// Should complete without hanging
assert!(status.code().is_some());
}
#[test]
fn hash_extraction() {
use eh::util::{HashExtractor, RegexHashExtractor};
let extractor = RegexHashExtractor;
let stderr = "error: hash mismatch in fixed-output derivation '/nix/store/...':
specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
got: sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=";
let hash = extractor.extract_hash(stderr);
assert!(hash.is_some());
assert_eq!(
hash.unwrap(),
"sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="
);
}
#[test]
fn error_classification() {
use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
let classifier = DefaultNixErrorClassifier;
assert!(classifier.should_retry("has an unfree license ('unfree'), refusing to evaluate"));
assert!(classifier.should_retry("has been marked as insecure, refusing to evaluate"));
assert!(classifier.should_retry("has been marked as broken, refusing to evaluate"));
assert!(!classifier.should_retry("random build error"));
}
#[test]
fn hash_mismatch_auto_fix() {
// Test that hash mismatches are automatically detected and fixed
// This is harder to test without creating actual files, so we test the regex
// for the time being. Alternatively I could do this inside a temporary directory
// but cba for now.
use eh::util::{HashExtractor, RegexHashExtractor};
let extractor = RegexHashExtractor;
let stderr_with_mismatch = r#"
error: hash mismatch in fixed-output derivation
specified: sha256-oldhashaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa=
got: sha256-newhashbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=
"#;
let extracted = extractor.extract_hash(stderr_with_mismatch);
assert_eq!(
extracted,
Some("sha256-newhashbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=".to_string())
);
let files = [
("test.nix", "stdenv.mkDerivation { name = \"test\"; }"),
("subdir/other.nix", "pkgs.hello"),
("not-nix.txt", "not a nix file"),
("default.nix", "import ./test.nix"),
];
for (path, content) in files {
fs::write(temp_dir.path().join(path), content).expect("Failed to write file");
}
// Change to temp dir for file discovery
let original_dir = std::env::current_dir().expect("Failed to get current dir");
std::env::set_current_dir(temp_dir.path()).expect("Failed to change directory");
let found_files = fixer.find_nix_files().expect("Failed to find Nix files");
// Should find 3 .nix files (not the .txt file)
assert_eq!(found_files.len(), 3, "Should find exactly 3 .nix files");
// Restore original directory
std::env::set_current_dir(original_dir).expect("Failed to restore directory");
}