eh: make tests more robust & remove flaky tests

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: Ibf43701f2c79b6be86df72e4298de7206a6a6964

Cargo.lock

@@ -17,6 +17,12 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
 
+[[package]]
+name = "bitflags"
+version = "2.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.1"
@@ -76,6 +82,7 @@ version = "0.1.2"
 dependencies = [
  "clap",
  "regex",
+ "tempfile",
  "thiserror",
  "tracing",
  "tracing-subscriber",
@@ -83,6 +90,34 @@ dependencies = [
  "yansi",
 ]
 
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -95,6 +130,18 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
+[[package]]
+name = "libc"
+version = "0.2.177"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
+
 [[package]]
 name = "log"
 version = "0.4.27"
@@ -146,6 +193,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
 [[package]]
 name = "regex"
 version = "1.12.2"
@@ -175,6 +228,19 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "rustix"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -210,6 +276,19 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
+dependencies = [
+ "fastrand",
+ "getrandom",
+ "once_cell",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.17"
@@ -318,6 +397,15 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "wasip2"
+version = "1.0.1+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
+dependencies = [
+ "wit-bindgen",
+]
+
 [[package]]
 name = "winapi-util"
 version = "0.1.11"
@@ -342,6 +430,12 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "wit-bindgen"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
+
 [[package]]
 name = "xtask"
 version = "0.1.2"

eh/Cargo.toml

@@ -18,3 +18,6 @@ tracing.workspace            = true
 tracing-subscriber.workspace = true
 walkdir.workspace            = true
 yansi.workspace              = true
+
+[dev-dependencies]
+tempfile = "3.0"

eh/tests/basic.rs

@@ -1,178 +1,219 @@
-//! I hate writing tests, and I hate writing integration tests. This is the best
+use eh::util::{
-//! that you are getting, deal with it.
+    DefaultNixErrorClassifier, DefaultNixFileFixer, HashExtractor, NixErrorClassifier,
-use std::process::{Command, Stdio};
+    NixFileFixer, RegexHashExtractor,
+};
+use std::fs;
+use std::process::Command;
+use tempfile::TempDir;
 
 #[test]
-fn nix_eval_validation() {
+fn test_hash_extraction_from_real_nix_errors() {
-    // Test that invalid expressions are caught early for all commands
+    // Test hash extraction from actual Nix error messages
-    let commands = ["build", "run", "shell"];
+    let extractor = RegexHashExtractor;
 
-    for cmd in &commands {
+    let test_cases = [
+        (
+            r#"error: hash mismatch in fixed-output derivation '/nix/store/xxx-foo.drv':
+  specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+       got:    sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="#,
+            Some("sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=".to_string()),
+        ),
+        (
+            "actual: sha256-abc123def456",
+            Some("sha256-abc123def456".to_string()),
+        ),
+        ("have: sha256-xyz789", Some("sha256-xyz789".to_string())),
+        ("no hash here", None),
+    ];
+
+    for (input, expected) in test_cases {
+        assert_eq!(extractor.extract_hash(input), expected);
+    }
+}
+
+#[test]
+fn test_error_classification_for_retry_logic() {
+    // Test that the classifier correctly identifies errors that should be retried
+    let classifier = DefaultNixErrorClassifier;
+
+    // These should trigger retries
+    let retry_cases = [
+        "Package 'discord-1.0.0' has an unfree license ('unfree'), refusing to evaluate.",
+        "Package 'openssl-1.1.1' has been marked as insecure, refusing to evaluate.",
+        "Package 'broken-1.0' has been marked as broken, refusing to evaluate.",
+        "hash mismatch in fixed-output derivation\ngot: sha256-newhash",
+    ];
+
+    for error in retry_cases {
+        assert!(classifier.should_retry(error), "Should retry: {}", error);
+    }
+
+    // These should NOT trigger retries
+    let no_retry_cases = [
+        "build failed",
+        "random error",
+        "permission denied",
+        "network error",
+    ];
+
+    for error in no_retry_cases {
+        assert!(
+            !classifier.should_retry(error),
+            "Should not retry: {}",
+            error
+        );
+    }
+}
+
+#[test]
+fn test_hash_fixing_in_nix_files() {
+    // Test that hash fixing actually works on real Nix files
+    let temp_dir = TempDir::new().expect("Failed to create temp dir");
+    let fixer = DefaultNixFileFixer;
+
+    // Create a mock Nix file with various hash formats
+    let nix_content = r#"
+stdenv.mkDerivation {
+  name = "test-package";
+  src = fetchurl {
+    url = "https://example.com.tar.gz";
+    hash = "sha256-oldhash123";
+  };
+
+  buildInputs = [ fetchurl {
+    url = "https://deps.com.tar.gz";
+    sha256 = "sha256-oldhash456";
+  }];
+
+  outputHash = "sha256-oldhash789";
+}
+"#;
+
+    let file_path = temp_dir.path().join("test.nix");
+    fs::write(&file_path, nix_content).expect("Failed to write test file");
+
+    // Test hash replacement
+    let new_hash = "sha256-newhashabc";
+    let was_fixed = fixer
+        .fix_hash_in_file(&file_path, new_hash)
+        .expect("Failed to fix hash");
+
+    assert!(was_fixed, "File should have been modified");
+
+    let updated_content = fs::read_to_string(&file_path).expect("Failed to read updated file");
+
+    // All hash formats should be updated
+    assert!(updated_content.contains(&format!(r#"hash = "{}""#, new_hash)));
+    assert!(updated_content.contains(&format!(r#"sha256 = "{}""#, new_hash)));
+    assert!(updated_content.contains(&format!(r#"outputHash = "{}""#, new_hash)));
+
+    // Old hashes should be gone
+    assert!(!updated_content.contains("oldhash123"));
+    assert!(!updated_content.contains("oldhash456"));
+    assert!(!updated_content.contains("oldhash789"));
+}
+
+#[test]
+fn test_multicall_binary_dispatch() {
+    // Test that multicall binaries work without needing actual Nix evaluation
+    let commands = [("nb", "build"), ("nr", "run"), ("ns", "shell")];
+
+    for (binary_name, _expected_command) in &commands {
+        // Test that the binary starts and handles invalid arguments gracefully
         let output = Command::new("timeout")
-            .args([
+            .args(["5", "cargo", "run", "--bin", "eh", "--"])
-                "10",
+            .env("CARGO_BIN_NAME", binary_name)
-                "cargo",
+            .arg("invalid-package-ref")
-                "run",
-                "--bin",
-                "eh",
-                "--",
-                cmd,
-                "invalid-flake-ref",
-            ])
             .output()
             .expect("Failed to execute command");
 
-        // Should fail fast with eval error
+        // Should fail gracefully (not panic or hang)
+        assert!(
+            output.status.code().is_some(),
+            "{} should exit with a code",
+            binary_name
+        );
+
+        // Should show an error message, not crash
         let stderr = String::from_utf8_lossy(&output.stderr);
-        assert!(stderr.contains("Error: Expression evaluation failed") || !output.status.success());
+        assert!(
+            stderr.contains("Error:") || stderr.contains("error:") || stderr.contains("failed"),
+            "{} should show error for invalid package",
+            binary_name
+        );
     }
 }
 
 #[test]
-fn unfree_package_handling() {
+fn test_invalid_expression_handling() {
-    // Test that unfree packages are detected and handled correctly
+    // Test that invalid Nix expressions fail fast with proper error messages
+    let invalid_refs = [
+        "invalid-flake-ref",
+        "nonexistent-package",
+        "file:///nonexistent/path",
+    ];
+
+    for invalid_ref in invalid_refs {
         let output = Command::new("timeout")
             .args([
-            "30",
+                "10",
                 "cargo",
                 "run",
                 "--bin",
                 "eh",
                 "--",
                 "build",
-            "nixpkgs#discord",
+                invalid_ref,
             ])
             .output()
             .expect("Failed to execute command");
 
+        // Should fail with a proper error, not hang or crash
+        assert!(
+            !output.status.success(),
+            "Invalid ref '{}' should fail",
+            invalid_ref
+        );
+
         let stderr = String::from_utf8_lossy(&output.stderr);
-    let stdout = String::from_utf8_lossy(&output.stdout);
-    let combined = format!("{}{}", stdout, stderr);
-
-    // Should detect unfree package and show appropriate message
         assert!(
-        combined.contains("has an unfree license")
+            stderr.contains("Error:") || stderr.contains("error:") || stderr.contains("failed"),
-            || combined.contains("NIXPKGS_ALLOW_UNFREE")
+            "Should show error message for invalid ref '{}': {}",
-            || combined.contains("⚠ Unfree package detected")
+            invalid_ref,
+            stderr
         );
     }
-
-#[test]
-fn insecure_package_handling() {
-    // Test that error classification works for insecure packages
-    use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
-
-    let classifier = DefaultNixErrorClassifier;
-    let stderr_insecure =
-        "Package 'example-1.0' has been marked as insecure, refusing to evaluate.";
-
-    assert!(classifier.should_retry(stderr_insecure));
 }
 
 #[test]
-fn broken_package_handling() {
+fn test_nix_file_discovery() {
-    // Test that error classification works for broken packages
+    // Test that the fixer can find Nix files in a directory structure
-    use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
+    let temp_dir = TempDir::new().expect("Failed to create temp dir");
+    let fixer = DefaultNixFileFixer;
 
-    let classifier = DefaultNixErrorClassifier;
+    // Create directory structure with Nix files
-    let stderr_broken = "Package 'example-1.0' has been marked as broken, refusing to evaluate.";
+    fs::create_dir_all(temp_dir.path().join("subdir")).expect("Failed to create subdir");
 
-    assert!(classifier.should_retry(stderr_broken));
+    let files = [
+        ("test.nix", "stdenv.mkDerivation { name = \"test\"; }"),
+        ("subdir/other.nix", "pkgs.hello"),
+        ("not-nix.txt", "not a nix file"),
+        ("default.nix", "import ./test.nix"),
+    ];
+
+    for (path, content) in files {
+        fs::write(temp_dir.path().join(path), content).expect("Failed to write file");
     }
 
-#[test]
+    // Change to temp dir for file discovery
-fn multicall_binary_dispatch() {
+    let original_dir = std::env::current_dir().expect("Failed to get current dir");
-    // Test that nb/nr/ns dispatch correctly based on binary name
+    std::env::set_current_dir(temp_dir.path()).expect("Failed to change directory");
-    let commands = [("nb", "build"), ("nr", "run"), ("ns", "shell")];
 
-    for (binary_name, _expected_cmd) in &commands {
+    let found_files = fixer.find_nix_files().expect("Failed to find Nix files");
-        let output = Command::new("timeout")
-            .args(["10", "cargo", "run", "--bin", "eh"])
-            .env("CARGO_BIN_NAME", binary_name)
-            .arg("nixpkgs#hello")
-            .arg("--help") // Use help to avoid actually building
-            .output()
-            .expect("Failed to execute command");
 
-        // Should execute without panicking (status code may vary)
+    // Should find 3 .nix files (not the .txt file)
-        assert!(output.status.code().is_some());
+    assert_eq!(found_files.len(), 3, "Should find exactly 3 .nix files");
-    }
+
-}
+    // Restore original directory
-
+    std::env::set_current_dir(original_dir).expect("Failed to restore directory");
-#[test]
-fn interactive_mode_inheritance() {
-    // Test that run commands inherit stdio properly
-    let mut child = Command::new("timeout")
-        .args([
-            "10",
-            "cargo",
-            "run",
-            "--bin",
-            "eh",
-            "--",
-            "run",
-            "nixpkgs#echo",
-            "test",
-        ])
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn command");
-
-    let status = child.wait().expect("Failed to wait for child");
-
-    // Should complete without hanging
-    assert!(status.code().is_some());
-}
-
-#[test]
-fn hash_extraction() {
-    use eh::util::{HashExtractor, RegexHashExtractor};
-
-    let extractor = RegexHashExtractor;
-    let stderr = "error: hash mismatch in fixed-output derivation '/nix/store/...':
-         specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
-            got:    sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=";
-
-    let hash = extractor.extract_hash(stderr);
-    assert!(hash.is_some());
-    assert_eq!(
-        hash.unwrap(),
-        "sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="
-    );
-}
-
-#[test]
-fn error_classification() {
-    use eh::util::{DefaultNixErrorClassifier, NixErrorClassifier};
-
-    let classifier = DefaultNixErrorClassifier;
-
-    assert!(classifier.should_retry("has an unfree license ('unfree'), refusing to evaluate"));
-    assert!(classifier.should_retry("has been marked as insecure, refusing to evaluate"));
-    assert!(classifier.should_retry("has been marked as broken, refusing to evaluate"));
-    assert!(!classifier.should_retry("random build error"));
-}
-
-#[test]
-fn hash_mismatch_auto_fix() {
-    // Test that hash mismatches are automatically detected and fixed
-    // This is harder to test without creating actual files, so we test the regex
-    // for the time being. Alternatively I could do this inside a temporary directory
-    // but cba for now.
-    use eh::util::{HashExtractor, RegexHashExtractor};
-
-    let extractor = RegexHashExtractor;
-    let stderr_with_mismatch = r#"
-error: hash mismatch in fixed-output derivation
-  specified: sha256-oldhashaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa=
-       got: sha256-newhashbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=
-"#;
-
-    let extracted = extractor.extract_hash(stderr_with_mismatch);
-    assert_eq!(
-        extracted,
-        Some("sha256-newhashbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=".to_string())
-    );
 }