machine tests take two

flake.nix

@@ -18,6 +18,8 @@
       systems = import inputs.systems;
       imports = [
         ./flake/templates
+        ./flake/tests
+
         ./flake/apps.nix
         ./flake/packages.nix
         ./flake/develop.nix

flake/tests/checks/hm-minimal.nix

@@ -0,0 +1,73 @@
+{
+  testers,
+  profiles,
+  modules,
+  ...
+}:
+testers.runNixOSTest {
+  name = "nvf-home-manager-test";
+  nodes.machine = {pkgs, ...}: {
+    imports = [
+      profiles.minimal
+      modules.home-manager.nvf
+    ];
+
+    programs.nvf = {
+      enable = true;
+
+      settings.vim = {
+        viAlias = true;
+        vimAlias = true;
+
+        globals = {
+          editorconfig = true;
+        };
+
+        extraPackages = [pkgs.lazygit];
+      };
+    };
+  };
+
+  testScript =
+    # python
+    ''
+      machine.start()
+      machine.wait_for_unit("multi-user.target")
+
+      with subtest("Verify that Neovim can be run by the test user and displays its version"):
+        machine.succeed("runuser -l test -c 'nvim --version'")
+
+      with subtest("Launch Neovim and immediately quit to verify it starts correctly"):
+        machine.succeed("runuser -l test -c 'nvim -c q'")
+
+      with subtest("Create a test file and open it with Neovim"):
+        machine.succeed("runuser -l test -c 'echo \"test content\" > /home/test/testfile.txt'")
+        machine.succeed("runuser -l test -c 'nvim -c \"wq\" /home/test/testfile.txt'")
+
+      with subtest("Verify the file was edited and saved correctly"):
+        machine.succeed("grep 'test content' /home/test/testfile.txt")
+
+      with subtest("Run specific Neovim commands and verify the output"):
+        machine.succeed("runuser -l test -c 'echo hello > /home/test/input.txt'")
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"normal iworld\" -c \"wq\" /home/test/input.txt'")
+        machine.succeed("grep 'worldhello' /home/test/input.txt")
+
+      with subtest("Test nvf configuration"):
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"lua if vim.g.editorconfig == true then io.open(\\\"/home/test/config_result.txt\\\", \\\"w\\\"):write(\\\"true\\\"):close() end\" -c q'")
+        machine.succeed("runuser -l test -c 'test -e /home/test/config_result.txt'")
+        machine.succeed("runuser -l test -c 'cat /home/test/config_result.txt | grep true'")
+
+      with subtest("Check for errors in startup messages"):
+        machine.succeed("runuser -l test -c 'nvim --headless --startuptime /home/test/startup.log +q'")
+        machine.succeed("runuser -l test -c 'grep -v -i \"error\" /home/test/startup.log | wc -l > /home/test/line_count.txt'")
+        machine.succeed("test $(cat /home/test/line_count.txt) -gt 0")
+
+      with subtest("Verify files in Neovim runtime path"):
+        machine.succeed("runuser -l test -c 'nvim --cmd \"set rtp\" --headless -c q 2>&1 | grep \"/nix/store/\" > /home/test/vimruntime.txt'")
+        machine.succeed("test -s /home/test/vimruntime.txt")
+
+      with subtest("Verify extrapackages can be executed inside Neovim"):
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"silent !which lazygit > /home/test/lazygit_path.txt\" -c q'")
+        machine.succeed("runuser -l test -c 'grep \"/nix/store/\" /home/test/lazygit_path.txt'")
+    '';
+}

flake/tests/checks/nixos-minimal.nix

@@ -0,0 +1,73 @@
+{
+  testers,
+  profiles,
+  modules,
+  ...
+}:
+testers.runNixOSTest {
+  name = "nvf-nixos-test";
+  nodes.machine = {pkgs, ...}: {
+    imports = [
+      profiles.minimal
+      modules.nixos.nvf
+    ];
+
+    programs.nvf = {
+      enable = true;
+
+      settings.vim = {
+        viAlias = true;
+        vimAlias = true;
+
+        globals = {
+          editorconfig = true;
+        };
+
+        extraPackages = [pkgs.lazygit];
+      };
+    };
+  };
+
+  testScript =
+    # python
+    ''
+      machine.start()
+      machine.wait_for_unit("multi-user.target")
+
+      with subtest("Verify that Neovim can be run by the test user and displays its version"):
+        machine.succeed("runuser -l test -c 'nvim --version'")
+
+      with subtest("Launch Neovim and immediately quit to verify it starts correctly"):
+        machine.succeed("runuser -l test -c 'nvim -c q'")
+
+      with subtest("Create a test file and open it with Neovim"):
+        machine.succeed("runuser -l test -c 'echo \"test content\" > /home/test/testfile.txt'")
+        machine.succeed("runuser -l test -c 'nvim -c \"wq\" /home/test/testfile.txt'")
+
+      with subtest("Verify the file was edited and saved correctly"):
+        machine.succeed("grep 'test content' /home/test/testfile.txt")
+
+      with subtest("Run specific Neovim commands and verify the output"):
+        machine.succeed("runuser -l test -c 'echo hello > /home/test/input.txt'")
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"normal iworld\" -c \"wq\" /home/test/input.txt'")
+        machine.succeed("grep 'worldhello' /home/test/input.txt")
+
+      with subtest("Test nvf configuration"):
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"lua if vim.g.editorconfig == true then io.open(\\\"/home/test/config_result.txt\\\", \\\"w\\\"):write(\\\"true\\\"):close() end\" -c q'")
+        machine.succeed("runuser -l test -c 'test -e /home/test/config_result.txt'")
+        machine.succeed("runuser -l test -c 'cat /home/test/config_result.txt | grep true'")
+
+      with subtest("Check for errors in startup messages"):
+        machine.succeed("runuser -l test -c 'nvim --headless --startuptime /home/test/startup.log +q'")
+        machine.succeed("runuser -l test -c 'grep -v -i \"error\" /home/test/startup.log | wc -l > /home/test/line_count.txt'")
+        machine.succeed("test $(cat /home/test/line_count.txt) -gt 0")
+
+      with subtest("Verify files in Neovim runtime path"):
+        machine.succeed("runuser -l test -c 'nvim --cmd \"set rtp\" --headless -c q 2>&1 | grep \"/nix/store/\" > /home/test/vimruntime.txt'")
+        machine.succeed("test -s /home/test/vimruntime.txt")
+
+      with subtest("Verify extrapackages can be executed inside Neovim"):
+        machine.succeed("runuser -l test -c 'nvim --headless -c \"silent !which lazygit > /home/test/lazygit_path.txt\" -c q'")
+        machine.succeed("runuser -l test -c 'grep \"/nix/store/\" /home/test/lazygit_path.txt'")
+    '';
+}

flake/tests/default.nix

@@ -0,0 +1,42 @@
+{
+  inputs,
+  self,
+  config,
+  lib,
+  ...
+}: {
+  perSystem = {pkgs, ...}: let
+    inherit (lib.filesystem) packagesFromDirectoryRecursive;
+    inherit (lib.customisation) callPackageWith;
+    inherit (lib.attrsets) recursiveUpdate;
+
+    # Attribute set containing paths to specific profiles. This is meant to be
+    # an easy path to where profiles are defined as the tests directory gets
+    # more complicated, and allow easier renames.
+    profiles = {
+      # Minimal profile is small and lightweight. Plugins that require dependencies
+      # or tests that are resource intensive should not use the minimal profile.
+      minimal = ./profiles/minimal.nix;
+    };
+
+    modules = {
+      nixos = config.flake.nixosModules;
+      home-manager = config.flake.homeManagerModules;
+    };
+
+    # Call tests with preset special arguments.
+    defaultInherits = {inherit inputs profiles modules;};
+    callPackage = callPackageWith (recursiveUpdate pkgs defaultInherits);
+    checkPackages = packagesFromDirectoryRecursive {
+      inherit callPackage;
+      directory = ./checks;
+    };
+  in {
+    # Machine tests that will be ran either manually with 'nix build .#checks.<system>.<check>'
+    # or automatically on 'nix flake check' unless '--no-build-checks' has been passed to it.
+    # We merge 'config.legacyPackages' here to make sure we also build and check plugin packages
+    # that are declared in legacyPackages are built as well, which is usually reserved to Neovim
+    # plugins tracked with npins.
+    checks = checkPackages;
+  };
+}

flake/tests/profiles/minimal.nix

@@ -0,0 +1,16 @@
+# The 'minimal' test profile for nvf. This exposes the bare minimum for defining the test
+# VMs. If an addition is test-specific (e.g., targeting at a specific functionality) then
+# it does not belong here. However machine configuration that must be propagated to *all*
+# tests should be defined here.
+{
+  virtualisation = {
+    cores = 2;
+    memorySize = 2048;
+    qemu.options = ["-vga none -enable-kvm -device virtio-gpu-pci,xres=720,yres=1440"];
+  };
+
+  users.users.test = {
+    isNormalUser = true;
+    password = "";
+  };
+}