Merge pull request #537 from nix-community/ubuntu-kexec-test

Add Ubuntu kexec test

Author: Mic92

flake.lock

@@ -11,6 +11,8 @@
     nixos-images.url = "github:nix-community/nixos-images";
     nixos-images.inputs.nixos-unstable.follows = "nixpkgs";
     nixos-images.inputs.nixos-stable.follows = "nixos-stable";
+    # https://github.com/numtide/nix-vm-test/pull/105
+    nix-vm-test = { url = "github:Mic92/nix-vm-test"; inputs.nixpkgs.follows = "nixpkgs"; };
 
     # used for development
     treefmt-nix = { url = "github:numtide/treefmt-nix"; inputs.nixpkgs.follows = "nixpkgs"; };

flake.nix

@@ -58,14 +58,14 @@ hasCurl=
 hasSetsid=
 hasNixOSFacter=
 
-sshKeyDir=$(mktemp -d)
-trap 'rm -rf "$sshKeyDir"' EXIT
-mkdir -p "$sshKeyDir"
+tempDir=$(mktemp -d)
+trap 'rm -rf "$tempDir"' EXIT
+mkdir -p "$tempDir"
 
 declare -A diskEncryptionKeys=()
 declare -A extraFilesOwnership=()
 declare -a nixCopyOptions=()
-declare -a sshArgs=("-o" "IdentitiesOnly=yes" "-i" "$sshKeyDir/nixos-anywhere" "-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no")
+declare -a sshArgs=("-o" "IdentitiesOnly=yes" "-i" "$tempDir/nixos-anywhere" "-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no")
 
 showUsage() {
   cat <<USAGE
@@ -465,19 +465,26 @@ runVmTest() {
 
 uploadSshKey() {
   # ssh-copy-id requires this directory
-  mkdir -p "$HOME/.ssh/"
+  local sshCopyHome="$HOME"
+  if ! mkdir -p "$HOME/.ssh/" 2>/dev/null; then
+    # Fallback: create a temporary home directory for ssh-copy-id in tempDir
+    sshCopyHome="$tempDir/ssh-home"
+    mkdir -p "$sshCopyHome/.ssh"
+    echo "Warning: Could not create $HOME/.ssh, using temporary directory: $sshCopyHome"
+  fi
+
   if [[ -n ${sshPrivateKeyFile} ]]; then
-    cp "$sshPrivateKeyFile" "$sshKeyDir/nixos-anywhere"
-    ssh-keygen -y -f "$sshKeyDir/nixos-anywhere" >"$sshKeyDir/nixos-anywhere.pub"
+    cp "$sshPrivateKeyFile" "$tempDir/nixos-anywhere"
+    ssh-keygen -y -f "$tempDir/nixos-anywhere" >"$tempDir/nixos-anywhere.pub"
   else
     # we generate a temporary ssh keypair that we can use during nixos-anywhere
-    ssh-keygen -t ed25519 -f "$sshKeyDir"/nixos-anywhere -P "" -C "nixos-anywhere" >/dev/null
+    ssh-keygen -t ed25519 -f "$tempDir"/nixos-anywhere -P "" -C "nixos-anywhere" >/dev/null
   fi
 
   step Uploading install SSH keys
   until
     if [[ ${envPassword} == y ]]; then
-      sshpass -e \
+      HOME="$sshCopyHome" sshpass -e \
         ssh-copy-id \
         -o ConnectTimeout=10 \
         "${sshArgs[@]}" \
@@ -486,7 +493,7 @@ uploadSshKey() {
       # To override `IdentitiesOnly=yes` set in `sshArgs` we need to set
       # `IdentitiesOnly=no` first as the first time an SSH option is
       # specified on the command line takes precedence
-      ssh-copy-id \
+      HOME="$sshCopyHome" ssh-copy-id \
         -o IdentitiesOnly=no \
         -o ConnectTimeout=10 \
         "${sshArgs[@]}" \
@@ -702,7 +709,7 @@ runDisko() {
     # If we don't use ssh-ng here, we get `error: operation 'getFSAccessor' is not supported by store`
     diskoScript=$(
       nixBuild "${flake}#${flakeAttr}.system.build.${diskoAttr}" \
-        --eval-store auto --store "ssh-ng://$sshConnection?ssh-key=$sshKeyDir%2Fnixos-anywhere&$sshStoreSettings"
+        --eval-store auto --store "ssh-ng://$sshConnection?ssh-key=$tempDir%2Fnixos-anywhere&$sshStoreSettings"
     )
   fi
 
@@ -724,7 +731,7 @@ nixosInstall() {
     # If we don't use ssh-ng here, we get `error: operation 'getFSAccessor' is not supported by store`
     nixosSystem=$(
       nixBuild "${flake}#${flakeAttr}.system.build.toplevel" \
-        --eval-store auto --store "ssh-ng://$sshConnection?ssh-key=$sshKeyDir%2Fnixos-anywhere&remote-store=local%3Froot=%2Fmnt&$sshStoreSettings"
+        --eval-store auto --store "ssh-ng://$sshConnection?ssh-key=$tempDir%2Fnixos-anywhere&remote-store=local%3Froot=%2Fmnt&$sshStoreSettings"
     )
   fi
 
@@ -823,8 +830,8 @@ main() {
   fi
 
   if [[ -n ${SSH_PRIVATE_KEY} ]] && [[ -z ${sshPrivateKeyFile} ]]; then
-    # $sshKeyDir is getting deleted on trap EXIT
-    sshPrivateKeyFile="$sshKeyDir/from-env"
+    # $tempDir is getting deleted on trap EXIT
+    sshPrivateKeyFile="$tempDir/from-env"
     (
       umask 077
       printf '%s\n' "$SSH_PRIVATE_KEY" >"$sshPrivateKeyFile"

src/nixos-anywhere.sh

@@ -3,15 +3,23 @@
 {
   flake.checks.x86_64-linux = withSystem "x86_64-linux" ({ pkgs, system, inputs', config, ... }:
     let
+      system-to-install = pkgs.nixos [
+        ./modules/system-to-install.nix
+        inputs.disko.nixosModules.disko
+      ];
       testInputsUnstable = {
         inherit pkgs;
         inherit (inputs.disko.nixosModules) disko;
         nixos-anywhere = config.packages.nixos-anywhere;
         kexec-installer = "${inputs'.nixos-images.packages.kexec-installer-nixos-unstable-noninteractive}/nixos-kexec-installer-noninteractive-${system}.tar.gz";
+        inherit system-to-install;
       };
       testInputsStable = testInputsUnstable // {
         kexec-installer = "${inputs'.nixos-images.packages.kexec-installer-nixos-stable-noninteractive}/nixos-kexec-installer-noninteractive-${system}.tar.gz";
       };
+      ubuntuTestInputs = testInputsUnstable // {
+        nix-vm-test = inputs.nix-vm-test;
+      };
     in
     {
       from-nixos = import ./from-nixos.nix testInputsUnstable;
@@ -21,5 +29,6 @@
       from-nixos-with-generated-config = import ./from-nixos-generate-config.nix testInputsUnstable;
       from-nixos-build-on-remote = import ./from-nixos-build-on-remote.nix testInputsUnstable;
       from-nixos-separated-phases = import ./from-nixos-separated-phases.nix testInputsUnstable;
+      ubuntu-kexec-test = import ./ubuntu-kexec-test.nix ubuntuTestInputs;
     });
 }

tests/flake-module.nix

@@ -1,8 +1,9 @@
 test:
-{ pkgs ? import <nixpkgs> { }
-, nixos-anywhere ? pkgs.callPackage ../../src { }
-, disko ? "${builtins.fetchTarball "https://github.com/nix-community/disko/archive/master.tar.gz"}/module.nix"
-, kexec-installer ? builtins.fetchurl "https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-kexec-installer-noninteractive-${pkgs.stdenv.hostPlatform.system}.tar.gz"
+{ pkgs
+, nixos-anywhere
+, disko
+, kexec-installer
+, system-to-install
 , ...
 }:
 let
@@ -14,6 +15,6 @@ in
   # speed-up evaluation
   defaults.documentation.enable = lib.mkDefault false;
   # to accept external dependencies such as disko
-  node.specialArgs.inputs = { inherit nixos-anywhere disko kexec-installer; };
+  node.specialArgs.inputs = { inherit nixos-anywhere disko kexec-installer system-to-install; };
   imports = [ test ];
 }).config.result

tests/lib/test-base.nix

@@ -1,12 +1,4 @@
 { pkgs, inputs, ... }:
-let
-  disko = inputs.disko;
-  kexec-installer = inputs.kexec-installer;
-  system-to-install = pkgs.nixos [
-    ./system-to-install.nix
-    disko
-  ];
-in
 {
   system.activationScripts.rsa-key = ''
     ${pkgs.coreutils}/bin/install -D -m600 ${./ssh-keys/ssh} /root/.ssh/install_key
@@ -15,8 +7,8 @@ in
   environment.systemPackages = [ inputs.nixos-anywhere ];
 
   environment.etc = {
-    "nixos-anywhere/disko".source = system-to-install.config.system.build.diskoScriptNoDeps;
-    "nixos-anywhere/system-to-install".source = system-to-install.config.system.build.toplevel;
-    "nixos-anywhere/kexec-installer".source = kexec-installer;
+    "nixos-anywhere/disko".source = inputs.system-to-install.config.system.build.diskoScriptNoDeps;
+    "nixos-anywhere/system-to-install".source = inputs.system-to-install.config.system.build.toplevel;
+    "nixos-anywhere/kexec-installer".source = inputs.kexec-installer;
   };
 }

tests/modules/installer.nix

@@ -0,0 +1,97 @@
+{ pkgs, nixos-anywhere, kexec-installer, nix-vm-test, system-to-install, ... }:
+
+(nix-vm-test.lib.${pkgs.system}.ubuntu."24_04" {
+  sharedDirs = { };
+
+  # Configure VM with 2GB memory
+  machineConfigModule = { ... }: {
+    nodes.vm.virtualisation.memorySize = 2048;
+  };
+
+  # The test script
+  testScript = ''
+    # Python imports
+    import subprocess
+    import tempfile
+    import shutil
+    import os
+
+    # Wait for the system to be fully booted
+    vm.wait_for_unit("multi-user.target")
+
+    # Unmask SSH service (which is masked by default in the test VM)
+    vm.succeed("systemctl unmask ssh.service ssh.socket")
+
+    # Generate SSH host keys (required for SSH to start)
+    vm.succeed("ssh-keygen -A")
+
+    # Setup SSH with the existing keys
+    vm.succeed("mkdir -p /root/.ssh")
+    vm.succeed(
+      "echo '${builtins.replaceStrings ["\n"] [""] (builtins.readFile ./modules/ssh-keys/ssh.pub)}' > /root/.ssh/authorized_keys"
+    )
+    vm.succeed("chmod 644 /root/.ssh/authorized_keys")
+
+    # Setup SSH for connection from host
+    vm.succeed(
+      "sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config"
+    )
+
+    # Start SSH service
+    vm.succeed("systemctl start ssh")
+
+    # Wait for SSH to be available
+    vm.wait_for_open_port(22)
+
+    # Forward SSH port using vm.forward_port method
+    ssh_port = 2222
+    vm.forward_port(host_port=ssh_port, guest_port=22)
+
+    # Use temporary file for SSH key with automatic cleanup
+    with tempfile.NamedTemporaryFile(mode='w', delete=True, suffix='_ssh_key') as temp_key:
+      temp_key_path = temp_key.name
+
+      # Copy SSH private key to temp file with correct permissions
+      shutil.copy2("${./modules/ssh-keys/ssh}", temp_key_path)
+      os.chmod(temp_key_path, 0o600)
+
+      nixos_anywhere_cmd = [
+        "${nixos-anywhere}/bin/nixos-anywhere",
+        "-i", temp_key_path,
+        "--ssh-port", str(ssh_port),
+        "--post-kexec-ssh-port", "2222",
+        "--phases", "kexec",
+        "--kexec", "${kexec-installer}",
+        "--store-paths", "${system-to-install.config.system.build.diskoScriptNoDeps}",
+        "${system-to-install.config.system.build.toplevel}",
+        "--debug",
+        "root@localhost"
+      ]
+
+      result = subprocess.run(nixos_anywhere_cmd, check=False)
+
+      if result.returncode != 0:
+        print(f"nixos-anywhere failed with exit code {result.returncode}")
+        vm.succeed("dmesg | tail -n 50")
+        vm.succeed("journalctl -n 50")
+        raise Exception(f"nixos-anywhere command failed with exit code {result.returncode}")
+
+      # Test SSH connection to verify we're in NixOS kexec environment
+      check_cmd = [
+        "${pkgs.openssh}/bin/ssh", "-v",
+        "-i", temp_key_path,
+        "-p", "2222",
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "root@localhost",
+        "cat /etc/os-release"
+      ]
+
+      test_result = subprocess.run(check_cmd, check=True, stdout=subprocess.PIPE, text=True)
+      assert "nixos" in test_result.stdout.lower(), f"Expected NixOS environment but got: {test_result.stdout}"
+
+    # After kexec we no longer have the machine driver,
+    # so we need to let the VM crash because the test driver backdoor gets confused by the terminal output.
+    vm.crash()
+  '';
+}).sandboxed