Merge pull request #60 from sid115/feature/vfio

feature/vfio: Expand virtualisation NixOS module with options for VFIO

Author: sid115

docs/modules/home/virtualization.md renamed to docs/modules/home/virtualisation.md

@@ -1,11 +1,11 @@
-# Virtualization
+# Virtualisation
 
-Home Manager module to go with the [Virtualization NixOS module](../nixos/virtualization.md).
+Home Manager module to go with the [Virtualisation NixOS module](../nixos/virtualisation.md).
 
-View the [*nix-core* Home Manager module on GitHub](https://github.com/sid115/nix-core/tree/master/modules/home/virtualization).
+View the [*nix-core* Home Manager module on GitHub](https://github.com/sid115/nix-core/tree/master/modules/home/virtualisation).
 
 ## Setup
 
-1. Import this module in your Home Manager configuration and the corresponding [NixOS module](../nixos/virtualization.md) in your NixOS configuration.
+1. Import this module in your Home Manager configuration and the corresponding [NixOS module](../nixos/virtualisation.md) in your NixOS configuration.
 1. Rebuild and reboot: `rebuild all && sudo reboot now`
 1. Start the default network: `virsh net-autostart default`

docs/modules/nixos/virtualisation.md

@@ -0,0 +1,171 @@
+# Virtualisation
+
+Virtualisation using QEMU via libvirt and managed through Virt-manager with VFIO support.
+
+View the [*nix-core* NixOS module on GitHub](https://github.com/sid115/nix-core/tree/master/modules/nixos/virtualisation).
+
+## Overview
+
+1. **QEMU** is the hypervisor that provides the core virtualisation capabilities.
+1. **libvirt** is a toolkit and API that manages virtualisation platforms, such as QEMU.
+1. **Virt-manager** is a GUI tool that interacts with libvirt to manage VMs.
+1. **virsh** is a CLI tool that interacts with libvirt to manage VMs.
+
+## Docs
+
+### QEMU
+
+- [Official docs](https://www.qemu.org/docs/master/)
+
+### libvirt
+
+- [Official docs](https://libvirt.org/docs.html)
+- [Arch Wiki](https://wiki.archlinux.org/title/Libvirt)
+- [virsh CLI](https://www.libvirt.org/manpages/virsh.html)
+
+> If you are using the [Home Manager module](../home/virtualisation.md) as well, then `virsh` is aliased to `virsh --connect qemu:///system`
+
+### Virt-manager
+
+- [GitHub Repository](https://github.com/virt-manager/virt-manager)
+- [NixOS Official Wiki](https://wiki.nixos.org/wiki/Virt-manager)
+- [NixOS Community Wiki](https://nixos.wiki/wiki/Virt-manager)
+- [Arch Wiki](https://wiki.archlinux.org/title/Virt-manager)
+
+## Setup
+
+1. Import this module in your NixOS config. It is recommended to use the [Virtualisation Manager module](../home/virtualisation.md) as well.
+1. Add your user to the `libvirtd` and `qemu-libvirtd` group:
+    ```nix
+    users.extraGroups.libvirtd.members = [ "<you>" ];
+    users.extraGroups.qemu-libvirtd.members = [ "<you>" ];
+    ```
+1. Rebuild and reboot: `rebuild all && sudo reboot now`
+1. Enable and start the default network and reboot again: `virsh net-autostart default && virsh net-start default`
+
+## VFIO
+
+### Setup
+
+For successful PCI device passthrough, devices must be properly isolated by IOMMU groups. A device can be safely passed through if:
+- It is the **only device** in its IOMMU group (recommended), OR
+- **All devices** in its IOMMU group are passed through together
+
+This module includes an `iommu-groups` command to help identify IOMMU groups:
+
+```bash
+iommu-groups
+```
+
+In this example, IOMMU group 9 contains only the Nvidia GPU which will get passed to the VM:
+
+```
+IOMMU Group 9 01:00.0 3D controller [0302]: NVIDIA Corporation TU117M [GeForce GTX 1650 Mobile / Max-Q] [10de:1f9d] (rev a1)
+```
+
+Take not of the PCI device ID. In this case: `10de:1f9d`.
+
+### Config
+
+This is an example with the Nvidia GPU above:
+
+```nix
+{ inputs, ... }:
+
+{
+  imports = [ inputs.core.nixosModules.virtualisation ];
+
+  virtualisation = {
+    vfio = {
+      enable = true;
+      IOMMUType = "amd";
+      devices = [
+        "10de:1f9d"
+      ];
+      blacklistNvidia = true;
+    };
+    hugepages.enable = true;
+  };
+}
+```
+
+### Virt Manager
+
+#### 1. Open VM Hardware Settings
+
+- Select your VM in Virt Manager
+- Click *"Show virtual hardware details"*
+
+#### 2. Add PCI Host Device
+
+- Click *"Add Hardware"* button at bottom
+- Select *"PCI Host Device"* from the list
+- Click *"Finish"*
+
+You may repeat this process for as many devices as you want to add to your VM.
+
+### Looking Glass
+
+TODO
+
+### Troubleshooting
+
+#### Check Kernel Parameters
+
+View current kernel parameters:
+
+```bash
+cat /proc/cmdline
+```
+
+Check VFIO-related parameters:
+
+```bash
+dmesg | grep -i vfio
+```
+
+Verify IOMMU is enabled:
+
+```bash
+dmesg | grep -i iommu
+```
+
+#### Verify device binding
+
+```bash
+lscpi -k
+```
+
+Look for your device you want to pass through. It should say:
+
+```
+Kernel driver in use: vfio-pci
+```
+
+For example:
+
+```
+01:00.0 3D controller: NVIDIA Corporation TU117M [GeForce GTX 1650 Mobile / Max-Q] (rev a1)
+	Subsystem: Lenovo Device 380d
+	Kernel driver in use: vfio-pci
+	Kernel modules: nvidiafb, nouveau
+```
+
+#### Verify module status
+
+Ensure blacklisted modules are not loaded:
+
+```bash
+lsmod | grep nvidia
+lsmod | grep nouveau
+```
+
+These should return nothing.
+
+#### `vfio-pci.ids` not appearing
+
+Check generated bootloader config:
+
+```bash
+cat /boot/loader/entries/nixos-*.conf
+```

docs/modules/nixos/virtualization.md

@@ -67,7 +67,7 @@ nav:
       - torrenting: modules/nixos/torrenting.md
       - tt-rss: modules/nixos/tt-rss.md
       - vaultwarden: modules/nixos/vaultwarden.md
-      - virtualization: modules/nixos/virtualization.md
+      - vfio: modules/nixos/vfio.md
       - webPage: modules/nixos/webpage.md
       - xrdp: modules/nixos/xrdp.md
     - Home Manager:
@@ -83,7 +83,7 @@ nav:
       - password-manager: modules/home/password-manager.md
       - sops: modules/home/sops.md
       - stylix: modules/home/stylix.md
-      - virtualization: modules/home/virtualization.md
+      - vfio: modules/home/vfio.md
       - waybar: modules/home/waybar.md
       - yazi: modules/home/yazi.md
   - Tips:

mkdocs.yml

@@ -12,6 +12,6 @@
   passwordManager = import ./password-manager;
   sops = import ./sops;
   stylix = import ./stylix;
-  virtualization = import ./virtualization;
+  virtualisation = import ./virtualisation;
   waybar = import ./waybar;
 }

modules/home/default.nix

@@ -39,7 +39,7 @@
   torrenting = import ./torrenting;
   tt-rss = import ./tt-rss;
   vaultwarden = import ./vaultwarden;
-  virtualization = import ./virtualization;
+  virtualisation = import ./virtualisation;
   webPage = import ./webPage;
   xrdp = import ./xrdp;
 }

modules/home/virtualization/default.nix renamed to modules/home/virtualisation/default.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.virtualisation;
+
+  boolToZeroOne = x: if x then "1" else "0";
+
+  aclString = strings.concatMapStringsSep ''
+    ,
+  '' strings.escapeNixString cfg.libvirtd.deviceACL;
+
+  inherit (lib)
+    mkDefault
+    mkOption
+    optionals
+    strings
+    types
+    ;
+in
+{
+  imports = [
+    ./hugepages.nix
+    ./vfio.nix
+  ];
+
+  options.virtualisation = {
+    libvirtd = {
+      deviceACL = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [
+          "/dev/kvm"
+          "/dev/net/tun"
+          "/dev/vfio/vfio"
+        ];
+        description = "List of device paths that QEMU processes are allowed to access.";
+      };
+
+      clearEmulationCapabilities = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Whether to remove privileged Linux capabilities from QEMU processes after they start.";
+      };
+    };
+  };
+
+  config = {
+    virtualisation = {
+      libvirtd = {
+        enable = mkDefault true;
+        onBoot = mkDefault "ignore";
+        onShutdown = mkDefault "shutdown";
+        qemu.ovmf.enable = mkDefault true;
+        qemu.runAsRoot = mkDefault false;
+        qemu.verbatimConfig = ''
+          clear_emulation_capabilities = ${boolToZeroOne cfg.libvirtd.clearEmulationCapabilities}
+          cgroup_device_acl = [
+            ${aclString}
+          ]
+        '';
+      };
+      spiceUSBRedirection.enable = mkDefault true;
+    };
+
+    users.users."qemu-libvirtd" = {
+      extraGroups = optionals (!cfg.libvirtd.qemu.runAsRoot) [
+        "kvm"
+        "input"
+      ];
+      isSystemUser = true;
+    };
+
+    programs.virt-manager.enable = mkDefault true;
+
+    environment.systemPackages = [
+      (pkgs.writeShellScriptBin "iommu-groups" (builtins.readFile ./iommu-groups.sh))
+      pkgs.dnsmasq
+      pkgs.qemu_full
+    ];
+  };
+}

modules/nixos/default.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+let
+  cfg = config.virtualisation.hugepages;
+
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    optionals
+    types
+    ;
+in
+{
+  options.virtualisation.hugepages = {
+    enable = mkEnableOption "Whether to enable huge pages.";
+
+    defaultPageSize = mkOption {
+      type = types.strMatching "[0-9]*[kKmMgG]";
+      default = "1M";
+      description = "Default size of huge pages. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+    };
+
+    pageSize = mkOption {
+      type = types.strMatching "[0-9]*[kKmMgG]";
+      default = "1M";
+      description = "Size of huge pages that are allocated at boot. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+    };
+
+    numPages = mkOption {
+      type = types.ints.positive;
+      default = 1;
+      description = "Number of huge pages to allocate at boot.";
+    };
+  };
+
+  config.boot.kernelParams = optionals cfg.enable [
+    "default_hugepagesz=${cfg.defaultPageSize}"
+    "hugepagesz=${cfg.pageSize}"
+    "hugepages=${toString cfg.numPages}"
+  ];
+}