vfio passthrough module

machines/AMD-Workstation/default.nix

@@ -2,6 +2,8 @@
   imports = [
     ./hardware-configuration.nix
     inputs.self.nixosRoles.workstation
+
+    # inputs.self.nixosModules.passthrough
   ];
 
   deviceSpecific.devInfo = {
@@ -35,5 +37,63 @@
 
   home-manager.users.alukard.home.packages = lib.mkIf config.deviceSpecific.enableVirtualisation [
     inputs.nixos-generators.packages.${pkgs.system}.nixos-generators
+
+    # pkgs.looking-glass-client
   ];
+
+  # VFIO Passthough
+  # virtualisation = {
+    # sharedMemoryFiles = {
+      # # scream = {
+      # #   user = "alukard";
+      # #   group = "qemu-libvirtd";
+      # #   mode = "666";
+      # # };
+      # looking-glass = {
+        # user = "alukard";
+        # group = "libvirtd";
+        # mode = "666";
+      # };
+    # };
+    # libvirtd = {
+      # enable = true;
+      # qemu = {
+        # ovmf.enable = true;
+        # runAsRoot = lib.mkForce true;
+      # };
+#
+      # onBoot = "ignore";
+      # onShutdown = "shutdown";
+#
+      # clearEmulationCapabilities = false;
+#
+      # deviceACL = [
+        # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.4:1.0-event-mouse" # Trackball
+        # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.0-event-kbd" # Tastatur
+        # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.1-event-mouse" # Tastatur
+        # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.1-mouse" # Tastatur
+        # "/dev/vfio/vfio"
+        # "/dev/vfio/17"
+        # "/dev/kvm"
+        # # "/dev/shm/scream"
+        # "/dev/shm/looking-glass"
+      # ];
+    # };
+    # vfio = {
+      # enable = true;
+      # IOMMUType = "amd";
+      # # group 17: 0b:00.0 and 0b:00.1
+      # devices = [ "10de:1244" "10de:0bee" ];
+      # blacklistNvidia = true;
+      # disableEFIfb = false;
+      # ignoreMSRs = true;
+      # applyACSpatch = false;
+    # };
+    # hugepages = {
+      # enable = true;
+      # defaultPageSize = "1G";
+      # pageSize = "1G";
+      # numPages = 6;
+    # };
+  # };
 }

modules/passthrough/default.nix

@@ -0,0 +1,8 @@
+{
+  # Add your NixOS modules here
+  #
+  # libvirt = ./libvirt.nix;
+  # vfio = ./vfio.nix;
+  # virtualisation.nix = ./virtualisation.nix;
+  imports = [ ./virtualisation.nix ./vfio.nix ./libvirt.nix ];
+}

modules/passthrough/libvirt.nix

@@ -0,0 +1,37 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.virtualisation.libvirtd;
+
+  boolToZeroOne = x: if x then "1" else "0";
+
+  aclString = with lib.strings;
+    concatMapStringsSep ''
+      ,
+        '' escapeNixString cfg.deviceACL;
+in {
+  options.virtualisation.libvirtd = {
+    deviceACL = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+    clearEmulationCapabilities = mkOption {
+      type = types.bool;
+      default = true;
+    };
+  };
+
+  config.users.users."qemu-libvirtd" = {
+    extraGroups = optionals (!cfg.qemu.runAsRoot) [ "kvm" "input" ];
+    isSystemUser = true;
+  };
+
+  config.virtualisation.libvirtd.qemu.verbatimConfig = ''
+    clear_emulation_capabilities = ${
+      boolToZeroOne cfg.clearEmulationCapabilities
+    }
+    cgroup_device_acl = [
+      ${aclString}
+    ]
+  '';
+}

modules/passthrough/vfio.nix

@@ -0,0 +1,102 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.virtualisation.vfio;
+  acscommit = "1ec4cb0753488353e111496a90bdfbe2a074827e";
+in {
+  options.virtualisation.vfio = {
+    enable = mkEnableOption "VFIO Configuration";
+    IOMMUType = mkOption {
+      type = types.enum [ "intel" "amd" ];
+      example = "intel";
+      description = "Type of the IOMMU used";
+    };
+    devices = mkOption {
+      type = types.listOf (types.strMatching "[0-9a-f]{4}:[0-9a-f]{4}");
+      default = [ ];
+      example = [ "10de:1b80" "10de:10f0" ];
+      description = "PCI IDs of devices to bind to vfio-pci";
+    };
+    disableEFIfb = mkOption {
+      type = types.bool;
+      default = false;
+      example = true;
+      description = "Disables the usage of the EFI framebuffer on boot.";
+    };
+    blacklistNvidia = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Add Nvidia GPU modules to blacklist";
+    };
+    ignoreMSRs = mkOption {
+      type = types.bool;
+      default = false;
+      example = true;
+      description =
+        "Enables or disables kvm guest access to model-specific registers";
+    };
+    applyACSpatch = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        If set, the following things will happen:
+          - The ACS override patch is applied
+          - Applies the i915-vga-arbiter patch
+          - Adds pcie_acs_override=downstream to the command line
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.udev.extraRules = ''
+      SUBSYSTEM=="vfio", OWNER="root", GROUP="kvm"
+    '';
+
+    boot.kernelParams = (if cfg.IOMMUType == "intel" then [
+      "intel_iommu=on"
+      "intel_iommu=igfx_off"
+    ] else
+      [ "amd_iommu=on" ]) ++ (optional (builtins.length cfg.devices > 0)
+        ("vfio-pci.ids=" + builtins.concatStringsSep "," cfg.devices))
+      ++ (optionals cfg.applyACSpatch [
+        "pcie_acs_override=downstream,multifunction"
+        "pci=nomsi"
+      ]) ++ (optional cfg.disableEFIfb "video=efifb:off")
+      ++ (optionals cfg.ignoreMSRs [
+        "kvm.ignore_msrs=1"
+        "kvm.report_ignored_msrs=0"
+      ]);
+
+    boot.kernelModules = [ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ];
+
+    boot.initrd.kernelModules =
+      [ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ];
+    boot.blacklistedKernelModules =
+      optionals cfg.blacklistNvidia [ "nvidia" "nouveau" ];
+
+    boot.kernelPatches = optionals cfg.applyACSpatch [
+      {
+        name = "add-acs-overrides";
+        patch = pkgs.fetchurl {
+          name = "add-acs-overrides.patch";
+          url =
+            "https://raw.githubusercontent.com/slowbro/linux-vfio/v5.5.4-arch1/add-acs-overrides.patch";
+          #url =
+          #  "https://aur.archlinux.org/cgit/aur.git/plain/add-acs-overrides.patch?h=linux-vfio&id=${acscommit}";
+          sha256 = "0nbmc5bwv7pl84l1mfhacvyp8vnzwhar0ahqgckvmzlhgf1n1bii";
+        };
+      }
+      {
+        name = "i915-vga-arbiter";
+        patch = pkgs.fetchurl {
+          name = "i915-vga-arbiter.patch";
+          url =
+            "https://raw.githubusercontent.com/slowbro/linux-vfio/v5.5.4-arch1/i915-vga-arbiter.patch";
+          #url =
+          #  "https://aur.archlinux.org/cgit/aur.git/plain/i915-vga-arbiter.patch?h=linux-vfio&id=${acscommit}";
+          sha256 = "1m5nn9pfkf685g31y31ip70jv61sblvxgskqn8a0ca60mmr38krk";
+        };
+      }
+    ];
+  };
+}

modules/passthrough/virtualisation.nix

@@ -0,0 +1,66 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.virtualisation;
+  tmpfileEntry = name: f: "f /dev/shm/${name} ${f.mode} ${f.user} ${f.group} -";
+in {
+  options.virtualisation = {
+    sharedMemoryFiles = mkOption {
+      type = types.attrsOf (types.submodule ({ name, ... }: {
+        options = {
+          name = mkOption {
+            visible = false;
+            default = name;
+            type = types.str;
+          };
+          user = mkOption {
+            type = types.str;
+            default = "root";
+            description = "Owner of the memory file";
+          };
+          group = mkOption {
+            type = types.str;
+            default = "root";
+            description = "Group of the memory file";
+          };
+          mode = mkOption {
+            type = types.str;
+            default = "0600";
+            description = "Group of the memory file";
+          };
+        };
+      }));
+      default = { };
+    };
+    hugepages = {
+      enable = mkEnableOption "Hugepages";
+
+      defaultPageSize = mkOption {
+        type = types.strMatching "[0-9]*[kKmMgG]";
+        default = "1M";
+        description =
+          "Default size of huge pages. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+      };
+      pageSize = mkOption {
+        type = types.strMatching "[0-9]*[kKmMgG]";
+        default = "1M";
+        description =
+          "Size of huge pages that are allocated at boot. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+      };
+      numPages = mkOption {
+        type = types.ints.positive;
+        default = 1;
+        description = "Number of huge pages to allocate at boot.";
+      };
+    };
+  };
+
+  config.systemd.tmpfiles.rules =
+    mapAttrsToList (tmpfileEntry) cfg.sharedMemoryFiles;
+
+  config.boot.kernelParams = optionals cfg.hugepages.enable [
+    "default_hugepagesz=${cfg.hugepages.defaultPageSize}"
+    "hugepagesz=${cfg.hugepages.pageSize}"
+    "hugepages=${toString cfg.hugepages.numPages}"
+  ];
+}