From a2b208a3f7bcef2bbb2eb57a07b37857a722082d Mon Sep 17 00:00:00 2001 From: Dmitriy Kholkin Date: Fri, 21 Oct 2022 14:01:19 +0300 Subject: [PATCH] vfio passthrough module --- machines/AMD-Workstation/default.nix | 60 +++++++++++++++ modules/passthrough/default.nix | 8 ++ modules/passthrough/libvirt.nix | 37 +++++++++ modules/passthrough/vfio.nix | 102 +++++++++++++++++++++++++ modules/passthrough/virtualisation.nix | 66 ++++++++++++++++ 5 files changed, 273 insertions(+) create mode 100644 modules/passthrough/default.nix create mode 100644 modules/passthrough/libvirt.nix create mode 100644 modules/passthrough/vfio.nix create mode 100644 modules/passthrough/virtualisation.nix diff --git a/machines/AMD-Workstation/default.nix b/machines/AMD-Workstation/default.nix index d57c8ef..d7fa1fb 100644 --- a/machines/AMD-Workstation/default.nix +++ b/machines/AMD-Workstation/default.nix @@ -2,6 +2,8 @@ imports = [ ./hardware-configuration.nix inputs.self.nixosRoles.workstation + + # inputs.self.nixosModules.passthrough ]; deviceSpecific.devInfo = { @@ -35,5 +37,63 @@ home-manager.users.alukard.home.packages = lib.mkIf config.deviceSpecific.enableVirtualisation [ inputs.nixos-generators.packages.${pkgs.system}.nixos-generators + + # pkgs.looking-glass-client ]; + + # VFIO Passthough + # virtualisation = { + # sharedMemoryFiles = { + # # scream = { + # # user = "alukard"; + # # group = "qemu-libvirtd"; + # # mode = "666"; + # # }; + # looking-glass = { + # user = "alukard"; + # group = "libvirtd"; + # mode = "666"; + # }; + # }; + # libvirtd = { + # enable = true; + # qemu = { + # ovmf.enable = true; + # runAsRoot = lib.mkForce true; + # }; +# + # onBoot = "ignore"; + # onShutdown = "shutdown"; +# + # clearEmulationCapabilities = false; +# + # deviceACL = [ + # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.4:1.0-event-mouse" # Trackball + # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.0-event-kbd" # Tastatur + # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.1-event-mouse" # Tastatur + # # "/dev/input/by-path/pci-0000:0b:00.3-usb-0:2.2.3:1.1-mouse" # Tastatur + # "/dev/vfio/vfio" + # "/dev/vfio/17" + # "/dev/kvm" + # # "/dev/shm/scream" + # "/dev/shm/looking-glass" + # ]; + # }; + # vfio = { + # enable = true; + # IOMMUType = "amd"; + # # group 17: 0b:00.0 and 0b:00.1 + # devices = [ "10de:1244" "10de:0bee" ]; + # blacklistNvidia = true; + # disableEFIfb = false; + # ignoreMSRs = true; + # applyACSpatch = false; + # }; + # hugepages = { + # enable = true; + # defaultPageSize = "1G"; + # pageSize = "1G"; + # numPages = 6; + # }; + # }; } diff --git a/modules/passthrough/default.nix b/modules/passthrough/default.nix new file mode 100644 index 0000000..e809adb --- /dev/null +++ b/modules/passthrough/default.nix @@ -0,0 +1,8 @@ +{ + # Add your NixOS modules here + # + # libvirt = ./libvirt.nix; + # vfio = ./vfio.nix; + # virtualisation.nix = ./virtualisation.nix; + imports = [ ./virtualisation.nix ./vfio.nix ./libvirt.nix ]; +} diff --git a/modules/passthrough/libvirt.nix b/modules/passthrough/libvirt.nix new file mode 100644 index 0000000..b688c84 --- /dev/null +++ b/modules/passthrough/libvirt.nix @@ -0,0 +1,37 @@ +{ lib, pkgs, config, ... }: +with lib; +let + cfg = config.virtualisation.libvirtd; + + boolToZeroOne = x: if x then "1" else "0"; + + aclString = with lib.strings; + concatMapStringsSep '' + , + '' escapeNixString cfg.deviceACL; +in { + options.virtualisation.libvirtd = { + deviceACL = mkOption { + type = types.listOf types.str; + default = [ ]; + }; + clearEmulationCapabilities = mkOption { + type = types.bool; + default = true; + }; + }; + + config.users.users."qemu-libvirtd" = { + extraGroups = optionals (!cfg.qemu.runAsRoot) [ "kvm" "input" ]; + isSystemUser = true; + }; + + config.virtualisation.libvirtd.qemu.verbatimConfig = '' + clear_emulation_capabilities = ${ + boolToZeroOne cfg.clearEmulationCapabilities + } + cgroup_device_acl = [ + ${aclString} + ] + ''; +} diff --git a/modules/passthrough/vfio.nix b/modules/passthrough/vfio.nix new file mode 100644 index 0000000..dd31142 --- /dev/null +++ b/modules/passthrough/vfio.nix @@ -0,0 +1,102 @@ +{ lib, pkgs, config, ... }: +with lib; +let + cfg = config.virtualisation.vfio; + acscommit = "1ec4cb0753488353e111496a90bdfbe2a074827e"; +in { + options.virtualisation.vfio = { + enable = mkEnableOption "VFIO Configuration"; + IOMMUType = mkOption { + type = types.enum [ "intel" "amd" ]; + example = "intel"; + description = "Type of the IOMMU used"; + }; + devices = mkOption { + type = types.listOf (types.strMatching "[0-9a-f]{4}:[0-9a-f]{4}"); + default = [ ]; + example = [ "10de:1b80" "10de:10f0" ]; + description = "PCI IDs of devices to bind to vfio-pci"; + }; + disableEFIfb = mkOption { + type = types.bool; + default = false; + example = true; + description = "Disables the usage of the EFI framebuffer on boot."; + }; + blacklistNvidia = mkOption { + type = types.bool; + default = false; + description = "Add Nvidia GPU modules to blacklist"; + }; + ignoreMSRs = mkOption { + type = types.bool; + default = false; + example = true; + description = + "Enables or disables kvm guest access to model-specific registers"; + }; + applyACSpatch = mkOption { + type = types.bool; + default = false; + description = '' + If set, the following things will happen: + - The ACS override patch is applied + - Applies the i915-vga-arbiter patch + - Adds pcie_acs_override=downstream to the command line + ''; + }; + }; + + config = lib.mkIf cfg.enable { + services.udev.extraRules = '' + SUBSYSTEM=="vfio", OWNER="root", GROUP="kvm" + ''; + + boot.kernelParams = (if cfg.IOMMUType == "intel" then [ + "intel_iommu=on" + "intel_iommu=igfx_off" + ] else + [ "amd_iommu=on" ]) ++ (optional (builtins.length cfg.devices > 0) + ("vfio-pci.ids=" + builtins.concatStringsSep "," cfg.devices)) + ++ (optionals cfg.applyACSpatch [ + "pcie_acs_override=downstream,multifunction" + "pci=nomsi" + ]) ++ (optional cfg.disableEFIfb "video=efifb:off") + ++ (optionals cfg.ignoreMSRs [ + "kvm.ignore_msrs=1" + "kvm.report_ignored_msrs=0" + ]); + + boot.kernelModules = [ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ]; + + boot.initrd.kernelModules = + [ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ]; + boot.blacklistedKernelModules = + optionals cfg.blacklistNvidia [ "nvidia" "nouveau" ]; + + boot.kernelPatches = optionals cfg.applyACSpatch [ + { + name = "add-acs-overrides"; + patch = pkgs.fetchurl { + name = "add-acs-overrides.patch"; + url = + "https://raw.githubusercontent.com/slowbro/linux-vfio/v5.5.4-arch1/add-acs-overrides.patch"; + #url = + # "https://aur.archlinux.org/cgit/aur.git/plain/add-acs-overrides.patch?h=linux-vfio&id=${acscommit}"; + sha256 = "0nbmc5bwv7pl84l1mfhacvyp8vnzwhar0ahqgckvmzlhgf1n1bii"; + }; + } + { + name = "i915-vga-arbiter"; + patch = pkgs.fetchurl { + name = "i915-vga-arbiter.patch"; + url = + "https://raw.githubusercontent.com/slowbro/linux-vfio/v5.5.4-arch1/i915-vga-arbiter.patch"; + #url = + # "https://aur.archlinux.org/cgit/aur.git/plain/i915-vga-arbiter.patch?h=linux-vfio&id=${acscommit}"; + sha256 = "1m5nn9pfkf685g31y31ip70jv61sblvxgskqn8a0ca60mmr38krk"; + }; + } + ]; + }; +} diff --git a/modules/passthrough/virtualisation.nix b/modules/passthrough/virtualisation.nix new file mode 100644 index 0000000..79e5783 --- /dev/null +++ b/modules/passthrough/virtualisation.nix @@ -0,0 +1,66 @@ +{ lib, pkgs, config, ... }: +with lib; +let + cfg = config.virtualisation; + tmpfileEntry = name: f: "f /dev/shm/${name} ${f.mode} ${f.user} ${f.group} -"; +in { + options.virtualisation = { + sharedMemoryFiles = mkOption { + type = types.attrsOf (types.submodule ({ name, ... }: { + options = { + name = mkOption { + visible = false; + default = name; + type = types.str; + }; + user = mkOption { + type = types.str; + default = "root"; + description = "Owner of the memory file"; + }; + group = mkOption { + type = types.str; + default = "root"; + description = "Group of the memory file"; + }; + mode = mkOption { + type = types.str; + default = "0600"; + description = "Group of the memory file"; + }; + }; + })); + default = { }; + }; + hugepages = { + enable = mkEnableOption "Hugepages"; + + defaultPageSize = mkOption { + type = types.strMatching "[0-9]*[kKmMgG]"; + default = "1M"; + description = + "Default size of huge pages. You can use suffixes K, M, and G to specify KB, MB, and GB."; + }; + pageSize = mkOption { + type = types.strMatching "[0-9]*[kKmMgG]"; + default = "1M"; + description = + "Size of huge pages that are allocated at boot. You can use suffixes K, M, and G to specify KB, MB, and GB."; + }; + numPages = mkOption { + type = types.ints.positive; + default = 1; + description = "Number of huge pages to allocate at boot."; + }; + }; + }; + + config.systemd.tmpfiles.rules = + mapAttrsToList (tmpfileEntry) cfg.sharedMemoryFiles; + + config.boot.kernelParams = optionals cfg.hugepages.enable [ + "default_hugepagesz=${cfg.hugepages.defaultPageSize}" + "hugepagesz=${cfg.hugepages.pageSize}" + "hugepages=${toString cfg.hugepages.numPages}" + ]; +}