update some modules to new install

machines/AMD-Workstation/autoinstall.nix

@@ -1,5 +1,6 @@
 { ... }: {
   autoinstall.AMD-Workstation = {
+#     debug = true;
     mainuser = "ataraxia";
     flakesPath = "/home/nixos/nixos-config";
     encryption.encryptBoot = false;
@@ -10,8 +11,10 @@
     partitioning.disk = "/dev/disk/by-id/nvme-XPG_GAMMIX_S11_Pro_2K342L2BBNUY";
     partitioning.nullifyDisk = false;
     partitioning.emptySpace = "100GiB";
+    partitioning.createBootPool = true;
     swapPartition.enable = true;
     swapPartition.size = "16GiB";
+    efiMountPoint = "/efi";
     zfsOpts.ashift = 13;
     zfsOpts.bootPoolReservation = "256M";
     zfsOpts.rootPoolReservation = "45G";

machines/AMD-Workstation/boot.nix

@@ -4,12 +4,13 @@ let
 in {
   boot = {
     initrd = {
+      supportedFilesystems = [ "zfs" ];
       luks.devices = {
         "cryptroot" = {
           preLVM = true;
           keyFile = "/keyfile0.bin";
           allowDiscards = true;
-          bypassWorkqueues = config.deviceSpecific.isSSD;
+          bypassWorkqueues = true;
           fallbackToPassword = true;
         };
       };
@@ -19,23 +20,53 @@ in {
     };
 
     loader = {
-      systemd-boot = {
+      grub = {
         enable = true;
+        device = "nodev";
+        version = 2;
+        efiSupport = true;
+        enableCryptodisk = true;
+        zfsSupport = true;
+#         efiInstallAsRemovable = true;
+        copyKernels = true;
+      };
+      systemd-boot = {
+        enable = lib.mkForce false;
         editor = false;
         configurationLimit = 10;
+#         graceful = true;
       };
-      efi.canTouchEfiVariables = false;
+      efi.canTouchEfiVariables = true;
-      efi.efiSysMountPoint = "/boot/efi";
+      efi.efiSysMountPoint = "/efi";
       generationsDir.copyKernels = true;
     };
 
-    binfmt.emulatedSystems = [ "aarch64-linux" ];
+#     binfmt.emulatedSystems = [ "aarch64-linux" ];
-    kernelPackages = pkgs.linuxPackages_lqx;
+#     kernelPackages = lib.mkForce pkgs.linuxPackages_lqx;
+    kernelPackages = lib.mkForce config.boot.zfs.package.latestCompatibleLinuxPackages;
     kernelParams = [
       "zfs.metaslab_lba_weighting_enabled=0"
       "zfs.zfs_arc_max=${zfs_arc_max}"
     ];
     tmpOnTmpfs = true;
     tmpOnTmpfsSize = "32G";
+
+#     zfs.extraPools = [ "rpool" ];
   };
+
+  persist = {
+    enable = true;
+    cache.clean.enable = false;
+    state.files = [
+#       "/etc/machine-id"
+      "/etc/NIXOS"
+    ];
+  };
+
+  fileSystems."/home".neededForBoot = true;
+  fileSystems."/persist".neededForBoot = true;
+  boot.initrd.postDeviceCommands = lib.mkAfter ''
+    zfs rollback -r rpool/nixos/root@empty
+    zfs rollback -r rpool/user/home@empty
+  '';
 }

machines/AMD-Workstation/default.nix

@@ -36,9 +36,9 @@
   deviceSpecific.vpn.mullvad.enable = true;
 
   hardware.video.hidpi.enable = lib.mkForce false;
-  hardware.firmware = [ pkgs.rtl8761b-firmware ];
+#   hardware.firmware = [ pkgs.rtl8761b-firmware ];
 
-  networking.firewall.allowedTCPPorts = [ 52736 ];
+#   networking.firewall.allowedTCPPorts = [ 52736 ];
 
   secrets.files-veracrypt = {};
   environment.etc.crypttab = {
@@ -79,17 +79,17 @@
       inputs.nixos-generators.packages.${pkgs.hostPlatform.system}.nixos-generate
       # inputs.prismlauncher.packages.${pkgs.hostPlatform.system}.default
       # pkgs.looking-glass-client
-      pkgs.prismlauncher
+#       pkgs.prismlauncher
       pkgs.piper
       pkgs.osu-lazer-bin
       pkgs.nixpkgs-review
       pkgs.anydesk
       pkgs.winbox
     ];
-    home.stateVersion = "22.11";
+    home.stateVersion = "23.05";
   };
 
-  system.stateVersion = "22.11";
+  system.stateVersion = "23.05";
 
   # VFIO Passthough
   # virtualisation = {

machines/AMD-Workstation/hardware-configuration.nix

@@ -8,7 +8,7 @@
     [ (modulesPath + "/installer/scan/not-detected.nix")
     ];
 
-  boot.initrd.availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-amd" ];
   boot.extraModulePackages = [ ];
@@ -18,63 +18,92 @@
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
-  fileSystems."/nix" =
-    { device = "rpool/nixos/nix";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
-    };
-
   fileSystems."/home" =
     { device = "rpool/user/home";
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
-  fileSystems."/var/lib" =
+  fileSystems."/persist" =
-    { device = "rpool/nixos/var/lib";
+    { device = "rpool/persistent/impermanence";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/srv" =
+    { device = "rpool/persistent/servers";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/etc/secrets" =
+    { device = "rpool/persistent/secrets";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/nix" =
+    { device = "rpool/persistent/nix";
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
   fileSystems."/var/log" =
-    { device = "rpool/nixos/var/log";
+    { device = "rpool/persistent/log";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/lib/docker" =
+    { device = "rpool/persistent/docker";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/lib/containers" =
+    { device = "rpool/persistent/containers";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/lib/nixos-containers" =
+    { device = "rpool/persistent/nixos-containers";
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
   fileSystems."/media/bittorrent" =
-    { device = "rpool/nixos/bittorrent";
+    { device = "rpool/persistent/bittorrent";
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
   fileSystems."/media/libvirt" =
-    { device = "rpool/nixos/libvirt";
+    { device = "rpool/persistent/libvirt";
       fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/948B-11EC";
+    { device = "bpool/nixos/boot";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/efi" =
+    { device = "/dev/disk/by-uuid/A556-CD19";
       fsType = "vfat";
     };
 
   swapDevices = [
     {
-      device = "/dev/disk/by-partuuid/7ffa34d9-862b-42ff-a649-da54f7b8fbf0";
+      device = "/dev/disk/by-partuuid/5305d817-d4ef-41a7-a51b-dc1fb8638227";
-      randomEncryption.enable = false;
+      randomEncryption.enable = true;
+      randomEncryption.allowDiscards = true;
     }
   ];
 
-
-
   # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
   # (the default) this is the recommended approach. When using systemd-networkd it's
   # still possible to use this option, but it's recommended to use it in conjunction
   # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
   networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp9s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp8s0.useDHCP = lib.mkDefault true;
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
   hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
   # high-resolution display
   hardware.video.hidpi.enable = lib.mkDefault true;
-  networking.hostId = "a32bd2dc";
+  networking.hostId = "0c00ab80";
-  boot.zfs.devNodes = "/dev/disk/by-partuuid";
+  boot.zfs.devNodes = "/dev/disk/by-id";
   boot.supportedFilesystems = [ "zfs" ];
-  boot.kernelParams = [ "zfs.zfs_arc_max=8589934592" ];
+  boot.initrd.luks.devices."cryptroot".device = "/dev/disk/by-partuuid/67e044d7-1a06-4a59-826a-bf24994934a7";
 }

modules/autoinstall/default.nix

@@ -30,7 +30,12 @@ let
         emptySpace = mkOption {
           type = types.str;
           default = "0";
-          description = "Empty space at the end of the disk`";
+          description = "Empty space at the end of the disk";
+        };
+        createBootPool = mkOption {
+          type = types.bool;
+          default = true;
+          description = "";
         };
         # partitions = mkOption {
         #   type = types.nullOr attrsOf partitionsAttrs;
@@ -58,6 +63,11 @@ let
         default = "512MiB";
         description = "Size of EFI partition";
       };
+      efiMountPoint = mkOption {
+        type = types.str;
+        default = "/boot";
+        description = "EFI mount point";
+      };
       bootSize = mkOption {
         type = types.str;
         default = "4GiB";

modules/autoinstall/install.nix

@@ -9,6 +9,7 @@ with lib; let
     bootPartition = opt.partitioning.partitions.bootPartition or "0";
     rootPartition = opt.partitioning.partitions.rootPartition or "0";
     swapPartition = opt.partitioning.partitions.swapPartition or "0";
+    createBootPool = boolToString opt.partitioning.createBootPool;
     emptySpace = opt.partitioning.emptySpace or "0";
     debug = boolToString opt.debug;
     useSwap = boolToString opt.swapPartition.enable;
@@ -43,6 +44,9 @@ in ''
     bootPartition="${cfg.bootPartition}"
     rootPartition="${cfg.rootPartition}"
     swapPartition="${cfg.swapPartition}"
+    efiMountPoint="${cfg.efiMountPoint}"
+    emptySpace="${cfg.emptySpace}"
+    createBootPool="${cfg.createBootPool}"
     efiSize="${cfg.efiSize}"
     bootSize="${cfg.bootSize}"
     rootSize="${cfg.rootSize}"
@@ -75,9 +79,11 @@ in ''
     exit 2
   fi
 
-  if [ "${cfg.encryptBoot}" = "true" || "${cfg.encryptRoot}" = "true" && ! -f "${cfg.passwordFile}" ]; then
+  if [ "${cfg.encryptBoot}" = "true" || "${cfg.encryptRoot}" = "true" ]; then
-    pprint "passwordFile does not exists!"
+    if [ ! -f "${cfg.passwordFile}" ]; then
-    exit 2
+      pprint "passwordFile does not exists!"
+      exit 2
+    fi
   fi
 
   create_new_part_table() {
@@ -123,13 +129,15 @@ in ''
     sgdisk -n1:1MiB:+${cfg.efiSize} -t1:EF00 "$diskByID"
     efiPart="$diskByID-part1"
 
-    pprint "Creating boot (ZFS) partition"
+    if [ "${cfg.createBootPool}" = "true" ]; then
-    if [ "${cfg.encryptBoot}" = "true" ]; then
+      pprint "Creating boot (ZFS) partition"
-      sgdisk -n2:0:+${cfg.bootSize} -t2:8309 "$diskByID"
+      if [ "${cfg.encryptBoot}" = "true" ]; then
-    else
+        sgdisk -n2:0:+${cfg.bootSize} -t2:8309 "$diskByID"
-      sgdisk -n2:0:+${cfg.bootSize} -t2:BF00 "$diskByID"
+      else
+        sgdisk -n2:0:+${cfg.bootSize} -t2:BF00 "$diskByID"
+      fi
+      bootPart="$diskByID-part2"
     fi
-    bootPart="$diskByID-part2"
 
     if [ "${cfg.emptySpace}" != "0" ]; then
       pprint "Creating temp empty partition at the end of the disk"
@@ -138,7 +146,7 @@ in ''
 
     if [ "${cfg.useSwap}" = "true" ]; then
       pprint "Creating SWAP partition"
-      sgdisk -n4:0:+${cfg.swapSize} -t4:8200 "$diskByID"
+      sgdisk -n4:-${cfg.swapSize}:0 -t4:8200 "$diskByID"
       swapPart="$diskByID-part4"
     fi
 
@@ -175,15 +183,19 @@ in ''
     password=$(cat ${cfg.passwordFile})
     dd if=/dev/urandom of=/tmp/keyfile0.bin bs=1024 count=4
 
-    if [ "${cfg.encryptBoot}" = "true" ]; then
+    if [ "${cfg.createBootPool}" = "true" ]; then
-      pprint "Creating LUKS container on $bootPart"
+      if [ "${cfg.encryptBoot}" = "true" ]; then
-      echo -n "$password" | cryptsetup --type luks2 --pbkdf argon2id --iter-time ${cfg.argonIterTime} -c aes-xts-plain64 -s 512 -h sha256 luksFormat "$bootPart" -
+        pprint "Creating LUKS container on $bootPart"
-      pprint "Add keyfile to LUKS container on $bootPart"
+        echo -n "$password" | cryptsetup --type luks2 --pbkdf argon2id --iter-time ${cfg.argonIterTime} -c aes-xts-plain64 -s 512 -h sha256 luksFormat "$bootPart" -
-      echo -n "$password" | cryptsetup luksAddKey $bootPart /tmp/keyfile0.bin -
+        pprint "Add keyfile to LUKS container on $bootPart"
+        echo -n "$password" | cryptsetup luksAddKey $bootPart /tmp/keyfile0.bin -
 
-      pprint "Open LUKS container on $bootPart"
+        pprint "Open LUKS container on $bootPart"
-      cryptsetup luksOpen --allow-discards "$bootPart" "${cfg.cryptBoot}" -d /tmp/keyfile0.bin
+        cryptsetup luksOpen --allow-discards "$bootPart" "${cfg.cryptBoot}" -d /tmp/keyfile0.bin
-      bootPool="$(ls /dev/disk/by-id/dm-uuid-*${cfg.cryptBoot})"
+        bootPool="$(ls /dev/disk/by-id/dm-uuid-*${cfg.cryptBoot})"
+      else
+        bootPool="$bootPart"
+      fi
     fi
 
     if [ "${cfg.encryptRoot}" = "true" ]; then
@@ -195,9 +207,11 @@ in ''
       pprint "Open LUKS container on $rootPart"
       cryptsetup luksOpen --allow-discards "$rootPart" "${cfg.cryptRoot}" -d /tmp/keyfile0.bin
       rootPool="$(ls /dev/disk/by-id/dm-uuid-*${cfg.cryptRoot})"
+    else
+      rootPool="$rootPart"
     fi
   else
-    bootPool="$bootPart"
+    [ "${cfg.createBootPool}" = "true" ] && bootPool="$bootPart"
     rootPool="$rootPart"
   fi
 
@@ -245,7 +259,7 @@ in ''
   zfs create -o canmount=on -o mountpoint=/var/log rpool/persistent/log
   zfs create -o canmount=noauto -o atime=off rpool/persistent/lxd
   zfs create -o canmount=on -o mountpoint=/var/lib/docker -o atime=off rpool/persistent/docker
-  zfs create -o canmount=on -o mountpoint=/var/lib/podman -o atime=off rpool/persistent/podman
+  zfs create -o canmount=on -o mountpoint=/var/lib/containers -o atime=off rpool/persistent/containers
   zfs create -o canmount=on -o mountpoint=/var/lib/nixos-containers -o atime=off rpool/persistent/nixos-containers
   zfs create -o canmount=on -o mountpoint=/media/bittorrent -o atime=off -o recordsize=16K -o compression=lz4 rpool/persistent/bittorrent
   chown 1000:100 /mnt/media/bittorrent
@@ -267,41 +281,42 @@ in ''
   zfs snapshot rpool/persistent/log@empty
   zfs snapshot rpool/persistent/lxd@empty
   zfs snapshot rpool/persistent/docker@empty
-  zfs snapshot rpool/persistent/podman@empty
+  zfs snapshot rpool/persistent/containers@empty
   zfs snapshot rpool/persistent/nixos-containers@empty
   zfs snapshot rpool/persistent/bittorrent@empty
   zfs snapshot rpool/persistent/libvirt@empty
 
+  if [ "${cfg.createBootPool}" = "true" ]; then
+    pprint "Create ZFS boot pool on $bootPool"
+    zpool create \
+      -f \
+      -o compatibility=grub2 \
+      -o ashift=${cfg.zfsAshift} \
+      -o autotrim=on \
+      -O acltype=posixacl \
+      -O atime=on \
+      -O canmount=off \
+      -O compression=lz4 \
+      -O devices=off \
+      -O normalization=formD \
+      -O relatime=on \
+      -O xattr=sa \
+      -O dedup=off \
+      -O mountpoint=/boot \
+      -R /mnt \
+      bpool "$bootPool"
 
-  pprint "Create ZFS boot pool on $bootPool"
+    pprint "Create ZFS boot datasets"
-  zpool create \
-    -f \
-    -o compatibility=grub2 \
-    -o ashift=${cfg.zfsAshift} \
-    -o autotrim=on \
-    -O acltype=posixacl \
-    -O atime=on \
-    -O canmount=off \
-    -O compression=lz4 \
-    -O devices=off \
-    -O normalization=formD \
-    -O relatime=on \
-    -O xattr=sa \
-    -O dedup=off \
-    -O mountpoint=/boot \
-    -R /mnt \
-    bpool "$bootPool"
 
-  pprint "Create ZFS boot datasets"
+    if [ "${cfg.bootPoolReservation}" != "0" ]; then
+      zfs create -o refreservation=${cfg.bootPoolReservation} -o canmount=off -o mountpoint=none bpool/reserved
+    fi
+    zfs create -o canmount=off -o mountpoint=none bpool/nixos
+    zfs create -o canmount=on -o mountpoint=/boot bpool/nixos/boot
 
-  if [ "${cfg.bootPoolReservation}" != "0" ]; then
+    zfs snapshot bpool/nixos@empty
-    zfs create -o refreservation=${cfg.bootPoolReservation} -o canmount=off -o mountpoint=none bpool/reserved
+    zfs snapshot bpool/nixos/boot@empty
   fi
-  zfs create -o canmount=off -o mountpoint=none bpool/nixos
-  zfs create -o canmount=on -o mountpoint=/boot bpool/nixos/boot
-
-  zfs snapshot bpool/nixos@empty
-  zfs snapshot bpool/nixos/boot@empty
 
   # Disable cache, stale cache will prevent system from booting
   if [ "${cfg.usePersistModule}" = "true" ]; then
@@ -318,8 +333,8 @@ in ''
       chattr +i /mnt/etc/zfs/zpool.cache
   fi
 
-  mkdir -p /mnt/boot/efi
+  mkdir -p /mnt${cfg.efiMountPoint}
-  mount -t vfat "$efiPart" /mnt/boot/efi
+  mount -t vfat "$efiPart" /mnt${cfg.efiMountPoint}
 
   if [ "${cfg.useSwap}" = "true" ]; then
       mkswap -L swap -f "$swapPart"
@@ -338,19 +353,19 @@ in ''
   hardwareConfig=$(mktemp)
   cat <<CONFIG > "$hardwareConfig"
     networking.hostId = "$hostID";
-    boot.zfs.devNodes = "/dev/disk/by-partuuid";
+    boot.zfs.devNodes = "/dev/disk/by-id";
     boot.supportedFilesystems = [ "zfs" ];
   CONFIG
   if [ "${cfg.encryptBoot}" = "true" ]; then
     bootPartUuid=$(blkid --match-tag PARTUUID --output value "$bootPart")
     cat <<CONFIG >> "$hardwareConfig"
-      boot.initrd.luks.devices."${cfg.cryptBoot}".device = "/dev/disk/by-partuuid/$bootPartUuid";
+    boot.initrd.luks.devices."${cfg.cryptBoot}".device = "/dev/disk/by-partuuid/$bootPartUuid";
   CONFIG
   fi
   if [ "${cfg.encryptRoot}" = "true" ]; then
     rootPartUuid=$(blkid --match-tag PARTUUID --output value "$rootPart")
     cat <<CONFIG >> "$hardwareConfig"
-      boot.initrd.luks.devices."${cfg.cryptRoot}".device = "/dev/disk/by-partuuid/$rootPartUuid";
+    boot.initrd.luks.devices."${cfg.cryptRoot}".device = "/dev/disk/by-partuuid/$rootPartUuid";
   CONFIG
   fi
 
@@ -376,31 +391,33 @@ in ''
 
   if [ "${cfg.debug}" != "true" ]; then
     nixos-install --flake "${cfg.flakesPath}/#${cfg.hostname}" --root /mnt --no-root-passwd
-
-    configPath="/mnt/persist/home/"${cfg.mainuser}"/nixos-config"
-    if [ ! -d "$configPath" ]; then
-      mkdir -p $configPath
-      chown 1000:100 $configPath
-    fi
-    cp -aT ${cfg.flakesPath} $configPath
   fi
 
+  configPath="/mnt/persist/home/"${cfg.mainuser}"/nixos-config"
+  if [ ! -d "$configPath" ]; then
+    mkdir -p $configPath
+    chown 1000:100 $configPath
+  fi
+  cp -aT ${cfg.flakesPath} $configPath
+
   if [ "${cfg.oldUefi}" = "true" ]; then
     mkdir -p /mnt/boot/efi/EFI/Microsoft/Boot
     cp /mnt/boot/efi/EFI/BOOT/BOOTX64.EFI /mnt/boot/efi/EFI/Microsoft/Boot/bootmgr.efi
     cp /mnt/boot/efi/EFI/BOOT/BOOTX64.EFI /mnt/boot/efi/EFI/Microsoft/Boot/bootmgfw.efi
   fi
 
-  umount -Rl /mnt
+  if [ "${cfg.debug}" != "true" ]; then
-  zpool export -a
+    umount -Rl /mnt
-  [ "${cfg.encryptBoot}" = "true" ] && cryptsetup luksClose ${cfg.cryptBoot}
+    zpool export -a
-  [ "${cfg.encryptRoot}" = "true" ] && cryptsetup luksClose ${cfg.cryptRoot}
+    [ "${cfg.encryptBoot}" = "true" ] && cryptsetup luksClose ${cfg.cryptBoot}
+    [ "${cfg.encryptRoot}" = "true" ] && cryptsetup luksClose ${cfg.cryptRoot}
 
-  if [ "${cfg.autoReboot}" = "true" ]; then
+    if [ "${cfg.autoReboot}" = "true" ]; then
-    if ! systemctl reboot --firmware-setup ; then
+      if ! systemctl reboot --firmware-setup ; then
-      pprint "Reboot into efi firmware setup failed! Shutdown in 30 seconds"
+        pprint "Reboot into efi firmware setup failed! Shutdown in 30 seconds"
-      sleep 30
+        sleep 30
-      systemctl poweroff
+        systemctl poweroff
+      fi
     fi
   fi
 ''

profiles/network.nix

@@ -54,7 +54,7 @@ with config.deviceSpecific;
     '';
   };
 
-  persist.state.directories = lib.mkIf config.networkmanager.enable [
+  persist.state.directories = lib.mkIf config.networking.networkmanager.enable [
     "/etc/NetworkManager/system-connections"
   ];
 }

profiles/virtualisation.nix

@@ -1,93 +1,98 @@
 { config, lib, pkgs, ... }:
 with config.deviceSpecific; {
   config = lib.mkIf enableVirtualisation {
-    programs.extra-container.enable = true;
+    virtualisation = {
-    virtualisation.docker = {
+      oci-containers.backend = lib.mkForce "podman";
-      enable = true;
+      docker = {
-      daemon.settings = {
+        enable = true;
-        features = { buildkit = true; };
+        daemon.settings = {
+          features = { buildkit = true; };
+        };
+        storageDriver = "overlay2";
       };
-      storageDriver = if (devInfo.fileSystem == "zfs") then
+      podman = {
-        "zfs"
+        enable = true;
-      else if (devInfo.fileSystem == "btrfs") then
+#         extraPackages = [ pkgs.zfs ];
-        "btrfs"
+#         dockerSocket.enable = true;
-      else
+        defaultNetwork.settings.dns_enabled = true;
-        "overlay2";
-    };
-    virtualisation.oci-containers.backend = "podman";
-    virtualisation.podman = {
-      enable = true;
-      extraPackages = [ pkgs.zfs ];
-      defaultNetwork.settings.dns_enabled = true;
-    };
-    virtualisation.containers.registries.search = [
-      "docker.io" "gcr.io" "quay.io"
-    ];
-    virtualisation.containers.storage.settings =
-      lib.mkIf (devInfo.fileSystem == "zfs") {
-      storage = {
-        driver = "zfs";
-        graphroot = "/var/lib/containers/storage";
-        runroot = "/run/containers/storage";
       };
+      containers.registries.search = [
+        "docker.io" "gcr.io" "quay.io"
+      ];
+      containers.storage.settings = {
+        storage = {
+          driver = "overlay2";
+          graphroot = "/var/lib/containers/storage";
+          runroot = "/run/containers/storage";
+        };
+      };
+      lxd = lib.mkIf (!isContainer) {
+        enable = true;
+        zfsSupport = devInfo.fileSystem == "zfs";
+        recommendedSysctlSettings = true;
+      };
+      lxc = {
+        enable = true;
+        lxcfs.enable = true;
+        systemConfig = ''
+          lxc.lxcpath = /var/lib/lxd/containers
+          ${if devInfo.fileSystem == "zfs" then ''
+            lxc.bdev.zfs.root = rpool/persistent/lxd
+          '' else ""}
+        '';
+#         defaultConfig = ''
+#           lxc.idmap = u 0 100000 65535
+#           lxc.idmap = g 0 100000 65535
+#           lxc.include = ${pkgs.lxcfs}/share/lxc/config/common.conf.d/00-lxcfs.conf
+#         '';
+      };
+      libvirtd = {
+        enable = true;
+        qemu = {
+          ovmf.enable = true;
+          ovmf.packages = [
+            pkgs.OVMFFull.fd
+            pkgs.pkgsCross.aarch64-multiplatform.OVMF.fd
+          ];
+          runAsRoot = false;
+        };
+        onBoot = "ignore";
+        onShutdown = "shutdown";
+      };
+
+      spiceUSBRedirection.enable = true;
     };
 
-    virtualisation.libvirtd = {
+    security.unprivilegedUsernsClone = true;
-      enable = true;
-      qemu = {
-        ovmf.enable = true;
-        ovmf.packages = [
-          pkgs.OVMFFull.fd
-          pkgs.pkgsCross.aarch64-multiplatform.OVMF.fd
-        ];
-        runAsRoot = false;
-      };
-      onBoot = "ignore";
-      onShutdown = "shutdown";
-    };
 
     home-manager.users.${config.mainuser} = {
+      home.file.".config/containers/storage.conf".text = ''
+        [storage]
+        driver = "overlay2"
+      '';
       home.file.".config/libvirt/libvirt.conf".text = ''
         uri_default = "qemu:///system"
       '';
-      home.packages = with pkgs; [
-        docker-compose
-        virt-manager
-      ];
     };
 
-    virtualisation.lxd = lib.mkIf (!isContainer) {
+#     users.users.${config.mainuser} = {
-      enable = true;
+#       subUidRanges = [{
-      zfsSupport = devInfo.fileSystem == "zfs";
+#         count = 1000;
-      recommendedSysctlSettings = true;
+#         startUid = 10000;
-    };
+#       }];
-    virtualisation.lxc = lib.mkIf (!isContainer) {
+#       subGidRanges = [{
-      enable = true;
+#         count = 1000;
-      lxcfs.enable = true;
+#         startGid = 10000;
-      systemConfig = ''
+#       }];
-        lxc.lxcpath = /var/lib/lxd/containers
+#     };
-        ${if devInfo.fileSystem == "zfs" then ''
-          lxc.bdev.zfs.root = rpool/nixos/lxd
-        '' else ""}
-      '';
-      defaultConfig = ''
-        lxc.idmap = u 0 100000 65535
-        lxc.idmap = g 0 100000 65535
-        lxc.include = ${pkgs.lxcfs}/share/lxc/config/common.conf.d/00-lxcfs.conf
-      '';
-    };
 
-    virtualisation.spiceUSBRedirection.enable = true;
+    programs.extra-container.enable = true;
 
-    networking.nat = {
+    persist.state.directories = lib.mkIf (devInfo.fileSystem != "zfs") [
-      enable = true;
-      internalInterfaces = [ "ve-+" ];
-      # externalInterface = "enp8s0";
-    };
-
-    persist.state.directories = lib.mkIf devInfo.fileSystem != "zfs" [
       "/var/lib/docker"
       "/var/lib/libvirt"
+      "/var/lib/containers"
+      "/var/lib/lxd"
     ];
   };
 }