setup hypervisor vm

flake.nix

@@ -182,9 +182,9 @@
           specialArgs = { inherit inputs; };
           format = "vm";
         };
-        Testing-VM = nixos-generators.nixosGenerate {
-          system = builtins.readFile (./machines/Testing-VM/system);
-          modules = [ (import (./machines/Testing-VM)) { device = "Testing-VM"; } ];
+        Hypervisor-VM = nixos-generators.nixosGenerate {
+          system = builtins.readFile (./machines/Hypervisor-VM/system);
+          modules = [ (import (./machines/Hypervisor-VM)) { device = "Hypervisor-VM"; } ];
           specialArgs = { inherit inputs; };
           format = "vm";
         };

install/install-zfs-swap.sh

@@ -0,0 +1,175 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p gptfdisk parted git
+
+set -e
+
+CONFIG_FOLDER="$(dirname "$(pwd)")"
+DEVICE_NAME=Hypervisor-VM
+IS_VM=true
+MAX_JOBS=4
+USE_SWAP=true
+SWAP_SIZE=1G
+ZFS_ARC_MAX=4294967296
+
+clean_stdin() {
+	while read -r -t 0; do read -r; done
+}
+
+pprint () {
+	local cyan="\e[96m"
+	local default="\e[39m"
+	local timestamp
+	timestamp=$(date +%FT%T.%3NZ)
+	echo -e "${cyan}${timestamp} $1${default}" 1>&2
+}
+
+# Create new partitions
+create_new_part_table() {
+    if [[ -z "$IS_VM" ]]; then
+        select ENTRY in $(ls /dev/disk/by-id/);
+        do
+            DISK="/dev/disk/by-id/$ENTRY"
+            echo "Installing system on $ENTRY"
+            break
+        done
+    else
+        select ENTRY in $(ls /dev/disk/by-path/);
+        do
+            DISK="/dev/disk/by-path/$ENTRY"
+            echo "Installing system on $ENTRY"
+            break
+        done
+    fi
+
+    read -s -p "> Do you want to wipe all data on $ENTRY ?" -n 1 -r
+    echo
+    if [[ "$REPLY" =~ ^[Yy]$ ]]
+    then
+        sgdisk --zap-all "$DISK"
+    fi
+
+    pprint "Creating boot (EFI) partition"
+    sgdisk -n1:1M:+512MiB -t1:EF00 "$DISK"
+    BOOT="$DISK-part1"
+
+    pprint "Creating ROOT partition"
+    sgdisk -n2:0:0 -t2:BF00 "$DISK"
+    ZFS="$DISK-part2"
+
+    partprobe "$DISK"
+    sleep 1
+
+    pprint "Format BOOT partition $BOOT"
+    mkfs.vfat -n EFI "$BOOT"
+}
+
+### INSTALLATION BEGIN ###
+create_new_part_table
+
+pprint "Create ZFS pool on $ZFS"
+zpool create \
+    -f \
+    -o ashift=12 \
+    -o autotrim=on \
+    -R /mnt \
+    -O acltype=posixacl \
+    -O atime=on \
+    -O canmount=off \
+    -O compression=zstd \
+    -O dnodesize=auto \
+    -O normalization=formD \
+    -O relatime=on \
+    -O xattr=sa \
+    -O dedup=off \
+    -O mountpoint=/ \
+    rpool "$ZFS"
+
+pprint "Create ZFS datasets"
+
+zfs create -o refreservation=10G -o mountpoint=none rpool/reserved
+zfs create -o canmount=off -o mountpoint=none -o encryption=aes-256-gcm -o keyformat=passphrase -o keylocation=prompt rpool/enc
+zfs create -o canmount=off -o mountpoint=none rpool/enc/nixos
+zfs create -o canmount=off -o mountpoint=none rpool/enc/user
+zfs create -o canmount=on -o mountpoint=/ rpool/enc/nixos/root
+zfs create -o canmount=noauto -o mountpoint=/ rpool/enc/nixos/empty
+zfs create -o canmount=on -o mountpoint=/nix rpool/enc/nixos/nix
+zfs create -o canmount=on -o mountpoint=/home rpool/enc/user/home
+zfs create -o canmount=off -o mountpoint=/var rpool/enc/nixos/var
+zfs create -o canmount=on rpool/enc/nixos/var/lib
+zfs create -o canmount=on rpool/enc/nixos/var/log
+zfs create -o canmount=noauto -o atime=off rpool/enc/nixos/lxd
+zfs create -o canmount=on -o mountpoint=/var/lib/docker -o atime=off rpool/enc/nixos/docker
+zfs create -o canmount=on -o mountpoint=/media/bittorrent -o atime=off -o recordsize=256K rpool/enc/nixos/bittorrent
+zfs create -o canmount=on -o mountpoint=/media/libvirt -o atime=off -o recordsize=64K rpool/enc/nixos/libvirt
+# swap
+if [[ "$USE_SWAP" = true ]]; then
+    zfs create -V $SWAP_SIZE -b $(getconf PAGESIZE) -o logbias=throughput -o sync=always \
+        -o primarycache=metadata -o secondarycache=none -o com.sun:auto-snapshot=false -o compression=zle rpool/enc/swap
+    while [ ! -e /dev/zvol/rpool/enc/swap ]; do sleep 0.2; done
+    mkswap -L swap -f /dev/zvol/rpool/enc/swap
+    SWAP=/dev/zvol/rpool/enc/swap
+fi
+
+# Create blank zfs snapshot
+zfs snapshot rpool/enc/nixos@blank
+zfs snapshot rpool/enc/user@blank
+zfs snapshot rpool/enc/nixos/empty@start
+
+# Disable cache, stale cache will prevent system from booting
+mkdir -p /mnt/etc/zfs/
+rm -f /mnt/etc/zfs/zpool.cache
+touch /mnt/etc/zfs/zpool.cache
+chmod a-w /mnt/etc/zfs/zpool.cache
+chattr +i /mnt/etc/zfs/zpool.cache
+
+mkdir /mnt/boot
+mount "$BOOT" /mnt/boot
+
+pprint "Generate NixOS configuration"
+nixos-generate-config --root /mnt --dir $CONFIG_FOLDER/machines/$DEVICE_NAME
+rm -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix
+
+HOSTID=$(head -c8 /etc/machine-id)
+
+HARDWARE_CONFIG=$(mktemp)
+cat <<CONFIG > "$HARDWARE_CONFIG"
+  networking.hostId = "$HOSTID";
+  boot.zfs.devNodes = "$ZFS";
+  boot.supportedFilesystems = [ "zfs" ];
+  boot.kernelParams = [ "zfs.zfs_arc_max=$ZFS_ARC_MAX" "nohibernate" ];
+CONFIG
+
+pprint "Append ZFS configuration to hardware-configuration.nix"
+sed -i "\$e cat $HARDWARE_CONFIG" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+sed -i 's|fsType = "zfs";|fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];|g' $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+
+if [[ -n "$SWAP" ]]; then
+    sed -i "s#swapDevices = \[ \];#swapDevices = \[\n    {\n      device = \"$SWAP\";\n    }\n  \];#" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+fi
+
+# pprint "Copy hardware config to machines folder"
+# cp /mnt/etc/nixos/hardware-configuration.nix $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+chown 1000:100 $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+# Change <not-detected> for flakes
+# sed -i "s#(modulesPath + \"/installer/scan/not-detected.nix\")#\"${toString modulesPath}/installer/scan/not-detected.nix\"#" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix
+git add -A
+
+clean_stdin
+read -s -p "> Do you want to execute nixos-install command?" -n 1 -r
+echo
+if [[ "$REPLY" =~ ^[Yy]$ ]]
+then
+    nixos-install --flake "../#$DEVICE_NAME" --max-jobs $MAX_JOBS --no-root-passwd
+fi
+
+pprint "Copy config to destination system"
+mkdir -p /mnt/home/alukard/nixos-config
+cp -aT $CONFIG_FOLDER /mnt/home/alukard/nixos-config
+
+pprint "Gen ssh host key for initrd"
+ssh-keygen -t ed25519 -N "" -f /mnt/root/ssh_host_key
+chown root:root /mnt/root/ssh_host_key
+cmod 644 /mnt/root/ssh_host_key
+
+umount -Rl /mnt
+zpool export -a

machines/Hypervisor-VM/default.nix

@@ -0,0 +1,159 @@
+{ modulesPath, inputs, lib, pkgs, config, options, ... }: {
+  imports = with inputs.self; [
+    # "${toString modulesPath}/profiles/qemu-guest.nix"
+    "${toString modulesPath}/profiles/hardened.nix"
+    # ./imports/qemu-vm.nix
+
+    ./hardware-configuration.nix
+    nixosRoles.hypervisor
+    nixosProfiles.direnv
+  ];
+
+  # build hell
+  environment.noXlibs = lib.mkForce false;
+  # minimal profile
+  documentation.nixos.enable = lib.mkForce false;
+  programs.command-not-found.enable = lib.mkForce false;
+  xdg.autostart.enable = lib.mkForce false;
+  xdg.icons.enable = lib.mkForce false;
+  xdg.mime.enable = lib.mkForce false;
+  xdg.sounds.enable = lib.mkForce false;
+  services.udisks2.enable = lib.mkForce false;
+
+  # boot
+  boot = {
+    zfs.forceImportAll = lib.mkForce false;
+    # loader.grub.enable = true;
+    loader.systemd-boot = {
+      enable = true;
+      editor = false;
+      configurationLimit = 8;
+    };
+    # loader.efi.canTouchEfiVariables = true;
+    kernelPackages = pkgs.linuxPackages_hardened;
+    kernelModules = [ "tcp_bbr" ];
+    kernelParams = [
+      "zswap.enabled=0"
+      "quiet"
+      "scsi_mod.use_blk_mq=1"
+      "modeset"
+      "nofb"
+      "pti=off"
+      "spectre_v2=off"
+      "kvm.ignore_msrs=1"
+      "rd.systemd.show_status=auto"
+      "rd.udev.log_priority=3"
+    ];
+    kernel.sysctl = {
+      "kernel.sysrq" = false;
+      "net.core.default_qdisc" = "sch_fq_codel";
+      "net.ipv4.conf.all.accept_source_route" = false;
+      "net.ipv4.icmp_ignore_bogus_error_responses" = true;
+      "net.ipv4.tcp_congestion_control" = "bbr";
+      "net.ipv4.tcp_fastopen" = 3;
+      "net.ipv4.tcp_rfc1337" = true;
+      "net.ipv4.tcp_syncookies" = true;
+      "net.ipv6.conf.all.accept_source_route" = false;
+      # disable ipv6
+      "net.ipv6.conf.all.disable_ipv6" = true;
+      "net.ipv6.conf.default.disable_ipv6" = true;
+    };
+    kernel.sysctl = {
+      "vm.swappiness" = 1;
+    };
+    cleanTmpDir = true;
+    initrd = {
+      availableKernelModules = [ "tg3" ];
+      postDeviceCommands = lib.mkAfter ''
+        zfs rollback -r rpool/enc/nixos/empty@start
+      '';
+      # network = {
+      #   enable = true;
+      #   ssh = {
+      #     enable = true;
+      #     port = 2222;
+      #     # hostKeys = [ /root/ssh_host_key ];
+      #     hostKeys = [ /home/alukard/ssh_host_key ];
+      #     authorizedKeys = config.users.users.alukard.openssh.authorizedKeys.keys;
+      #   };
+      #   postCommands = ''
+      #     echo "zfs load-key -a; killall zfs" >> /root/.profile
+      #   '';
+      # };
+    };
+  };
+
+  # security.polkit.enable = true;
+  # system.nssModules = lib.mkForce [ ];
+
+  # services.nscd.enable = false;
+
+  deviceSpecific.devInfo = {
+    cpu = {
+      vendor = "intel";
+      clock = 2300;
+      cores = 4;
+    };
+    drive = {
+      type = "sdd";
+      speed = 500;
+      size = 500;
+    };
+    gpu = {
+      vendor = "other";
+    };
+    bigScreen = false;
+    ram = 12;
+  };
+  deviceSpecific.enableVirtualisation = true;
+  deviceSpecific.wireguard.enable = false;
+  deviceSpecific.isServer = true;
+
+  services.zfs.autoScrub.enable = true;
+  services.zfs.autoScrub.interval = "daily";
+
+  # hardened
+  networking.firewall.enable = true;
+  networking.firewall.allowedTCPPorts = [];
+  networking.firewall.allowedUDPPorts = [];
+  systemd.coredump.enable = false;
+  programs.firejail.enable = true;
+  # scudo memalloc is unstable
+  # environment.memoryAllocator.provider = "libc";
+  # environment.memoryAllocator.provider = "graphene-hardened";
+
+  networking.wireless.enable = false;
+  networking.networkmanager.enable = false;
+  networking.hostName = config.device;
+
+  services.timesyncd.enable = false;
+  services.openntpd.enable = true;
+  networking.timeServers = [
+    "0.ru.pool.ntp.org"
+    "1.ru.pool.ntp.org"
+    "2.ru.pool.ntp.org"
+    "3.ru.pool.ntp.org"
+    "0.europe.pool.ntp.org"
+    "1.europe.pool.ntp.org"
+    "2.europe.pool.ntp.org"
+    "3.europe.pool.ntp.org"
+  ] ++ options.networking.timeServers.default;
+
+  # virtualisation
+  virtualisation.oci-containers.backend = lib.mkForce "podman";
+  virtualisation.docker.enable = lib.mkForce false;
+  virtualisation.podman = {
+    enable = true;
+    dockerCompat = true;
+    dockerSocket.enable = true;
+  };
+
+  fonts.enableDefaultFonts = lib.mkForce false;
+  fonts.fonts = [ (pkgs.nerdfonts.override { fonts = [ "FiraCode" "VictorMono" ]; }) ];
+
+  home-manager.users.alukard.home.packages = with pkgs; [ bat podman-compose ];
+
+  home-manager.users.alukard.xdg.mime.enable = false;
+  home-manager.users.alukard.home.stateVersion = "22.11";
+  system.stateVersion = "22.11";
+}

machines/Hypervisor-VM/hardware-configuration.nix

@@ -0,0 +1,81 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "virtio_pci" "xhci_pci" "sr_mod" "virtio_blk" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { # device = "rpool/enc/nixos/root";
+      device = "rpool/enc/nixos/empty";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/nix" =
+    { device = "rpool/enc/nixos/nix";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/home" =
+    { device = "rpool/enc/user/home";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/lib" =
+    { device = "rpool/enc/nixos/var/lib";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/log" =
+    { device = "rpool/enc/nixos/var/log";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/var/lib/docker" =
+    { device = "rpool/enc/nixos/docker";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/media/bittorrent" =
+    { device = "rpool/enc/nixos/bittorrent";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/media/libvirt" =
+    { device = "rpool/enc/nixos/libvirt";
+      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/6B88-F626";
+      fsType = "vfat";
+    };
+
+  # swapDevices = [
+  #   {
+  #     device = "/dev/zvol/rpool/enc/swap";
+  #   }
+  # ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp2s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+  networking.hostId = "41d97526";
+  boot.zfs.devNodes = "/dev/disk/by-path/virtio-pci-0000:05:00.0-part2";
+  boot.supportedFilesystems = [ "zfs" ];
+  boot.kernelParams = [ "zfs.zfs_arc_max=4294967296" "nohibernate" ];
+}

machines/Hypervisor-VM/imports/qemu-vm.nix

@@ -0,0 +1,23 @@
+{ modulesPath, config, lib, pkgs, ... }: {
+  imports = [
+    "${toString modulesPath}/profiles/qemu-guest.nix"
+    "${toString modulesPath}/virtualisation/qemu-vm.nix"
+  ];
+  virtualisation = {
+    qemu.options = [ "-vga none" "-device virtio-vga-gl" "-display gtk,gl=on" ];
+    cores = 1;
+    memorySize = 4096;
+    msize = 65536;
+    diskSize = 10240;
+    diskImage = "/media/libvirt/vm-images/${config.device}.qcow2";
+    # resolution = { x = 1920; y = 1080; };
+    # useNixStoreImage = true;
+    # writableStore = false;
+    # writableStore = true;
+    # useNixStoreImage = true;
+    # writableStoreUseTmpfs = true;
+  };
+  # services.spice-vdagentd.enable = lib.mkOverride 0 true;
+  # services.xserver.videoDrivers = [ "qxl" ];
+  # services.qemuGuest.enable = true;
+}

machines/Testing-VM/default.nix

@@ -1,70 +0,0 @@
-{ inputs, modulesPath, config, pkgs, lib, ... }: {
-  imports = with inputs.self.nixosModules; with inputs.self.nixosProfiles; [
-    "${modulesPath}/profiles/qemu-guest.nix"
-    ./qemu-vm.nix
-    # ./hardware-configuration.nix
-    inputs.self.nixosRoles.base
-    inputs.base16.hmModule
-
-    direnv
-    firefox
-    fonts
-    themes
-    vscode
-  ];
-
-  deviceSpecific.devInfo = {
-    cpu = {
-      vendor = "amd";
-      clock = 3700;
-      cores = 2;
-    };
-    drive = {
-      type = "ssd";
-      speed = 2000;
-      size = 30;
-    };
-    gpu = {
-      vendor = "other";
-    };
-    bigScreen = false;
-    ram = 4;
-  };
-  deviceSpecific.isHost = true;
-  deviceSpecific.isShared = false;
-  deviceSpecific.isGaming = false;
-  deviceSpecific.enableVirtualisation = false;
-  deviceSpecific.wireguard.enable = false;
-
-  hardware.video.hidpi.enable = lib.mkForce false;
-
-  boot.kernelPackages = lib.mkForce config.boot.zfs.package.latestCompatibleLinuxPackages;
-  services.xserver = {
-    # enable = false;
-    enable = true;
-    displayManager.sddm.enable = true;
-    desktopManager.plasma5.enable = true;
-  };
-  # services.greetd = {
-  #   enable = true;
-  #   package = pkgs.greetd.gtkgreet;
-  #   settings = {
-  #     default_session = {
-  #       command = "${pkgs.cage}/bin/cage -s -- gtkgreet";
-  #     };
-  #   };
-  # };
-  # services.greetd = {
-  #   enable = true;
-  #   package = pkgs.greetd.tuigreet;
-  #   settings = {
-  #     default_session = {
-  #       # command = "${pkgs.greetd.tuigreet}/bin/tuigreet --time --remember --remember-session \"${pkgs.plasma5Packages.plasma-workspace}/share/wayland-sessions\"";
-  #       command = "${pkgs.greetd.tuigreet}/bin/tuigreet";
-  #       # user = "alukard";
-  #     };
-  #   };
-  # };
-  networking.usePredictableInterfaceNames = lib.mkForce false;
-  # environment.systemPackages = with pkgs; [ firefox ];
-}

machines/Testing-VM/qemu-vm.nix

@@ -1,17 +0,0 @@
-{ modulesPath, config, ... }: {
-  imports = [
-    "${modulesPath}/virtualisation/qemu-vm.nix"
-  ];
-  virtualisation = {
-    qemu.options = [ "-vga none" "-device virtio-vga-gl" "-display gtk,gl=on" ];
-    cores = 4;
-    memorySize = 4096;
-    #msize = 262144;
-    diskSize = 10240;
-    diskImage = "./vm-images/${config.device}.qcow2";
-    # resolution = { x = 1920; y = 1080; };
-
-    useNixStoreImage = true;
-    writableStore = false;
-  };
-}

roles/default.nix

@@ -1,5 +1,5 @@
 {
-  server = ./server.nix;
+  hypervisor = ./hypervisor.nix;
   desktop = ./desktop.nix;
   base = ./base.nix;
   workstation = ./workstation.nix;

roles/hypervisor.nix

@@ -0,0 +1,25 @@
+{ inputs, pkgs, ... }: {
+  imports = with inputs.self.nixosModules; with inputs.self.nixosProfiles; [
+    inputs.home-manager.nixosModules.home-manager {
+      home-manager.useGlobalPkgs = true;
+      home-manager.useUserPackages = true;
+    }
+
+    applications
+    devices
+    git
+    gpg
+    locale
+    misc
+    nix
+    nix-index
+    overlay
+    secrets
+    secrets-envsubst
+    security
+    ssh
+    zsh
+  ];
+
+  environment.systemPackages = [ pkgs.kitty ];
+}

roles/server.nix

@@ -1,23 +0,0 @@
-{ inputs, ... }: {
-  imports = with inputs.self.nixosModules; with inputs.self.nixosProfiles; [
-    ./base.nix
-    inputs.base16.hmModule
-
-    fonts
-    themes
-
-    direnv
-    kitty
-    nix-index
-
-    coturn
-    cloudflare-ddns
-    # gitea
-    #mailserver
-    matrix-synapse
-    # nginx
-    stubby
-    caddy
-    vscode-server
-  ];
-}