From 268d8c6e1f92735dced6026e71f0dcdf34e9b344 Mon Sep 17 00:00:00 2001 From: Dmitriy Kholkin Date: Thu, 8 Dec 2022 23:11:08 +0300 Subject: [PATCH] hypervisor-vm with zfs and encrypted boot --- install/install-zfs-swap-luks.sh | 278 ++++++++++++++++++ install/install-zfs-swap.sh | 265 ++++++++++++----- machines/Hypervisor-VM/default.nix | 72 +++-- .../Hypervisor-VM/hardware-configuration.nix | 58 ++-- machines/Hypervisor-VM/imports/qemu-vm.nix | 23 -- 5 files changed, 541 insertions(+), 155 deletions(-) create mode 100755 install/install-zfs-swap-luks.sh delete mode 100644 machines/Hypervisor-VM/imports/qemu-vm.nix diff --git a/install/install-zfs-swap-luks.sh b/install/install-zfs-swap-luks.sh new file mode 100755 index 0000000..d8e6529 --- /dev/null +++ b/install/install-zfs-swap-luks.sh @@ -0,0 +1,278 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p gptfdisk parted git + +set -e + +CONFIG_FOLDER="$(dirname "$(pwd)")" +LUKS_DEVICE_NAME=cryptroot +BOOT_DEVICE_NAME=cryptboot +DEVICE_NAME=Hypervisor-VM +# IS_VM=true +MAX_JOBS=2 +USE_SWAP=true +BOOT_POOL_SIZE=4GiB +SWAP_SIZE=1GiB +BOOT_RESERVATION=128M +ROOT_RESERVATION=1G +USE_ECNRYPTION=true + + +if [[ "$IS_VM" = true ]]; then + DISK_DEV_NODES="/dev/disk/by-path" +else + DISK_DEV_NODES="/dev/disk/by-id" +fi + +clean_stdin() { + while read -r -t 0; do read -r; done +} + +pprint () { + local cyan="\e[96m" + local default="\e[39m" + local timestamp + timestamp=$(date +%FT%T.%3NZ) + echo -e "${cyan}${timestamp} $1${default}" 1>&2 +} + +# Create new partitions +create_new_part_table() { + select ENTRY in $(ls $DISK_DEV_NODES); + do + DISK="$DISK_DEV_NODES/$ENTRY" + echo "Installing system on $ENTRY" + break + done + + read -s -p "> Do you want to wipe all data on $ENTRY ?" -n 1 -r + echo + if [[ "$REPLY" =~ ^[Yy]$ ]] + then + sgdisk --zap-all "$DISK" + fi + + pprint "Creating boot (EFI) partition" + sgdisk -n1:1MiB:+512MiB -t1:EF00 "$DISK" + EFI="$DISK-part1" + + pprint "Creating boot (ZFS) partition" + if [[ "$USE_ECNRYPTION" = true ]] + then + sgdisk -n2:0:+$BOOT_POOL_SIZE -t2:8309 "$DISK" + else + sgdisk -n2:0:+$BOOT_POOL_SIZE -t2:BF00 "$DISK" + fi + BOOT="$DISK-part2" + + if [[ "$USE_SWAP" = true ]] + then + pprint "Creating SWAP partition" + sgdisk -n4:0:+$SWAP_SIZE -t4:8200 "$DISK" + fi + + if [[ "$USE_ECNRYPTION" = true ]] + then + pprint "Creating LUKS partition" + sgdisk -n3:0:0 -t3:8309 "$DISK" + else + pprint "Creating ROOT partition" + sgdisk -n3:0:0 -t3:BF00 "$DISK" + fi + ROOT="$DISK-part3" + + partprobe "$DISK" + sleep 1 + + pprint "Format EFI partition $EFI" + mkfs.vfat -n EFI "$EFI" +} + +### INSTALLATION BEGIN ### +create_new_part_table + +if [[ "$USE_ECNRYPTION" = true ]] +then + dd if=/dev/urandom of=./keyfile0.bin bs=1024 count=4 + + pprint "Creating LUKS container on $BOOT" + clean_stdin + cryptsetup --type luks1 -c aes-xts-plain64 -s 512 -h sha512 --iter-time 5000 --use-random luksFormat "$BOOT" + clean_stdin + pprint "Add keyfile to LUKS container on $BOOT" + cryptsetup luksAddKey $BOOT keyfile0.bin + + pprint "Open LUKS container on $BOOT" + cryptsetup luksOpen --allow-discards "$BOOT" "$BOOT_DEVICE_NAME" -d keyfile0.bin + + pprint "Creating LUKS container on $ROOT" + clean_stdin + cryptsetup --type luks1 -c aes-xts-plain64 -s 512 -h sha512 --iter-time 5000 --use-random luksFormat "$ROOT" + clean_stdin + pprint "Add keyfile to LUKS container on $ROOT" + cryptsetup luksAddKey $ROOT keyfile0.bin + + pprint "Open LUKS container on $ROOT" + cryptsetup luksOpen --allow-discards "$ROOT" "$LUKS_DEVICE_NAME" -d keyfile0.bin + + BOOT_POOL="$(ls /dev/disk/by-id/dm-uuid-*$BOOT_DEVICE_NAME)" + # BOOT_POOL="$BOOT" + ROOT_POOL="$(ls /dev/disk/by-id/dm-uuid-*$LUKS_DEVICE_NAME)" +else + BOOT_POOL="$BOOT" + ROOT_POOL="$ROOT" +fi + +pprint "Create ZFS root pool on $ROOT_POOL" +zpool create \ + -f \ + -o ashift=12 \ + -o autotrim=on \ + -O acltype=posixacl \ + -O atime=on \ + -O canmount=off \ + -O compression=zstd \ + -O dnodesize=auto \ + -O normalization=formD \ + -O relatime=on \ + -O xattr=sa \ + -O dedup=off \ + -O mountpoint=/ \ + -R /mnt \ + rpool "$ROOT_POOL" + +pprint "Create ZFS root datasets" + +zfs create -o refreservation=$ROOT_RESERVATION -o canmount=off -o mountpoint=none rpool/reserved +# top level datasets +zfs create -o canmount=off -o mountpoint=none rpool/nixos +zfs create -o canmount=off -o mountpoint=none rpool/user +zfs create -o canmount=off -o mountpoint=none rpool/persistent +# empty root +zfs create -o canmount=noauto -o mountpoint=/ rpool/nixos/root +zfs mount rpool/nixos/root +zfs create -o canmount=on -o mountpoint=/home rpool/user/home +# persistent across boots +zfs create -o canmount=on -o mountpoint=/persistent rpool/persistent/impermanence +zfs create -o canmount=on -o mountpoint=/etc/secrets rpool/persistent/secrets +zfs create -o canmount=on -o mountpoint=/nix rpool/persistent/nix +# zfs create -o canmount=on -o mountpoint=/boot rpool/persistent/boot +zfs create -o canmount=on -o mountpoint=/var/log rpool/persistent/log +zfs create -o canmount=noauto -o atime=off rpool/persistent/lxd +zfs create -o canmount=on -o mountpoint=/var/lib/docker -o atime=off rpool/persistent/docker +zfs create -o canmount=on -o mountpoint=/media/bittorrent -o atime=off -o recordsize=256K rpool/persistent/bittorrent +zfs create -o canmount=on -o mountpoint=/media/libvirt -o atime=off -o recordsize=64K rpool/persistent/libvirt + +# Create empty zfs snapshots +zfs snapshot rpool/nixos@empty +zfs snapshot rpool/nixos/root@empty +zfs snapshot rpool/user@empty +zfs snapshot rpool/user/home@empty + +pprint "Create ZFS boot pool on $BOOT_POOL" +zpool create \ + -f \ + -o compatibility=grub2 \ + -o ashift=12 \ + -o autotrim=on \ + -O acltype=posixacl \ + -O atime=on \ + -O canmount=off \ + -O compression=lz4 \ + -O devices=off \ + -O normalization=formD \ + -O relatime=on \ + -O xattr=sa \ + -O dedup=off \ + -O mountpoint=/boot \ + -R /mnt \ + bpool "$BOOT_POOL" + +pprint "Create ZFS boot datasets" + +zfs create -o refreservation=$BOOT_RESERVATION -o canmount=off -o mountpoint=none bpool/reserved +zfs create -o canmount=off -o mountpoint=none bpool/nixos +zfs create -o canmount=on -o mountpoint=/boot bpool/nixos/boot + +zfs snapshot bpool/nixos@empty +zfs snapshot bpool/nixos/boot@empty + +# Disable cache, stale cache will prevent system from booting +mkdir -p /mnt/etc/zfs/ +rm -f /mnt/etc/zfs/zpool.cache +touch /mnt/etc/zfs/zpool.cache +chmod a-w /mnt/etc/zfs/zpool.cache +chattr +i /mnt/etc/zfs/zpool.cache + +mkdir -p /mnt/boot/efi +mount -t vfat "$EFI" /mnt/boot/efi + +if [[ "$USE_SWAP" = true ]]; then + SWAP="$DISK-part4" + mkswap -L swap -f "$SWAP" +fi + +pprint "Generate NixOS configuration" +[[ -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix ]] && CONFIG_EXISTS=true +nixos-generate-config --root /mnt --dir $CONFIG_FOLDER/machines/$DEVICE_NAME +[[ -z "$CONFIG_EXISTS" ]] && rm -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix + +HOSTID=$(head -c8 /etc/machine-id) + +BOOT_PARTUUID=$(blkid --match-tag PARTUUID --output value "$BOOT") +ROOT_PARTUUID=$(blkid --match-tag PARTUUID --output value "$ROOT") +[[ ! -z "$SWAP" ]] && SWAP_PARTUUID=$(blkid --match-tag PARTUUID --output value "$SWAP") + +HARDWARE_CONFIG=$(mktemp) +if [[ "$USE_ECNRYPTION" = true ]] +then +cat < "$HARDWARE_CONFIG" + networking.hostId = "$HOSTID"; + boot.zfs.devNodes = "$DISK_DEV_NODES"; + boot.supportedFilesystems = [ "zfs" ]; + boot.initrd.luks.devices."$BOOT_DEVICE_NAME".device = "/dev/disk/by-partuuid/$BOOT_PARTUUID"; + boot.initrd.luks.devices."$LUKS_DEVICE_NAME".device = "/dev/disk/by-partuuid/$ROOT_PARTUUID"; +CONFIG +else +cat < "$HARDWARE_CONFIG" + networking.hostId = "$HOSTID"; + boot.zfs.devNodes = "$DISK_DEV_NODES"; + boot.supportedFilesystems = [ "zfs" ]; +CONFIG +fi + +pprint "Append ZFS configuration to hardware-configuration.nix" +sed -i "\$e cat $HARDWARE_CONFIG" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix +sed -i 's|fsType = "zfs";|fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];|g' $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix +if [[ ! -z "$SWAP" ]]; then + sed -i "s|swapDevices = \[ \];|swapDevices = \[\n {\n device = \"/dev/disk/by-partuuid/$SWAP_PARTUUID\";\n randomEncryption.enable = true;\n randomEncryption.allowDiscards = true;\n }\n \];|" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix +fi +chown 1000:100 $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix +git add -A + +pprint "Copy config to destination system" +mkdir -p /mnt/home/alukard/nixos-config +cp -aT $CONFIG_FOLDER /mnt/home/alukard/nixos-config + +pprint "Gen ssh host key for initrd" +ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/ssh_host_key +chown root:root /mnt/etc/secrets/ssh_host_key +chmod 600 /mnt/etc/secrets/ssh_host_key + +if [[ "$USE_ECNRYPTION" = true ]] +then + cp keyfile0.bin /mnt/etc/secrets/keyfile0.bin + chmod 000 /mnt/etc/secrets/keyfile*.bin +fi + +clean_stdin +read -s -p "> Do you want to execute nixos-install command?" -n 1 -r +echo +if [[ "$REPLY" =~ ^[Yy]$ ]] +then + nixos-install --flake "../#$DEVICE_NAME" --root /mnt --max-jobs $MAX_JOBS --no-root-passwd +fi + +umount -Rl /mnt && \ +zpool export -a && \ +cryptsetup luksClose $BOOT_DEVICE_NAME && \ +cryptsetup luksClose $LUKS_DEVICE_NAME diff --git a/install/install-zfs-swap.sh b/install/install-zfs-swap.sh index e373b7f..eef2024 100755 --- a/install/install-zfs-swap.sh +++ b/install/install-zfs-swap.sh @@ -4,12 +4,24 @@ set -e CONFIG_FOLDER="$(dirname "$(pwd)")" +LUKS_DEVICE_NAME=cryptroot +BOOT_DEVICE_NAME=cryptboot DEVICE_NAME=Hypervisor-VM IS_VM=true MAX_JOBS=4 USE_SWAP=true -SWAP_SIZE=1G -ZFS_ARC_MAX=4294967296 +BOOT_POOL_SIZE=4GiB +SWAP_SIZE=1GiB +BOOT_RESERVATION=128M +ROOT_RESERVATION=1G +# USE_ECNRYPTION=true + + +if [[ "$IS_VM" = true ]]; then + DISK_DEV_NODES="/dev/disk/by-path" +else + DISK_DEV_NODES="/dev/disk/by-id" +fi clean_stdin() { while read -r -t 0; do read -r; done @@ -25,21 +37,12 @@ pprint () { # Create new partitions create_new_part_table() { - if [[ -z "$IS_VM" ]]; then - select ENTRY in $(ls /dev/disk/by-id/); - do - DISK="/dev/disk/by-id/$ENTRY" - echo "Installing system on $ENTRY" - break - done - else - select ENTRY in $(ls /dev/disk/by-path/); - do - DISK="/dev/disk/by-path/$ENTRY" - echo "Installing system on $ENTRY" - break - done - fi + select ENTRY in $(ls $DISK_DEV_NODES); + do + DISK="$DISK_DEV_NODES/$ENTRY" + echo "Installing system on $ENTRY" + break + done read -s -p "> Do you want to wipe all data on $ENTRY ?" -n 1 -r echo @@ -49,29 +52,80 @@ create_new_part_table() { fi pprint "Creating boot (EFI) partition" - sgdisk -n1:1M:+512MiB -t1:EF00 "$DISK" - BOOT="$DISK-part1" + sgdisk -n1:1MiB:+512MiB -t1:EF00 "$DISK" + EFI="$DISK-part1" - pprint "Creating ROOT partition" - sgdisk -n2:0:0 -t2:BF00 "$DISK" - ZFS="$DISK-part2" + pprint "Creating boot (ZFS) partition" + if [[ "$USE_ECNRYPTION" = true ]] + then + sgdisk -n2:0:+$BOOT_POOL_SIZE -t2:8309 "$DISK" + else + sgdisk -n2:0:+$BOOT_POOL_SIZE -t2:EF00 "$DISK" + fi + BOOT="$DISK-part2" + + if [[ "$USE_SWAP" = true ]] + then + pprint "Creating SWAP partition" + sgdisk -n4:0:+$SWAP_SIZE -t4:8200 "$DISK" + fi + + if [[ "$USE_ECNRYPTION" = true ]] + then + pprint "Creating LUKS partition" + sgdisk -n3:0:0 -t3:8309 "$DISK" + else + pprint "Creating ROOT partition" + sgdisk -n3:0:0 -t3:BF00 "$DISK" + fi + ROOT="$DISK-part3" partprobe "$DISK" sleep 1 - pprint "Format BOOT partition $BOOT" - mkfs.vfat -n EFI "$BOOT" + pprint "Format EFI partition $EFI" + mkfs.vfat -n EFI "$EFI" } ### INSTALLATION BEGIN ### create_new_part_table -pprint "Create ZFS pool on $ZFS" +if [[ "$USE_ECNRYPTION" = true ]] +then + dd if=/dev/urandom of=./keyfile0.bin bs=1024 count=4 + + pprint "Creating LUKS container on $BOOT" + clean_stdin + cryptsetup --type luks1 -c aes-xts-plain64 -s 512 -h sha512 --iter-time 5000 --use-random luksFormat "$BOOT" + clean_stdin + pprint "Add keyfile to LUKS container on $BOOT" + cryptsetup luksAddKey $BOOT keyfile0.bin + + pprint "Open LUKS container on $BOOT" + cryptsetup luksOpen --allow-discards "$BOOT" "$BOOT_DEVICE_NAME" -d keyfile0.bin + + pprint "Creating LUKS container on $ROOT" + clean_stdin + cryptsetup --type luks1 -c aes-xts-plain64 -s 512 -h sha512 --iter-time 5000 --use-random luksFormat "$ROOT" + clean_stdin + pprint "Add keyfile to LUKS container on $ROOT" + cryptsetup luksAddKey $ROOT keyfile0.bin + + pprint "Open LUKS container on $ROOT" + cryptsetup luksOpen --allow-discards "$ROOT" "$LUKS_DEVICE_NAME" -d keyfile0.bin + + BOOT_POOL="$(ls /dev/disk/by-id/dm-uuid-*$BOOT_DEVICE_NAME)" + ROOT_POOL="$(ls /dev/disk/by-id/dm-uuid-*$LUKS_DEVICE_NAME)" +else + BOOT_POOL="$BOOT" + ROOT_POOL="$ROOT" +fi + +pprint "Create ZFS root pool on $ROOT_POOL" zpool create \ -f \ -o ashift=12 \ -o autotrim=on \ - -R /mnt \ -O acltype=posixacl \ -O atime=on \ -O canmount=off \ @@ -82,38 +136,64 @@ zpool create \ -O xattr=sa \ -O dedup=off \ -O mountpoint=/ \ - rpool "$ZFS" + -R /mnt \ + rpool "$ROOT_POOL" -pprint "Create ZFS datasets" +pprint "Create ZFS root datasets" -zfs create -o refreservation=10G -o mountpoint=none rpool/reserved -zfs create -o canmount=off -o mountpoint=none -o encryption=aes-256-gcm -o keyformat=passphrase -o keylocation=prompt rpool/enc -zfs create -o canmount=off -o mountpoint=none rpool/enc/nixos -zfs create -o canmount=off -o mountpoint=none rpool/enc/user -zfs create -o canmount=on -o mountpoint=/ rpool/enc/nixos/root -zfs create -o canmount=noauto -o mountpoint=/ rpool/enc/nixos/empty -zfs create -o canmount=on -o mountpoint=/nix rpool/enc/nixos/nix -zfs create -o canmount=on -o mountpoint=/home rpool/enc/user/home -zfs create -o canmount=off -o mountpoint=/var rpool/enc/nixos/var -zfs create -o canmount=on rpool/enc/nixos/var/lib -zfs create -o canmount=on rpool/enc/nixos/var/log -zfs create -o canmount=noauto -o atime=off rpool/enc/nixos/lxd -zfs create -o canmount=on -o mountpoint=/var/lib/docker -o atime=off rpool/enc/nixos/docker -zfs create -o canmount=on -o mountpoint=/media/bittorrent -o atime=off -o recordsize=256K rpool/enc/nixos/bittorrent -zfs create -o canmount=on -o mountpoint=/media/libvirt -o atime=off -o recordsize=64K rpool/enc/nixos/libvirt -# swap -if [[ "$USE_SWAP" = true ]]; then - zfs create -V $SWAP_SIZE -b $(getconf PAGESIZE) -o logbias=throughput -o sync=always \ - -o primarycache=metadata -o secondarycache=none -o com.sun:auto-snapshot=false -o compression=zle rpool/enc/swap - while [ ! -e /dev/zvol/rpool/enc/swap ]; do sleep 0.2; done - mkswap -L swap -f /dev/zvol/rpool/enc/swap - SWAP=/dev/zvol/rpool/enc/swap -fi +zfs create -o refreservation=$ROOT_RESERVATION -o canmount=off -o mountpoint=none rpool/reserved +# top level datasets +zfs create -o canmount=off -o mountpoint=none rpool/nixos +zfs create -o canmount=off -o mountpoint=none rpool/user +zfs create -o canmount=off -o mountpoint=none rpool/persistent +# empty root +zfs create -o canmount=noauto -o mountpoint=/ rpool/nixos/root +zfs mount rpool/nixos/root +zfs create -o canmount=on -o mountpoint=/home rpool/user/home +# persistent across boots +zfs create -o canmount=on -o mountpoint=/persistent rpool/persistent/impermanence +zfs create -o canmount=on -o mountpoint=/etc/secrets rpool/persistent/secrets +zfs create -o canmount=on -o mountpoint=/nix rpool/persistent/nix +# zfs create -o canmount=on -o mountpoint=/boot rpool/persistent/boot +zfs create -o canmount=on -o mountpoint=/var/log rpool/persistent/log +zfs create -o canmount=noauto -o atime=off rpool/persistent/lxd +zfs create -o canmount=on -o mountpoint=/var/lib/docker -o atime=off rpool/persistent/docker +zfs create -o canmount=on -o mountpoint=/media/bittorrent -o atime=off -o recordsize=256K rpool/persistent/bittorrent +zfs create -o canmount=on -o mountpoint=/media/libvirt -o atime=off -o recordsize=64K rpool/persistent/libvirt -# Create blank zfs snapshot -zfs snapshot rpool/enc/nixos@blank -zfs snapshot rpool/enc/user@blank -zfs snapshot rpool/enc/nixos/empty@start +# Create empty zfs snapshots +zfs snapshot rpool/nixos@empty +zfs snapshot rpool/nixos/root@empty +zfs snapshot rpool/user@empty +zfs snapshot rpool/user/home@empty + +pprint "Create ZFS boot pool on $BOOT_POOL" +zpool create \ + -f \ + -o compatibility=grub2 \ + -o ashift=12 \ + -o autotrim=on \ + -O acltype=posixacl \ + -O atime=on \ + -O canmount=off \ + -O compression=lz4 \ + -O devices=off \ + -O normalization=formD \ + -O relatime=on \ + -O xattr=sa \ + -O dedup=off \ + -O mountpoint=/boot \ + -R /mnt \ + bpool "$BOOT_POOL" + +pprint "Create ZFS boot datasets" + +zfs create -o refreservation=$BOOT_RESERVATION -o canmount=off -o mountpoint=none bpool/reserved +zfs create -o canmount=off -o mountpoint=none bpool/nixos +zfs create -o canmount=on -o mountpoint=/boot bpool/nixos/boot + +zfs snapshot bpool/nixos@empty +zfs snapshot bpool/nixos/boot@empty # Disable cache, stale cache will prevent system from booting mkdir -p /mnt/etc/zfs/ @@ -122,54 +202,77 @@ touch /mnt/etc/zfs/zpool.cache chmod a-w /mnt/etc/zfs/zpool.cache chattr +i /mnt/etc/zfs/zpool.cache -mkdir /mnt/boot -mount "$BOOT" /mnt/boot +mkdir -p /mnt/boot/efi +mount -t vfat "$EFI" /mnt/boot/efi + +if [[ "$USE_SWAP" = true ]]; then + SWAP="$DISK-part4" + mkswap -L swap -f "$SWAP" +fi pprint "Generate NixOS configuration" +# nixos-generate-config --root /mnt +[[ -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix ]] && CONFIG_EXISTS=true nixos-generate-config --root /mnt --dir $CONFIG_FOLDER/machines/$DEVICE_NAME -rm -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix +[[ -z "$CONFIG_EXISTS" ]] && rm -f $CONFIG_FOLDER/machines/$DEVICE_NAME/configuration.nix HOSTID=$(head -c8 /etc/machine-id) +BOOT_PARTUUID=$(blkid --match-tag PARTUUID --output value "$BOOT") +ROOT_PARTUUID=$(blkid --match-tag PARTUUID --output value "$ROOT") +[[ ! -z "$SWAP" ]] && SWAP_PARTUUID=$(blkid --match-tag PARTUUID --output value "$SWAP") + + HARDWARE_CONFIG=$(mktemp) +if [[ "$USE_ECNRYPTION" = true ]] +then cat < "$HARDWARE_CONFIG" networking.hostId = "$HOSTID"; - boot.zfs.devNodes = "$ZFS"; + boot.zfs.devNodes = "$DISK_DEV_NODES"; boot.supportedFilesystems = [ "zfs" ]; - boot.kernelParams = [ "zfs.zfs_arc_max=$ZFS_ARC_MAX" "nohibernate" ]; + boot.initrd.luks.devices."$BOOT_DEVICE_NAME".device = "/dev/disk/by-partuuid/$BOOT_PARTUUID"; + boot.initrd.luks.devices."$LUKS_DEVICE_NAME".device = "/dev/disk/by-partuuid/$ROOT_PARTUUID"; CONFIG +else +cat < "$HARDWARE_CONFIG" + networking.hostId = "$HOSTID"; + boot.zfs.devNodes = "$DISK_DEV_NODES"; + boot.supportedFilesystems = [ "zfs" ]; +CONFIG +fi pprint "Append ZFS configuration to hardware-configuration.nix" sed -i "\$e cat $HARDWARE_CONFIG" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix sed -i 's|fsType = "zfs";|fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];|g' $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix - -if [[ -n "$SWAP" ]]; then - sed -i "s#swapDevices = \[ \];#swapDevices = \[\n {\n device = \"$SWAP\";\n }\n \];#" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix +if [[ ! -z "$SWAP" ]]; then + sed -i "s|swapDevices = \[ \];|swapDevices = \[\n {\n device = \"/dev/disk/by-partuuid/$SWAP_PARTUUID\";\n randomEncryption.enable = true;\n randomEncryption.allowDiscards = true;\n }\n \];|" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix fi - -# pprint "Copy hardware config to machines folder" -# cp /mnt/etc/nixos/hardware-configuration.nix $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix chown 1000:100 $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix -# Change for flakes -# sed -i "s#(modulesPath + \"/installer/scan/not-detected.nix\")#\"${toString modulesPath}/installer/scan/not-detected.nix\"#" $CONFIG_FOLDER/machines/$DEVICE_NAME/hardware-configuration.nix git add -A -clean_stdin -read -s -p "> Do you want to execute nixos-install command?" -n 1 -r -echo -if [[ "$REPLY" =~ ^[Yy]$ ]] -then - nixos-install --flake "../#$DEVICE_NAME" --max-jobs $MAX_JOBS --no-root-passwd -fi - pprint "Copy config to destination system" mkdir -p /mnt/home/alukard/nixos-config cp -aT $CONFIG_FOLDER /mnt/home/alukard/nixos-config pprint "Gen ssh host key for initrd" -ssh-keygen -t ed25519 -N "" -f /mnt/root/ssh_host_key -chown root:root /mnt/root/ssh_host_key -cmod 644 /mnt/root/ssh_host_key +ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/ssh_host_key +chown root:root /mnt/etc/secrets/ssh_host_key +chmod 600 /mnt/etc/secrets/ssh_host_key -umount -Rl /mnt -zpool export -a \ No newline at end of file +if [[ "$USE_ECNRYPTION" = true ]] +then + cp keyfile0.bin /mnt/etc/secrets/keyfile0.bin + chmod 000 /mnt/etc/secrets/keyfile*.bin +fi + +clean_stdin +read -s -p "> Do you want to execute nixos-install command?" -n 1 -r +echo +if [[ "$REPLY" =~ ^[Yy]$ ]] +then + nixos-install --flake "../#$DEVICE_NAME" --root /mnt --max-jobs $MAX_JOBS --no-root-passwd +fi + +# umount -Rl /mnt +# zpool export -a +# cryptsetup luksClose $LUKS_DEVICE_NAME \ No newline at end of file diff --git a/machines/Hypervisor-VM/default.nix b/machines/Hypervisor-VM/default.nix index d74fc9f..45546eb 100644 --- a/machines/Hypervisor-VM/default.nix +++ b/machines/Hypervisor-VM/default.nix @@ -1,8 +1,9 @@ -{ modulesPath, inputs, lib, pkgs, config, options, ... }: { +{ modulesPath, inputs, lib, pkgs, config, options, ... }: +let + zfs_arc_max = toString (1 * 1024 * 1024 * 1024); +in { imports = with inputs.self; [ - # "${toString modulesPath}/profiles/qemu-guest.nix" "${toString modulesPath}/profiles/hardened.nix" - # ./imports/qemu-vm.nix ./hardware-configuration.nix nixosRoles.hypervisor @@ -23,16 +24,50 @@ # boot boot = { zfs.forceImportAll = lib.mkForce false; - # loader.grub.enable = true; - loader.systemd-boot = { + loader.efi.canTouchEfiVariables = false; + loader.efi.efiSysMountPoint = "/boot/efi"; + loader.systemd-boot.enable = false; + loader.generationsDir.copyKernels = true; + loader.grub = { enable = true; - editor = false; - configurationLimit = 8; + device = "nodev"; + version = 2; + efiSupport = true; + enableCryptodisk = true; + zfsSupport = true; + efiInstallAsRemovable = true; + copyKernels = true; + # extraPrepareConfig = '' + # ''; + }; + initrd = { + supportedFilesystems = [ "zfs" ]; + luks.devices = { + "cryptboot" = { + preLVM = true; + keyFile = "/keyfile0.bin"; + allowDiscards = true; + bypassWorkqueues = config.deviceSpecific.isSSD; + fallbackToPassword = true; + # postOpenCommands = ""; + # preOpenCommands = ""; + }; + "cryptroot" = { + preLVM = true; + keyFile = "/keyfile0.bin"; + allowDiscards = true; + bypassWorkqueues = config.deviceSpecific.isSSD; + fallbackToPassword = true; + }; + }; + secrets = { + "keyfile0.bin" = "/etc/secrets/keyfile0.bin"; + }; }; - # loader.efi.canTouchEfiVariables = true; kernelPackages = pkgs.linuxPackages_hardened; kernelModules = [ "tcp_bbr" ]; kernelParams = [ + "zfs.zfs_arc_max=${zfs_arc_max}" "zswap.enabled=0" "quiet" "scsi_mod.use_blk_mq=1" @@ -62,25 +97,6 @@ "vm.swappiness" = 1; }; cleanTmpDir = true; - initrd = { - availableKernelModules = [ "tg3" ]; - postDeviceCommands = lib.mkAfter '' - zfs rollback -r rpool/enc/nixos/empty@start - ''; - # network = { - # enable = true; - # ssh = { - # enable = true; - # port = 2222; - # # hostKeys = [ /root/ssh_host_key ]; - # hostKeys = [ /home/alukard/ssh_host_key ]; - # authorizedKeys = config.users.users.alukard.openssh.authorizedKeys.keys; - # }; - # postCommands = '' - # echo "zfs load-key -a; killall zfs" >> /root/.profile - # ''; - # }; - }; }; # security.polkit.enable = true; @@ -95,7 +111,7 @@ cores = 4; }; drive = { - type = "sdd"; + type = "ssd"; speed = 500; size = 500; }; diff --git a/machines/Hypervisor-VM/hardware-configuration.nix b/machines/Hypervisor-VM/hardware-configuration.nix index ba062f6..5060f84 100644 --- a/machines/Hypervisor-VM/hardware-configuration.nix +++ b/machines/Hypervisor-VM/hardware-configuration.nix @@ -8,62 +8,73 @@ [ (modulesPath + "/profiles/qemu-guest.nix") ]; - boot.initrd.availableKernelModules = [ "ahci" "virtio_pci" "xhci_pci" "sr_mod" "virtio_blk" ]; + boot.initrd.availableKernelModules = [ "ahci" "virtio_pci" "xhci_pci" "sd_mod" "sr_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { # device = "rpool/enc/nixos/root"; - device = "rpool/enc/nixos/empty"; - fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; - }; - - fileSystems."/nix" = - { device = "rpool/enc/nixos/nix"; + { device = "rpool/nixos/root"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/home" = - { device = "rpool/enc/user/home"; + { device = "rpool/user/home"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; - fileSystems."/var/lib" = - { device = "rpool/enc/nixos/var/lib"; + fileSystems."/persistent" = + { device = "rpool/persistent/impermanence"; + fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; + }; + + fileSystems."/etc/secrets" = + { device = "rpool/persistent/secrets"; + fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; + }; + + fileSystems."/nix" = + { device = "rpool/persistent/nix"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/var/log" = - { device = "rpool/enc/nixos/var/log"; + { device = "rpool/persistent/log"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/var/lib/docker" = - { device = "rpool/enc/nixos/docker"; + { device = "rpool/persistent/docker"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/media/bittorrent" = - { device = "rpool/enc/nixos/bittorrent"; + { device = "rpool/persistent/bittorrent"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/media/libvirt" = - { device = "rpool/enc/nixos/libvirt"; + { device = "rpool/persistent/libvirt"; fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/6B88-F626"; + { device = "bpool/nixos/boot"; + fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ]; + }; + + fileSystems."/boot/efi" = + { device = "/dev/disk/by-uuid/D76A-09CE"; fsType = "vfat"; }; - # swapDevices = [ - # { - # device = "/dev/zvol/rpool/enc/swap"; - # } - # ]; + swapDevices = [ + { + device = "/dev/disk/by-partuuid/d92c3ab0-8205-49fb-99ab-abe98a943d39"; + randomEncryption.enable = true; + randomEncryption.allowDiscards = true; + } + ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking # (the default) this is the recommended approach. When using systemd-networkd it's @@ -75,7 +86,8 @@ nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; networking.hostId = "41d97526"; - boot.zfs.devNodes = "/dev/disk/by-path/virtio-pci-0000:05:00.0-part2"; + boot.zfs.devNodes = "/dev/disk/by-id"; boot.supportedFilesystems = [ "zfs" ]; - boot.kernelParams = [ "zfs.zfs_arc_max=4294967296" "nohibernate" ]; + boot.initrd.luks.devices."cryptboot".device = "/dev/disk/by-partuuid/45979da9-33b7-4c7a-8eaf-005e642a974d"; + boot.initrd.luks.devices."cryptroot".device = "/dev/disk/by-partuuid/74d318d9-1349-4281-8bbd-82ac718f052c"; } diff --git a/machines/Hypervisor-VM/imports/qemu-vm.nix b/machines/Hypervisor-VM/imports/qemu-vm.nix deleted file mode 100644 index 0755aad..0000000 --- a/machines/Hypervisor-VM/imports/qemu-vm.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ modulesPath, config, lib, pkgs, ... }: { - imports = [ - "${toString modulesPath}/profiles/qemu-guest.nix" - "${toString modulesPath}/virtualisation/qemu-vm.nix" - ]; - virtualisation = { - qemu.options = [ "-vga none" "-device virtio-vga-gl" "-display gtk,gl=on" ]; - cores = 1; - memorySize = 4096; - msize = 65536; - diskSize = 10240; - diskImage = "/media/libvirt/vm-images/${config.device}.qcow2"; - # resolution = { x = 1920; y = 1080; }; - # useNixStoreImage = true; - # writableStore = false; - # writableStore = true; - # useNixStoreImage = true; - # writableStoreUseTmpfs = true; - }; - # services.spice-vdagentd.enable = lib.mkOverride 0 true; - # services.xserver.videoDrivers = [ "qxl" ]; - # services.qemuGuest.enable = true; -}