nixos-on-zfs.md

Install NixOS on ZFS mirror with opt-in state

Commands

# ON TARGET MACHINE

# Set password
passwd

# Get IP address
ip address

# ON OTHER MACHINE

# SSH in
ssh nixos@192.168.1.X

# Become root
sudo su

# Create convenient alias for drives
DISK1=/dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_S6S1NG0R901747V
DISK2=/dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_S6S1NG0R840804J

# Clear disks
wipefs --all --force $DISK1
wipefs --all --force $DISK2
sgdisk --zap-all --clear $DISK1
sgdisk --zap-all --clear $DISK2

# Create GPT partition tables
parted $DISK1 -- mklabel gpt
parted $DISK2 -- mklabel gpt

# Create boot partitions
parted $DISK1 -- mkpart ESP fat32 1MiB 1GiB
parted $DISK2 -- mkpart ESP fat32 1MiB 1GiB
parted $DISK1 -- set 1 boot on
parted $DISK2 -- set 1 boot on

# Create root partitions
parted $DISK1 -- mkpart primary 1GiB 100%
parted $DISK2 -- mkpart primary 1GiB 100%

# Create LUKS containers
cryptsetup luksFormat --type luks2 $DISK1-part2
cryptsetup luksFormat --type luks2 $DISK2-part2

# Open LUKS containers
cryptsetup open $DISK1-part2 cryptroot1
cryptsetup open $DISK2-part2 cryptroot2

# Format root volumes as ZFS mirror
zpool create -o ashift=12 -O mountpoint=none -O compression=zstd -O acltype=posix -O dnodesize=auto -R /mnt tank mirror /dev/mapper/cryptroot1 /dev/mapper/cryptroot2

# Create root ZFS dataset
zfs create -p -o mountpoint=legacy tank/local/root

# Create blank snapshot of root ZFS dataset
zfs snapshot tank/local/root@blank

# Mount root ZFS dataset
mount -t zfs tank/local/root /mnt

# Format boot partitions as FAT32
mkfs.vfat -F32 $DISK1-part1
mkfs.vfat -F32 $DISK2-part1

# Mount first boot partition
mkdir /mnt/boot
mount $DISK1-part1 /mnt/boot

# Create and mount dataset for `/nix`
zfs create -p -o mountpoint=legacy tank/local/nix
mkdir /mnt/nix
mount -t zfs tank/local/nix /mnt/nix

# Create and mount dataset for `/home`
zfs create -p -o mountpoint=legacy tank/safe/home
mkdir /mnt/home
mount -t zfs tank/safe/home /mnt/home

# Create and mount dataset for persisted state
zfs create -p -o mountpoint=legacy tank/safe/persist
mkdir /mnt/persist
mount -t zfs tank/safe/persist /mnt/persist

# Generate initial NixOS configuration
nixos-generate-config --root /mnt

# Edit the NixOS configuration to include the following:
# {
#   # Configure networking
#   networking.networkmanager.enable = true;
#
#   # Configure LUKS
#   # blkid --match-tag UUID --output value "$DISK1-part2"
#   boot.initrd.luks.devices = {
#     "cryptroot1".device = "/dev/disk/by-uuid/<disk1-uuid>";
#     "cryptroot2".device = "/dev/disk/by-uuid/<disk2-uuid>";
#   }
#
#   # Configure ZFS
#   boot.supportedFilesystems = [ "zfs" ];
#   networking.hostId = "<random 8-digit hex string>"; # head -c8 /etc/machine-id
#   boot.zfs.devNodes = "/dev/mapper";
#   services.zfs.autoScrub.enable = true;
#
#   # Roll back to blank snapshot on boot
#   boot.initrd.postDeviceCommands = lib.mkAfter ''
#     zfs rollback -r tank/local/root@blank
#   '';
#
#   # Add users
#   users.users.evanrelf = {
#     isNormalUser = true;
#     extraGroups = [ "wheel" "networkmanager" ];
#     initialPassword = "banana";
#   };
#
#  # Persist state
#  environment.etc = {
#    "nixos".source = "/persist/etc/nixos";
#    "NetworkManager/system-connections".source = "/persist/etc/NetworkManager/system-connections";
#    "adjtime".source = "/persist/etc/adjtime";
#    "NIXOS".source = "/persist/etc/NIXOS";
#    "machine-id".source = "/persist/etc/machine-id";
#  };
#
#  systemd.tmpfiles.rules = [
#    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
#    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
#    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
#  ];
#
#  security.sudo.extraConfig = ''
#    Defaults lecture = never
#  '';
# }
# Also make sure to uncomment any useful options already in the file.
vim /mnt/etc/nixos/configuration.nix

# Copy files to `/persist`
mkdir -p /mnt/persist/etc/nixos/
cp -r /mnt/etc/nixos/* /mnt/persist/etc/nixos/

# Install NixOS
nixos-install

# Reboot
reboot

# Configuring what state to keep using instructions from "Erase Your Darlings":
# https://grahamc.com/blog/erase-your-darlings#opting-in

Resources

• https://grahamc.com/blog/erase-your-darlings
• https://nixos.wiki/wiki/NixOS_on_ZFS#How_to_install_NixOS_on_a_ZFS_root_filesystem
• https://nixos.org/nixos/manual/index.html#sec-installation
• https://mt-caret.github.io/blog/posts/2020-06-29-optin-state.html
• https://elis.nu/blog/2020/05/nixos-tmpfs-as-root/
• https://elis.nu/blog/2020/06/nixos-tmpfs-as-home/

You have to add the persist stuff after the first boot. And you have to manually copy over things to /persist.