hacky.nix

{
  pkgs,
  lib,
  config,
  ...
}:
{
  options.ec2.nix-instance-store = {
    device = lib.mkOption {
      type = lib.types.string;
      default = "/dev/ephemeral0";
      description = "The instance store to use for the nix store";
    };
  };
  # this would all be a lot easier if nix store of the image was on a separate partition. Appliance-style
  # TODO: Move to nix apliance
  config = {
    fileSystems = {
      "/mnt/instance-store" = {
        device = config.ec2.nix-instance-store.device;
        fsType = "ext4";
        autoFormat = true;
        autoResize = true;
        # enabling this makes the kernel not boot. but unclear to me "why"
        # neededForBoot = true;
      };
      # bind mount the old store
      "/nix/old-store" = {
        device = "/nix/store";
        options = [ "bind" ];
      };
      "/mnt/nix-store-merged" = {
        overlay = {
          lowerdir = [ "/nix/old-store" ];
          upperdir = "/mnt/instance-store/nix-upperdir";
          workdir = "/mnt/instance-store/nix-workdir";
        };
      };
    };

    # TODO: IDK how to do this more elegantly?
    systemd.services.remount-nix-store = {
      description = "Remount the nix store";
      unitConfig = {
        DefaultDependencies = false;
        Before = [ "local-fs.target" ];
        RequiresMountsFor = [
          "/nix/store"
          "/mnt/nix-store-merged"
        ];
      };
      wantedBy = [ "sysinit.target" ];
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.util-linux}/bin/mount --bind /mnt/nix-store-merged /nix/store";
      };
    };
  };
}