From 848c57caa04b2c6b103144b552c4cad8a555a511 Mon Sep 17 00:00:00 2001
From: Leyla Becker <git@jan-leila.com>
Date: Sun, 8 Dec 2024 21:58:11 -0600
Subject: [PATCH] got impermanence to wipe drive

---
 configurations/home-manager/leyla/default.nix |   1 +
 .../home-manager/leyla/impermanence.nix       |  21 ++
 .../nixos/defiant/configuration.nix           |  57 +++--
 configurations/nixos/defiant/default.nix      |   3 +-
 configurations/nixos/defiant/disko-config.nix | 196 ++++++++++--------
 .../nixos/defiant/hardware-configuration.nix  | 122 +----------
 configurations/nixos/defiant/impermanence.nix |  87 ++++++++
 flake.lock                                    |  16 ++
 flake.nix                                     |   9 +-
 util/default.nix                              |   3 +
 10 files changed, 281 insertions(+), 234 deletions(-)
 create mode 100644 configurations/home-manager/leyla/impermanence.nix
 create mode 100644 configurations/nixos/defiant/impermanence.nix

diff --git a/configurations/home-manager/leyla/default.nix b/configurations/home-manager/leyla/default.nix
index ba8535d..48d38db 100644
--- a/configurations/home-manager/leyla/default.nix
+++ b/configurations/home-manager/leyla/default.nix
@@ -6,6 +6,7 @@
   imports = [
     ./i18n.nix
     ./packages.nix
+    ./impermanence.nix
   ];
 
   config = {
diff --git a/configurations/home-manager/leyla/impermanence.nix b/configurations/home-manager/leyla/impermanence.nix
new file mode 100644
index 0000000..c5f228f
--- /dev/null
+++ b/configurations/home-manager/leyla/impermanence.nix
@@ -0,0 +1,21 @@
+{...}: {
+  home.persistence."/persistent/home/leyla" = {
+    directories = [
+      "desktop"
+      "downloads"
+      "documents"
+      ".ssh"
+      ".nixops"
+      ".local/share/keyrings"
+      ".local/share/direnv"
+      {
+        directory = ".local/share/Steam";
+        method = "symlink";
+      }
+    ];
+    # files = [
+    #   ".screenrc"
+    # ];
+    allowOther = true;
+  };
+}
diff --git a/configurations/nixos/defiant/configuration.nix b/configurations/nixos/defiant/configuration.nix
index 009442e..ec728ca 100644
--- a/configurations/nixos/defiant/configuration.nix
+++ b/configurations/nixos/defiant/configuration.nix
@@ -6,7 +6,7 @@
 }: {
   imports = [
     inputs.disko.nixosModules.disko
-    ./services.nix
+    # ./services.nix
   ];
 
   nixpkgs.config.allowUnfree = true;
@@ -23,37 +23,30 @@
     };
   };
 
-  boot.loader.grub = {
-    enable = true;
-    zfsSupport = true;
-    efiSupport = true;
-    efiInstallAsRemovable = true;
-  };
-
-  apps = {
-    base_domain = "jan-leila.com";
-    macvlan = {
-      subnet = "192.168.1.0/24";
-      gateway = "192.168.1.1";
-      networkInterface = "bond0";
-    };
-    pihole = {
-      image = "pihole/pihole:2024.07.0";
-      ip = "192.168.1.201";
-    };
-    headscale = {
-      subdomain = "vpn";
-    };
-    jellyfin = {
-      subdomain = "media";
-    };
-    forgejo = {
-      subdomain = "git";
-    };
-    nextcloud = {
-      subdomain = "drive";
-    };
-  };
+  # apps = {
+  #   base_domain = "jan-leila.com";
+  #   macvlan = {
+  #     subnet = "192.168.1.0/24";
+  #     gateway = "192.168.1.1";
+  #     networkInterface = "bond0";
+  #   };
+  #   pihole = {
+  #     image = "pihole/pihole:2024.07.0";
+  #     ip = "192.168.1.201";
+  #   };
+  #   headscale = {
+  #     subdomain = "vpn";
+  #   };
+  #   jellyfin = {
+  #     subdomain = "media";
+  #   };
+  #   forgejo = {
+  #     subdomain = "git";
+  #   };
+  #   nextcloud = {
+  #     subdomain = "drive";
+  #   };
+  # };
 
   services = {
     zfs = {
diff --git a/configurations/nixos/defiant/default.nix b/configurations/nixos/defiant/default.nix
index 36e1b6f..d5ea594 100644
--- a/configurations/nixos/defiant/default.nix
+++ b/configurations/nixos/defiant/default.nix
@@ -1,8 +1,9 @@
 # server nas
 {...}: {
   imports = [
-    ./configuration.nix
     ./disko-config.nix
     ./hardware-configuration.nix
+    ./impermanence.nix
+    ./configuration.nix
   ];
 }
diff --git a/configurations/nixos/defiant/disko-config.nix b/configurations/nixos/defiant/disko-config.nix
index 0be19c4..82fd480 100644
--- a/configurations/nixos/defiant/disko-config.nix
+++ b/configurations/nixos/defiant/disko-config.nix
@@ -1,26 +1,4 @@
-{...}: let
-  bootDisk = devicePath: {
-    type = "disk";
-    device = devicePath;
-    content = {
-      type = "gpt";
-      partitions = {
-        boot = {
-          size = "1M";
-          type = "EF02"; # for grub MBR
-        };
-        ESP = {
-          size = "1G";
-          type = "EF00";
-          content = {
-            type = "filesystem";
-            format = "vfat";
-            mountpoint = "/boot";
-          };
-        };
-      };
-    };
-  };
+{lib, ...}: let
   zfsDisk = devicePath: {
     type = "disk";
     device = devicePath;
@@ -37,41 +15,41 @@
       };
     };
   };
-  # cacheDisk = devicePath: swapSize: {
-  #   type = "disk";
-  #   device = devicePath;
-  #   content = {
-  #     type = "gpt";
-  #     partitions = {
-  #       encryptedSwap = {
-  #         size = swapSize;
-  #         content = {
-  #           type = "swap";
-  #           randomEncryption = true;
-  #           discardPolicy = "both";
-  #           resumeDevice = true;
-  #         };
-  #       };
-  #       zfs = {
-  #         size = "100%";
-  #         content = {
-  #           type = "zfs";
-  #           pool = "rpool";
-  #         };
-  #       };
-  #     };
-  #   };
-  # };
+  cacheDisk = devicePath: {
+    type = "disk";
+    device = devicePath;
+    content = {
+      type = "gpt";
+      partitions = {
+        # We are having to boot off of the nvm cache drive because I cant figure out how to boot via the HBA
+        ESP = {
+          size = "64M";
+          type = "EF00";
+          content = {
+            type = "filesystem";
+            format = "vfat";
+            mountpoint = "/boot";
+            mountOptions = ["umask=0077"];
+          };
+        };
+        zfs = {
+          size = "100%";
+          content = {
+            type = "zfs";
+            pool = "rpool";
+          };
+        };
+      };
+    };
+  };
 in {
   disko.devices = {
     disk = {
-      boot = bootDisk "/dev/disk/by-path/pci-0000:23:00.3-usb-0:1:1.0-scsi-0:0:0:0";
+      hd_18_tb_a = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTCXVEB";
+      hd_18_tb_b = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTCXWSC";
+      hd_18_tb_c = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTD10EH";
 
-      hd_13_tb_a = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTCXVEB";
-      hd_13_tb_b = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTCXWSC";
-      hd_13_tb_c = zfsDisk "/dev/disk/by-id/ata-ST18000NE000-3G6101_ZVTD10EH";
-
-      # ssd_2_tb_a = cacheDisk "64G" "/dev/disk/by-id/XXX";
+      ssd_4_tb_a = cacheDisk "/dev/disk/by-id/nvme-Samsung_SSD_990_PRO_4TB_S7KGNU0X907881F";
     };
     zpool = {
       rpool = {
@@ -84,58 +62,106 @@ in {
                 # should this only mirror for this inital config with 3 drives we will used raidz2 for future configs???
                 mode = "mirror";
                 members = [
-                  "hd_13_tb_a"
-                  "hd_13_tb_b"
-                  "hd_13_tb_c"
+                  "hd_18_tb_a"
+                  "hd_18_tb_b"
+                  "hd_18_tb_c"
                 ];
               }
             ];
-            cache = [];
-            # cache = [ "ssd_2_tb_a" ];z
+            cache = ["ssd_4_tb_a"];
           };
         };
 
         options = {
           ashift = "12";
+          autotrim = "on";
         };
 
-        rootFsOptions = {
-          # encryption = "on";
-          # keyformat = "hex";
-          # keylocation = "prompt";
-          compression = "lz4";
-          xattr = "sa";
-          acltype = "posixacl";
-          canmount = "off";
-          "com.sun:auto-snapshot" = "false";
-        };
+        rootFsOptions =
+          {
+            canmount = "off";
+            mountpoint = "none";
+
+            xattr = "sa";
+            acltype = "posixacl";
+            relatime = "on";
+
+            compression = "lz4";
+
+            "com.sun:auto-snapshot" = "false";
+          }
+          # TODO: have an option to enable encryption
+          // lib.attrsets.optionalAttrs false {
+            encryption = "on";
+            keyformat = "hex";
+            keylocation = "prompt";
+          };
 
         datasets = {
-          root = {
+          # local datasets are for data that should be considered ephemeral
+          "local" = {
             type = "zfs_fs";
-            mountpoint = "/";
-            options.mountpoint = "legacy";
-            postCreateHook = "zfs snapshot rpool/root@blank";
+            options.canmount = "off";
           };
-          home = {
-            type = "zfs_fs";
-            options.mountpoint = "legacy";
-            mountpoint = "/home";
-            postCreateHook = "zfs snapshot rpool/home@blank";
-          };
-          nix = {
+          # the nix directory is local because its all generable from our configuration
+          "local/system/nix" = {
             type = "zfs_fs";
             mountpoint = "/nix";
-          };
-          persistent = {
-            type = "zfs_fs";
-            mountpoint = "/persistent";
             options = {
-              "com.sun:auto-snapshot" = "true";
+              atime = "off";
+              relatime = "off";
+              canmount = "on";
             };
           };
+          "local/system/root" = {
+            type = "zfs_fs";
+            mountpoint = "/";
+            options = {
+              canmount = "on";
+            };
+            postCreateHook = ''
+              zfs snapshot rpool/local/system/root@blank
+            '';
+          };
+          "local/home/leyla" = {
+            type = "zfs_fs";
+            mountpoint = "/home/leyla";
+            options = {
+              canmount = "on";
+            };
+            postCreateHook = ''
+              zfs snapshot rpool/local/home/leyla@blank
+            '';
+          };
+
+          # persist datasets are datasets that contain information that we would like to keep around
+          "persist" = {
+            type = "zfs_fs";
+            options.canmount = "off";
+          };
+          "persist/system/root" = {
+            type = "zfs_fs";
+            mountpoint = "/persist/system/root";
+            options = {
+              "com.sun:auto-snapshot" = "true";
+              mountpoint = "/persist/system/root";
+            };
+          };
+          "persist/home/leyla" = {
+            type = "zfs_fs";
+            mountpoint = "/persist/home/leyla";
+            options = {
+              "com.sun:auto-snapshot" = "true";
+              mountpoint = "/persist/home/leyla";
+            };
+          };
+
+          # TODO: separate dataset for logs that wont participate in snapshots and rollbacks with the rest of the system
         };
       };
     };
   };
+  networking = {
+    hostId = "c51763d6";
+  };
 }
diff --git a/configurations/nixos/defiant/hardware-configuration.nix b/configurations/nixos/defiant/hardware-configuration.nix
index b55bcfd..425cbc2 100644
--- a/configurations/nixos/defiant/hardware-configuration.nix
+++ b/configurations/nixos/defiant/hardware-configuration.nix
@@ -11,136 +11,32 @@
     (modulesPath + "/installer/scan/not-detected.nix")
   ];
 
-  security.sudo.extraConfig = "Defaults lecture=never";
-
   boot = {
     initrd = {
-      availableKernelModules = ["xhci_pci" "aacraid" "ahci" "usbhid" "usb_storage" "sd_mod"];
+      availableKernelModules = ["xhci_pci" "aacraid" "ahci" "usbhid" "nvme" "usb_storage" "sd_mod"];
       kernelModules = [];
-      # TODO: figure out some kind of snapshotting before rollbacks
-      # postDeviceCommands = lib.mkAfter ''
-      #   zfs rollback -r rpool/root@blank
-      #   zfs rollback -r rpool/home@blank
-      # '';
-      # systemd = {
-      #   enable = lib.mkDefault true;
-      #   services.rollback = {
-      #     description = "Rollback root filesystem to a pristine state on boot";
-      #     wantedBy = [
-      #       "zfs.target"
-      #       "initrd.target"
-      #     ];
-      #     after = [
-      #       "zfs-import-rpool.service"
-      #     ];
-      #     before = [
-      #       "sysroot.mount"
-      #       "fs.target"
-      #     ];
-      #     path = with pkgs; [
-      #       zfs
-      #     ];
-      #     unitConfig.DefaultDependencies = "no";
-      #     # serviceConfig = {
-      #     #   Type = "oneshot";
-      #     #   ExecStart =
-      #     #     "${config.boot.zfs.package}/sbin/zfs rollback -r rpool/home@blank";
-      #     # };
-      #     serviceConfig.Type = "oneshot";
-      #     script = ''
-      #       zfs list -t snapshot || echo
-      #       zfs rollback -r rpool/root@blank
-      #       zfs rollback -r rpool/home@blank
-      #     '';
-      #   };
-      # };
     };
     kernelModules = ["kvm-amd"];
-    kernelParams = ["quiet"];
     extraModulePackages = [];
 
+    # Bootloader.
+    loader = {
+      systemd-boot.enable = true;
+      efi = {
+        canTouchEfiVariables = true;
+        efiSysMountPoint = "/boot";
+      };
+    };
     supportedFilesystems = ["zfs"];
 
     zfs.extraPools = ["rpool"];
   };
 
-  swapDevices = [];
-
-  # fileSystems = {
-  #   "/" = {
-  #     neededForBoot = true;
-  #   };
-
-  #   "/home" = {
-  #     neededForBoot = true;
-  #   };
-
-  #   "/persistent" = {
-  #     neededForBoot = true;
-  #   };
-  # };
-
   networking = {
-    hostId = "c51763d6";
     hostName = "defiant"; # Define your hostname.
     useNetworkd = true;
   };
 
-  # environment.persistence."/persistent" = {
-  #   enable = true;
-  #   hideMounts = true;
-  #   directories = [
-  #     # "/run/secrets"
-
-  #     "/etc/ssh"
-
-  #     "/var/log"
-  #     "/var/lib/nixos"
-  #     "/var/lib/systemd/coredump"
-
-  #     # config.apps.pihole.directory.root
-
-  #     # config.apps.jellyfin.mediaDirectory
-  #     # config.services.jellyfin.configDir
-  #     # config.services.jellyfin.cacheDir
-  #     # config.services.jellyfin.dataDir
-
-  #     # "/var/hass" # config.users.users.hass.home
-  #     # "/var/postgresql" # config.users.users.postgresql.home
-  #     # "/var/forgejo" # config.users.users.forgejo.home
-  #     # "/var/nextcloud" # config.users.users.nextcloud.home
-  #     # "/var/headscale" # config.users.users.headscale.home
-  #   ];
-  #   files = [
-  #     "/etc/machine-id"
-  #     # config.environment.sessionVariables.SOPS_AGE_KEY_FILE
-  #   ];
-  #   users.leyla = {
-  #     directories = [
-  #       "documents"
-  #       ".ssh"
-  #     ];
-  #     files = [];
-  #   };
-  # };
-
-  # systemd.services = {
-  #   # https://github.com/openzfs/zfs/issues/10891
-  #   systemd-udev-settle.enable = false;
-  #   # Snapshots are not accessible on boot for some reason this should fix it
-  #   # https://github.com/NixOS/nixpkgs/issues/257505
-  #   zfs-mount = {
-  #     serviceConfig = {
-  #       # ExecStart = [ "${lib.getExe' pkgs.util-linux "mount"} -a -t zfs -o remount" ];
-  #       ExecStart = [
-  #         "${lib.getExe' pkgs.util-linux "mount"} -t zfs rpool/root -o remount"
-  #         "${lib.getExe' pkgs.util-linux "mount"} -t zfs rpool/home -o remount"
-  #         "${lib.getExe' pkgs.util-linux "mount"} -t zfs rpool/persistent -o remount"
-  #       ];
-  #     };
-  #   };
-  # };
-
   systemd.network = {
     enable = true;
 
diff --git a/configurations/nixos/defiant/impermanence.nix b/configurations/nixos/defiant/impermanence.nix
new file mode 100644
index 0000000..9d8b2af
--- /dev/null
+++ b/configurations/nixos/defiant/impermanence.nix
@@ -0,0 +1,87 @@
+{
+  lib,
+  config,
+  ...
+}: {
+  boot.initrd.postResumeCommands = lib.mkAfter ''
+    zfs rollback -r rpool/local/system/root@blank
+    zfs rollback -r rpool/local/home/leyla@blank
+  '';
+
+  # systemd.services = {
+  #   # https://github.com/openzfs/zfs/issues/10891
+  #   systemd-udev-settle.enable = false;
+  #   # Snapshots are not accessible on boot for some reason this should fix it
+  #   # https://github.com/NixOS/nixpkgs/issues/257505
+  #   zfs-mount = {
+  #     serviceConfig = {
+  #       ExecStart = ["zfs mount -a -o remount"];
+  #       # ExecStart = [
+  #       #   "${lib.getExe' pkgs.util-linux "mount"} -t zfs rpool/local -o remount"
+  #       #   "${lib.getExe' pkgs.util-linux "mount"} -t zfs rpool/persistent -o remount"
+  #       # ];
+  #     };
+  #   };
+  # };
+
+  # boot.initrd.systemd.services.rollback = {
+  #   description = "Rollback filesystem to a pristine state on boot";
+  #   wantedBy = [
+  #     "initrd.target"
+  #   ];
+  #   after = [
+  #     "zfs-import-rpool.service"
+  #   ];
+  #   before = [
+  #     "sysroot.mount"
+  #   ];
+  #   requiredBy = [
+  #     "sysroot.mount"
+  #   ];
+  #   serviceConfig = {
+  #     Type = "oneshot";
+  #     ExecStart = ''
+  #       zfs rollback -r rpool/local/system/root@blank
+  #       zfs rollback -r rpool/local/home@blank
+  #     '';
+  #   };
+  # };
+
+  fileSystems."/".neededForBoot = true;
+  fileSystems."/home/leyla".neededForBoot = true;
+  fileSystems."/persist/system/root".neededForBoot = true;
+  fileSystems."/persist/home/leyla".neededForBoot = true;
+
+  environment.persistence."/persist/system/root" = {
+    enable = true;
+    hideMounts = true;
+    directories = [
+      "/run/secrets"
+
+      "/etc/ssh"
+
+      "/var/log"
+      "/var/lib/nixos"
+      "/var/lib/systemd/coredump"
+
+      # config.apps.pihole.directory.root
+
+      # config.apps.jellyfin.mediaDirectory
+      # config.services.jellyfin.configDir
+      # config.services.jellyfin.cacheDir
+      # config.services.jellyfin.dataDir
+
+      # "/var/hass" # config.users.users.hass.home
+      # "/var/postgresql" # config.users.users.postgresql.home
+      # "/var/forgejo" # config.users.users.forgejo.home
+      # "/var/nextcloud" # config.users.users.nextcloud.home
+      # "/var/headscale" # config.users.users.headscale.home
+    ];
+    files = [
+      "/etc/machine-id"
+      config.environment.sessionVariables.SOPS_AGE_KEY_FILE
+    ];
+  };
+
+  security.sudo.extraConfig = "Defaults lecture=never";
+}
diff --git a/flake.lock b/flake.lock
index 7d882de..83f85f8 100644
--- a/flake.lock
+++ b/flake.lock
@@ -127,6 +127,21 @@
         "type": "github"
       }
     },
+    "impermanence": {
+      "locked": {
+        "lastModified": 1731242966,
+        "narHash": "sha256-B3C3JLbGw0FtLSWCjBxU961gLNv+BOOBC6WvstKLYMw=",
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "rev": "3ed3f0eaae9fcc0a8331e77e9319c8a4abd8a71a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "type": "github"
+      }
+    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -207,6 +222,7 @@
         "firefox-addons": "firefox-addons",
         "flake-compat": "flake-compat",
         "home-manager": "home-manager",
+        "impermanence": "impermanence",
         "nix-darwin": "nix-darwin",
         "nix-vscode-extensions": "nix-vscode-extensions",
         "nixos-hardware": "nixos-hardware",
diff --git a/flake.nix b/flake.nix
index bcc31d7..54d1f7d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -24,9 +24,9 @@
     };
 
     # delete your darlings
-    # impermanence = {
-    #   url = "github:nix-community/impermanence";
-    # };
+    impermanence = {
+      url = "github:nix-community/impermanence";
+    };
 
     nix-darwin = {
       url = "github:LnL7/nix-darwin";
@@ -65,6 +65,9 @@
   outputs = {
     self,
     nixpkgs,
+    sops-nix,
+    home-manager,
+    impermanence,
     ...
   } @ inputs: let
     util = import ./util {inherit inputs;};
diff --git a/util/default.nix b/util/default.nix
index ce85a7f..9ddb5e8 100644
--- a/util/default.nix
+++ b/util/default.nix
@@ -7,6 +7,7 @@
   home-manager = inputs.home-manager;
   nix-darwin = inputs.nix-darwin;
   sops-nix = inputs.sops-nix;
+  impermanence = inputs.impermanence;
 
   systems = [
     "aarch64-darwin"
@@ -25,6 +26,7 @@
     common-modules
     ++ [
       sops-nix.homeManagerModules.sops
+      impermanence.homeManagerModules.impermanence
       ../modules/home-manager-modules
     ];
 
@@ -70,6 +72,7 @@ in {
         system-modules
         ++ [
           sops-nix.nixosModules.sops
+          impermanence.nixosModules.impermanence
           home-manager.nixosModules.home-manager
           ../modules/nixos-modules
           ../configurations/nixos/${host}