storage-refactor #9

Open
jan-leila wants to merge 40 commits from storage-refactor into main
6 changed files with 295 additions and 271 deletions
Showing only changes of commit 9df29cc07f - Show all commits

View file

@ -33,44 +33,6 @@
isPrincipleUser = true;
};
};
impermanence.enable = true;
storage = {
enable = true;
encryption = true;
notifications = {
enable = true;
host = "smtp.protonmail.ch";
port = 587;
to = "leyla@jan-leila.com";
user = "noreply@jan-leila.com";
tokenFile = config.sops.secrets."services/zfs_smtp_token".path;
};
pool = {
# We are having to boot off of the nvm cache drive because I cant figure out how to boot via the HBA
bootDrives = ["nvme-Samsung_SSD_990_PRO_4TB_S7KGNU0X907881F"];
vdevs = [
[
"ata-ST18000NE000-3G6101_ZVTCXVEB"
"ata-ST18000NE000-3G6101_ZVTCXWSC"
"ata-ST18000NE000-3G6101_ZVTD10EH"
"ata-ST18000NT001-3NF101_ZVTE0S3Q"
"ata-ST18000NT001-3NF101_ZVTEF27J"
"ata-ST18000NE000-3G6101_ZVTJ7359"
]
[
"ata-ST4000NE001-2MA101_WS2275P3"
"ata-ST4000NE001-2MA101_WS227B9F"
"ata-ST4000NE001-2MA101_WS227CEW"
"ata-ST4000NE001-2MA101_WS227CYN"
"ata-ST4000NE001-2MA101_WS23TBWV"
"ata-ST4000NE001-2MA101_WS23TC5F"
]
];
cache = [
"nvme-Samsung_SSD_990_PRO_4TB_S7KGNU0X907881F"
];
};
};
network_storage = {
enable = true;
directories = [
@ -104,6 +66,53 @@
};
};
storage = {
zfs = {
enable = true;
notifications = {
enable = true;
host = "smtp.protonmail.ch";
port = 587;
to = "leyla@jan-leila.com";
user = "noreply@jan-leila.com";
tokenFile = config.sops.secrets."services/zfs_smtp_token".path;
};
pool = {
encryption = {
enable = true;
};
vdevs = [
[
"ata-ST18000NE000-3G6101_ZVTCXVEB"
"ata-ST18000NE000-3G6101_ZVTCXWSC"
"ata-ST18000NE000-3G6101_ZVTD10EH"
"ata-ST18000NT001-3NF101_ZVTE0S3Q"
"ata-ST18000NT001-3NF101_ZVTEF27J"
"ata-ST18000NE000-3G6101_ZVTJ7359"
]
[
"ata-ST4000NE001-2MA101_WS2275P3"
"ata-ST4000NE001-2MA101_WS227B9F"
"ata-ST4000NE001-2MA101_WS227CEW"
"ata-ST4000NE001-2MA101_WS227CYN"
"ata-ST4000NE001-2MA101_WS23TBWV"
"ata-ST4000NE001-2MA101_WS23TC5F"
]
];
# We are having to boot off of the nvm cache drive because I cant figure out how to boot via the HBA
cache = {
cache0 = {
device = "nvme-Samsung_SSD_990_PRO_4TB_S7KGNU0X907881F";
boot = true;
};
};
};
};
impermanence = {
enable = true;
};
};
systemd.network = {
enable = true;

View file

@ -8,5 +8,6 @@
imports = [
./impermanence.nix
./zfs.nix
./storage.nix
];
}

View file

@ -1,7 +1,6 @@
{
lib,
config,
util,
...
}: {
# TODO: create all of the datasets from option and home-manager datasets
@ -13,50 +12,49 @@
storage.zfs.datasets = {
"persist/system/nix" = {
type = "zfs_fs";
mountpoint = "/nix";
options = {
mount = {
enable = true;
mountPoint = "/nix";
};
snapshot = {
autoSnapshot = false;
};
atime = "off";
relatime = "off";
canmount = "on";
"com.sun:auto-snapshot" = "false";
};
};
"persist/system/var/log" = {
type = "zfs_fs";
mountpoint = "/var/log";
options = {
"com.sun:auto-snapshot" = "false";
mount = {
enable = true;
mountPoint = "/var/log";
};
snapshot = {
autoSnapshot = false;
};
};
"persist/system/root" = {
type = "zfs_fs";
mount = {
enable = true;
mountPoint = "/";
};
};
};
}
(util.mkUnless config.storage.impermanence.enable {
(lib.mkIf (!config.storage.impermanence.enable) {
# TODO: create datasets for systemd.services.<name>.storage.impermanence.datasets
storage.zfs.datasets = {
"persist/system/root" = {
type = "zfs_fs";
mountpoint = "/";
canmount = "on";
snapshot = {
autoSnapshot = true;
};
};
};
})
(lib.mkIf config.storage.impermanence.enable {
storage.impermanence.datasets = {
"persist/system/root" = {
type = "zfs_fs";
};
};
storage.zfs.datasets = {
"local/system/root" = {
type = "zfs_fs";
mountpoint = "/";
options = {
canmount = "on";
};
postCreateHook = ''
zfs snapshot rpool/local/system/root@blank
'';
directories = {
"/var/lib/nixos".enable = true;
"/var/lib/systemd/coredump".enable = true;
@ -66,6 +64,18 @@
};
};
};
storage.zfs.datasets = {
"local/system/root" = {
type = "zfs_fs";
mount = {
enable = true;
mountPoint = "/";
};
snapshot = {
blankSnapshot = true;
};
};
};
# TODO: home-manager.users.<user>.storage.impermanence.enable
# is false then persist the entire directory of the user

View file

@ -6,25 +6,6 @@
description = "Type of ZFS dataset (filesystem or volume)";
};
# ZFS dataset options that match what's currently hardcoded in rootFsOptions
canmount = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off" "noauto"]);
default = null;
description = "Controls whether the file system can be mounted";
};
mountpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Controls the mount point used for this file system";
};
xattr = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off" "sa" "dir"]);
default = null;
description = "Extended attribute storage method";
};
acltype = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["off" "nfsv4" "posixacl"]);
default = null;
@ -37,56 +18,82 @@
description = "Controls when access time is updated";
};
atime = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off"]);
default = null;
description = "Controls whether access time is updated";
};
xattr = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off" "sa" "dir"]);
default = null;
description = "Extended attribute storage method";
};
compression = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off" "lz4" "gzip" "zstd" "lzjb" "zle"]);
default = null;
description = "Compression algorithm to use";
};
encryption = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off" "aes-128-ccm" "aes-192-ccm" "aes-256-ccm" "aes-128-gcm" "aes-192-gcm" "aes-256-gcm"]);
default = null;
description = "Encryption algorithm to use";
};
keyformat = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["raw" "hex" "passphrase"]);
default = null;
description = "Format of the encryption key";
};
keylocation = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Location of the encryption key";
};
autoSnapshot = lib.mkOption {
type = lib.types.nullOr lib.types.bool;
default = null;
description = "Enable automatic snapshots for this dataset";
};
# Additional common ZFS options
recordsize = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Suggested block size for files in the file system";
};
sync = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["standard" "always" "disabled"]);
default = null;
description = "Synchronous write behavior";
};
atime = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["on" "off"]);
mount = {
enable = lib.mkOption {
type = lib.types.nullOr (lib.types.either lib.types.bool (lib.types.enum ["on" "off" "noauto"]));
default = null;
description = "Controls whether access time is updated";
};
mountPoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Controls the mount point used for this file system";
};
};
encryption = {
enable = lib.mkEnableOption "should encryption be enabled";
type = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["aes-128-ccm" "aes-192-ccm" "aes-256-ccm" "aes-128-gcm" "aes-192-gcm" "aes-256-gcm"]);
default = null;
description = "What encryption type to use";
};
keyformat = lib.mkOption {
type = lib.types.nullOr (lib.types.enum ["raw" "hex" "passphrase"]);
default = null;
description = "Format of the encryption key";
};
keylocation = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Location of the encryption key";
};
};
snapshot = {
# This option should set this option flag
# "com.sun:auto-snapshot" = "false";
autoSnapshot = lib.mkOption {
type = lib.types.nullOr lib.types.bool;
default = null;
description = "Enable automatic snapshots for this dataset";
};
# TODO: this is what blank snapshot should set
# postCreateHook = ''
# zfs snapshot rpool/local/system/root@blank
# '';
blankSnapshot = lib.mkEnableOption "Should a blank snapshot be auto created in the post create hook";
};
recordSize = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Suggested block size for files in the file system";
};
# Custom options for disko integration
postCreateHook = lib.mkOption {
type = lib.types.str;
default = "";

View file

@ -1,10 +1,5 @@
args @ {
lib,
name,
...
}: {...}: let
args @ {lib, ...}: {name, ...}: let
datasetSubmodule = (import ./dataset.nix) args;
pathPermissions = {
read = lib.mkEnableOption "should the path have read permissions";
write = lib.mkEnableOption "should the path have read permissions";
@ -52,6 +47,8 @@ in {
};
config = {
mountpoint = "/${name}";
mount = {
mountPoint = lib.mkDefault "/${name}";
};
};
}

View file

@ -171,153 +171,153 @@ in {
};
# Disko configuration based on pool settings
disko.devices = {
disk = (
builtins.listToAttrs (
builtins.map
(drive:
lib.attrsets.nameValuePair (drive.name) {
type = "disk";
device = "/dev/disk/by-id/${drive.value}";
content = {
type = "gpt";
partitions = {
ESP = lib.mkIf (builtins.elem drive.value bootDrives) {
size = config.storage.zfs.pool.bootPartitionSize;
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = ["umask=0077"];
};
};
zfs = {
size = "100%";
content = {
type = "zfs";
pool = "rpool";
};
};
};
};
})
allDrives
)
);
zpool = {
rpool = {
type = "zpool";
mode = {
topology = {
type = "topology";
vdev = (
builtins.map (disks: {
mode = config.storage.zfs.pool.mode;
members =
builtins.map (disk: disk.name) disks;
})
poolVdevs
);
cache = builtins.map (disk: disk.name) poolCache;
};
};
# disko.devices = {
# disk = (
# builtins.listToAttrs (
# builtins.map
# (drive:
# lib.attrsets.nameValuePair (drive.name) {
# type = "disk";
# device = "/dev/disk/by-id/${drive.value}";
# content = {
# type = "gpt";
# partitions = {
# ESP = lib.mkIf (builtins.elem drive.value bootDrives) {
# size = config.storage.zfs.pool.bootPartitionSize;
# type = "EF00";
# content = {
# type = "filesystem";
# format = "vfat";
# mountpoint = "/boot";
# mountOptions = ["umask=0077"];
# };
# };
# zfs = {
# size = "100%";
# content = {
# type = "zfs";
# pool = "rpool";
# };
# };
# };
# };
# })
# allDrives
# )
# );
# zpool = {
# rpool = {
# type = "zpool";
# mode = {
# topology = {
# type = "topology";
# vdev = (
# builtins.map (disks: {
# mode = config.storage.zfs.pool.mode;
# members =
# builtins.map (disk: disk.name) disks;
# })
# poolVdevs
# );
# cache = builtins.map (disk: disk.name) poolCache;
# };
# };
options = {
ashift = "12";
autotrim = "on";
};
# options = {
# ashift = "12";
# autotrim = "on";
# };
rootFsOptions = let
rootDataset = config.storage.zfs.rootDataset;
# Start with defaults that match the original hardcoded values
defaults = {
canmount = "off";
mountpoint = "none";
xattr = "sa";
acltype = "posixacl";
relatime = "on";
compression = "lz4";
"com.sun:auto-snapshot" = "false";
};
# Override defaults with non-null values from rootDataset
userOptions = lib.attrsets.filterAttrs (_: v: v != null) {
canmount = rootDataset.canmount;
mountpoint = rootDataset.mountpoint;
xattr = rootDataset.xattr;
acltype = rootDataset.acltype;
relatime = rootDataset.relatime;
compression = rootDataset.compression;
encryption = rootDataset.encryption;
keyformat = rootDataset.keyformat;
keylocation = rootDataset.keylocation;
recordsize = rootDataset.recordsize;
sync = rootDataset.sync;
atime = rootDataset.atime;
"com.sun:auto-snapshot" =
if rootDataset.autoSnapshot == null
then null
else
(
if rootDataset.autoSnapshot
then "true"
else "false"
);
};
# Only apply pool encryption if user hasn't set encryption options in rootDataset
poolEncryptionOptions =
lib.attrsets.optionalAttrs (
config.storage.zfs.pool.encryption.enable
&& rootDataset.encryption == null
&& rootDataset.keyformat == null
&& rootDataset.keylocation == null
) {
encryption = "on";
keyformat = config.storage.zfs.pool.encryption.keyformat;
keylocation = config.storage.zfs.pool.encryption.keylocation;
};
in
defaults // userOptions // rootDataset.options // poolEncryptionOptions;
# rootFsOptions = let
# rootDataset = config.storage.zfs.rootDataset;
# # Start with defaults that match the original hardcoded values
# defaults = {
# canmount = "off";
# mountpoint = "none";
# xattr = "sa";
# acltype = "posixacl";
# relatime = "on";
# compression = "lz4";
# "com.sun:auto-snapshot" = "false";
# };
# # Override defaults with non-null values from rootDataset
# userOptions = lib.attrsets.filterAttrs (_: v: v != null) {
# canmount = rootDataset.canmount;
# mountpoint = rootDataset.mountpoint;
# xattr = rootDataset.xattr;
# acltype = rootDataset.acltype;
# relatime = rootDataset.relatime;
# compression = rootDataset.compression;
# encryption = rootDataset.encryption;
# keyformat = rootDataset.keyformat;
# keylocation = rootDataset.keylocation;
# recordsize = rootDataset.recordsize;
# sync = rootDataset.sync;
# atime = rootDataset.atime;
# "com.sun:auto-snapshot" =
# if rootDataset.autoSnapshot == null
# then null
# else
# (
# if rootDataset.autoSnapshot
# then "true"
# else "false"
# );
# };
# # Only apply pool encryption if user hasn't set encryption options in rootDataset
# poolEncryptionOptions =
# lib.attrsets.optionalAttrs (
# config.storage.zfs.pool.encryption.enable
# && rootDataset.encryption == null
# && rootDataset.keyformat == null
# && rootDataset.keylocation == null
# ) {
# encryption = "on";
# keyformat = config.storage.zfs.pool.encryption.keyformat;
# keylocation = config.storage.zfs.pool.encryption.keylocation;
# };
# in
# defaults // userOptions // rootDataset.options // poolEncryptionOptions;
datasets = lib.mkMerge [
(
lib.attrsets.mapAttrs (name: value: {
type = value.type;
options = let
# For datasets, only include non-null user-specified values
userOptions = lib.attrsets.filterAttrs (_: v: v != null) {
canmount = value.canmount;
xattr = value.xattr;
acltype = value.acltype;
relatime = value.relatime;
compression = value.compression;
encryption = value.encryption;
keyformat = value.keyformat;
keylocation = value.keylocation;
recordsize = value.recordsize;
sync = value.sync;
atime = value.atime;
"com.sun:auto-snapshot" =
if value.autoSnapshot == null
then null
else
(
if value.autoSnapshot
then "true"
else "false"
);
};
in
userOptions // (value.options or {});
mountpoint = value.mountpoint;
postCreateHook = value.postCreateHook or "";
})
config.storage.zfs.datasets
)
];
};
};
};
# datasets = lib.mkMerge [
# (
# lib.attrsets.mapAttrs (name: value: {
# type = value.type;
# options = let
# # For datasets, only include non-null user-specified values
# userOptions = lib.attrsets.filterAttrs (_: v: v != null) {
# canmount = value.canmount;
# xattr = value.xattr;
# acltype = value.acltype;
# relatime = value.relatime;
# compression = value.compression;
# encryption = value.encryption;
# keyformat = value.keyformat;
# keylocation = value.keylocation;
# recordsize = value.recordsize;
# sync = value.sync;
# atime = value.atime;
# "com.sun:auto-snapshot" =
# if value.autoSnapshot == null
# then null
# else
# (
# if value.autoSnapshot
# then "true"
# else "false"
# );
# };
# in
# userOptions // (value.options or {});
# mountpoint = value.mountpoint;
# postCreateHook = value.postCreateHook or "";
# })
# config.storage.zfs.datasets
# )
# ];
# };
# };
# };
# Post-activation scripts for validation
system.activationScripts = {