fde-lock: duskwood clean budget 40 s, non-DB containers get up to 25 s to stop

auto-upgrade.nix

@@ -1,13 +0,0 @@
-{ config, ... }:
-
-{
-  # Rebuild once a day to automatically apply security patches
-  system.autoUpgrade.enable = true;
-  nix.optimise.automatic = true;
-  nix.gc = {
-    automatic = true;
-    dates = "daily";
-    options = "--delete-older-than 30d";
-  };
-}
-

automatic-upgrades.nix

@@ -4,6 +4,16 @@
   # Enable automatic garbage collection (prevents /boot or / filling up)
   nix.gc.automatic     = true;
   nix.gc.dates         = "03:00";           # run GC at 03:00 daily
+  nix.gc.options       = "--delete-older-than 30d"; # drop generations older than 30 days
+  nix.optimise.automatic = true;            # hard-link identical store files
+
+  # This host has no configuration.nix: point <nixos-config> at master.nix,
+  # otherwise nixos-upgrade.service (and plain nixos-rebuild) cannot find the config.
+  nix.nixPath = [
+    "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos"
+    "nixos-config=/etc/nixos/master.nix"
+    "/nix/var/nix/profiles/per-user/root/channels"
+  ];
 
   # Enable automatic system upgrades
   system.autoUpgrade = {
@@ -14,4 +24,3 @@
   };
 
 }
-

cron.nix

@@ -0,0 +1,11 @@
+{ config, pkgs, ... }:
+
+{
+  services.cron = {
+    enable = true;
+    systemCronJobs = [
+      # ISP speed test (unify-admin: scripts/uplink_logger.py), hourly at :05
+      "5 * * * *  stormwind  ${pkgs.python3}/bin/python3 /home/stormwind/isp-speed-test/uplink_logger.py 2>> /home/stormwind/isp-speed-test/uplink_logger.err"
+    ];
+  };
+}

fde-lock.nix

@@ -0,0 +1,80 @@
+{ config, pkgs, ... }:
+
+# Home Assistant endpoint that puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest.
+# HTTP API (unprivileged) -> trigger file in /run/fde-lock-api -> path unit -> root oneshot.
+# Token: /etc/nixos/secrets/fde-lock-token (not versioned).
+
+let
+  listen = "192.168.0.19";
+  port = 8742;
+  allowed = "192.168.0.14"; # varian (Home Assistant)
+
+  fde-lock = pkgs.writeShellApplication {
+    name = "fde-lock";
+    runtimeInputs = with pkgs; [
+      coreutils util-linux procps gawk gnused
+      lvm2 cryptsetup
+      config.virtualisation.incus.clientPackage
+    ];
+    text = builtins.readFile ./fde-lock/lock.sh;
+  };
+
+  lockService = mode: {
+    description = "Put Nethergarde, ashenvale_crypt and duskwood_crypt to rest (${mode})";
+    # Never (re)started by a rebuild, only by its path unit.
+    restartIfChanged = false;
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${fde-lock}/bin/fde-lock ${mode}";
+      StandardInput = "null";
+      TimeoutStartSec = "4min";
+    };
+  };
+
+  trigger = name: {
+    wantedBy = [ "paths.target" ];
+    pathConfig = {
+      PathChanged = "/run/fde-lock-api/${name}";
+      Unit = "fde-${name}.service";
+    };
+  };
+in
+{
+  systemd.services.fde-lock = lockService "lock";
+  systemd.services.fde-check = lockService "check";
+  systemd.paths.fde-lock = trigger "lock";
+  systemd.paths.fde-check = trigger "check";
+
+  users.users.fde-lock-api = { isSystemUser = true; group = "fde-lock-api"; };
+  users.groups.fde-lock-api = { };
+
+  systemd.tmpfiles.rules = [
+    "d /run/fde-lock-api 0755 fde-lock-api fde-lock-api -"
+    "d /run/fde-lock 0755 root root -"
+  ];
+
+  systemd.services.fde-lock-api = {
+    description = "HTTP trigger for fde-lock (Home Assistant)";
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    environment = {
+      FDE_LOCK_LISTEN = listen;
+      FDE_LOCK_PORT = toString port;
+      FDE_LOCK_ALLOWED = allowed;
+    };
+    serviceConfig = {
+      ExecStart = "${pkgs.python3}/bin/python3 ${./fde-lock/api.py}";
+      User = "fde-lock-api";
+      Group = "fde-lock-api";
+      LoadCredential = "token:/etc/nixos/secrets/fde-lock-token";
+      Restart = "always";
+      RestartSec = 5;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ReadWritePaths = [ "/run/fde-lock-api" ];
+    };
+  };
+}

fde-lock/api.py

@@ -0,0 +1,71 @@
+# HTTP trigger for fde-lock, called by Home Assistant.
+#   POST /lock    lock Nethergarde, ashenvale_crypt and duskwood_crypt (202, runs in the background)
+#   POST /check   dry run, changes nothing
+#   GET  /status  result of the last run
+# Needs "Authorization: Bearer <token>" and a source address from FDE_LOCK_ALLOWED.
+# This process has no privileges: it only touches a trigger file that a root path unit watches.
+
+import hmac
+import json
+import os
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+TOKEN = (Path(os.environ["CREDENTIALS_DIRECTORY"]) / "token").read_text().strip().encode()
+ALLOWED = set(os.environ["FDE_LOCK_ALLOWED"].split())
+TRIGGER_DIR = Path("/run/fde-lock-api")
+STATUS = Path("/run/fde-lock/status")
+
+
+def read_status():
+    if not STATUS.exists():
+        return {"state": "never-run"}
+    result = {"steps": []}
+    for line in STATUS.read_text().splitlines():
+        parts = line.split("\t")
+        if parts[0] == "step" and len(parts) >= 3:
+            result["steps"].append({"name": parts[1], "result": parts[2], "detail": "\t".join(parts[3:])})
+        elif len(parts) == 2:
+            result[parts[0]] = parts[1]
+    return result
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "fde-lock"
+
+    def authorized(self):
+        if self.client_address[0] not in ALLOWED:
+            return False
+        given = self.headers.get("Authorization", "").encode()
+        return hmac.compare_digest(given, b"Bearer " + TOKEN)
+
+    def reply(self, code, body):
+        data = json.dumps(body).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_POST(self):
+        if not self.authorized():
+            return self.reply(403, {"error": "forbidden"})
+        action = {"/lock": "lock", "/check": "check"}.get(self.path)
+        if action is None:
+            return self.reply(404, {"error": "not found"})
+        (TRIGGER_DIR / action).write_text(f"{time.time()}\n")
+        self.reply(202, {"accepted": action})
+
+    def do_GET(self):
+        if not self.authorized():
+            return self.reply(403, {"error": "forbidden"})
+        if self.path != "/status":
+            return self.reply(404, {"error": "not found"})
+        self.reply(200, read_status())
+
+    def log_message(self, fmt, *args):
+        print(f"{self.client_address[0]} {fmt % args}", flush=True)
+
+
+ThreadingHTTPServer((os.environ["FDE_LOCK_LISTEN"], int(os.environ["FDE_LOCK_PORT"])), Handler).serve_forever()

fde-lock/lock.sh

@@ -0,0 +1,340 @@
+# Puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest.
+#   fde-lock lock    lock all three, never prompts, every step time-boxed
+#   fde-lock check   dry run: report what lock would act on, change nothing
+# Each part first gets a clean attempt (stop containers, unmount, lock/close, shutdown) with a budget of
+# $clean_secs ($quiesce_secs for duskwood, whose containers need it). Whatever fails or is still open
+# after that is forced: truenas is killed, a store's users are killed and its key is wiped even if the
+# device stays busy.
+# duskwood holds the VMs' /mnt/mariadb and /mnt/appdata: before unmounting, the guest stops every
+# container that binds them, databases first. Their names go to $state_dir/duskwood-<vm>.containers.
+# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).
+
+# writeShellApplication enables errexit; failures are handled per step instead.
+set +o errexit
+
+mode="${1:-}"
+case "$mode" in
+  lock | check) ;;
+  *) echo "usage: fde-lock lock|check" >&2; exit 2 ;;
+esac
+
+status=/run/fde-lock/status
+tmp="$status.tmp"
+state_dir=/var/lib/fde-lock
+truenas_ip=192.168.0.16
+clean_secs=20
+# Measured 2026-11-02: DBs stop in <= 5 s, the rest in <= 10 s, crusader's uptime-coomer in 20 s.
+quiesce_secs=40
+failed=0
+# incus exec into a VM starts commands with a PATH that misses the NixOS system profile.
+gpath=/run/wrappers/bin:/run/current-system/sw/bin
+db_re='mariadb|mysql|postgres|redis|(^|[-_])db([-_0-9]|$)'
+
+mkdir -p /run/fde-lock "$state_dir"
+: > "$tmp"
+put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
+put "mode	$mode"
+put "state	running"
+put "started	$(date -Is)"
+
+# step NAME SECONDS CMD...: run CMD with a hard timeout, no stdin, record result.
+step() {
+  local name="$1" secs="$2" out rc
+  shift 2
+  out="$(timeout --kill-after=3 "$secs" "$@" < /dev/null 2>&1)"
+  rc=$?
+  out="$(printf '%s' "$out" | tr '\t\n' '  ' | cut -c1-300)"
+  if [ "$rc" -eq 0 ]; then
+    put "step	$name	ok	$out"
+  else
+    put "step	$name	fail($rc)	$out"
+  fi
+  echo "$name: rc=$rc $out"
+  return "$rc"
+}
+note() { put "step	$1	$2	$3"; echo "$1: $2 $3"; }
+
+# Clean steps share one deadline per part; once it has passed they are skipped.
+left() { local l=$(( $1 - $(date +%s) )); [ "$l" -gt 0 ] || l=0; echo "$l"; }
+cstep() {
+  local name="$1" t
+  t="$(left "$2")"
+  shift 2
+  if [ "$t" -eq 0 ]; then note "$name" skip "clean budget used up"; return 1; fi
+  step "$name" "$t" "$@"
+}
+
+# incus queries must not hang the run either.
+inc() { timeout --kill-after=3 10 incus "$@" < /dev/null 2> /dev/null; }
+running() { [ "$(inc list "^$1\$" -c s -f csv)" = RUNNING ]; }
+
+# truenas counts as gone only when its qemu process is gone, whatever incus says.
+truenas_qemu() { pgrep -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; }
+truenas_alive() { running truenas || truenas_qemu > /dev/null; }
+
+# ---------------------------------------------------------------- Nethergarde
+lock_nethergarde() {
+  # Hard NFS mounts from truenas would block forever once it is gone: detach them first.
+  # findmnt only reads mountinfo, it does not touch the NFS server.
+  local m
+  findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do
+    case "$src" in
+      "$truenas_ip":*)
+        if [ "$mode" = check ]; then note "nfs-umount" info "would detach $m"
+        else step "nfs-umount $m" 5 umount -f -l "$m" || true; fi ;;
+    esac
+  done
+
+  if ! truenas_alive; then
+    note "truenas" info "not running, keys not in memory"
+    return 0
+  fi
+
+  if [ "$mode" = check ]; then
+    step "truenas-query" 15 incus exec -T truenas -- midclt call pool.dataset.query \
+      '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}'
+    note "truenas-qemu" info "pid $(truenas_qemu | tr '\n' ' ')"
+    return 0
+  fi
+
+  # Clean: unload-key zeroes the ZFS key, then a regular shutdown, both within the budget.
+  local dl t
+  dl=$(( $(date +%s) + clean_secs ))
+  cstep "truenas-lock" "$dl" incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \
+    '{"force_umount": true}'
+  t="$(left "$dl")"
+  if [ "$t" -gt 3 ]; then
+    step "truenas-stop" "$t" incus stop truenas --timeout $(( t - 3 ))
+  else
+    note "truenas-stop" skip "clean budget used up"
+  fi
+
+  # Forced: whatever is left of the VM goes, its memory (and every key in it) with it.
+  if truenas_alive; then step "truenas-stop-force" 10 incus stop truenas --force; fi
+  if truenas_qemu > /dev/null; then step "truenas-kill" 5 pkill -9 -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; fi
+  sleep 1
+  if truenas_qemu > /dev/null; then
+    note "truenas" fail "qemu still alive: $(truenas_qemu | tr '\n' ' ')"
+  else
+    note "truenas" ok "stopped, memory released"
+  fi
+  return 0
+}
+
+# ------------------------------------------------- LUKS stores (ashenvale, duskwood)
+# store_mounts VG: host mount points of the VG's volumes. findmnt names the source /dev/dm-N
+# (bind mounts add "[/subdir]"), so match both spellings.
+store_mounts() {
+  local d devs=" "
+  for d in /dev/mapper/"$1"-*; do
+    [ -e "$d" ] && devs+="$d $(realpath "$d") "
+  done
+  findmnt -rn -o TARGET,SOURCE | while read -r t src; do
+    case "$devs" in *" ${src%%\[*} "*) echo "$t" ;; esac
+  done
+}
+
+# guest_mount INST VOL: where the guest mounted a block volume (by-id path from the Incus device name).
+guest_mount() {
+  # shellcheck disable=SC2016 # $1 expands in the guest shell
+  timeout --kill-after=3 10 incus exec -T "$1" --env "PATH=$gpath" -- sh -c \
+    'd="$(realpath -e "$1")" && findmnt -rn -o TARGET -S "$d"' sh \
+    "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_incus_$2-part1" < /dev/null 2> /dev/null | head -n1
+}
+
+# guest_containers INST MOUNT...: running containers in the guest that bind anything below MOUNT.
+guest_containers() {
+  local inst="$1"
+  shift
+  # shellcheck disable=SC2016 # $@, $s, $n expand in the guest shell
+  timeout --kill-after=3 10 incus exec -T "$inst" --env "PATH=$gpath" -- sh -c '
+    docker ps -q | xargs -r docker inspect -f "{{.Name}}{{range .Mounts}} {{.Source}}{{end}}" |
+    while read -r n srcs; do
+      for s in $srcs; do
+        for m in "$@"; do
+          case "$s/" in "$m"/*) echo "${n#/}"; continue 3 ;; esac
+        done
+      done
+    done' sh "$@" < /dev/null 2> /dev/null
+}
+
+# release_instance NAME VG INST DEADLINE QUIESCE "VOL|PATH"...: free the store's volumes in one VM.
+# Clean within DEADLINE: stop the containers that use them (QUIESCE=1, databases first), unmount in
+# the guest, detach. Forced after: kill those containers, lazy unmount, detach anyway.
+# A stopped VM keeps its devices, so it cannot start while the store is locked.
+release_instance() {
+  local name="$1" vg="$2" inst="$3" dl="$4" quiesce="$5" e vol path s c
+  shift 5
+  local vols=() mounts=() ctrs=() dbs=() rest=() stuck=()
+  if ! running "$inst"; then
+    note "$name $inst" info "not running, devices stay attached"
+    return 0
+  fi
+  for e in "$@"; do
+    vol="${e%%|*}"
+    path="${e#*|}"
+    vols+=("$vol")
+    [ -n "$path" ] || path="$(guest_mount "$inst" "$vol")"
+    [ -n "$path" ] && mounts+=("$path")
+  done
+
+  if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ]; then
+    mapfile -t ctrs < <(guest_containers "$inst" "${mounts[@]}")
+    # Kept for the unlock; an empty list (second lock in a row) must not overwrite a good one.
+    if [ "${#ctrs[@]}" -gt 0 ]; then printf '%s\n' "${ctrs[@]}" > "$state_dir/$name-$inst.containers"; fi
+    for c in "${ctrs[@]}"; do
+      if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
+    done
+    # Databases first while the budget is fresh; docker itself kills what outlives -t.
+    # The rest gets what is left, minus 8 s for unmount, detach and close.
+    if [ "${#dbs[@]}" -gt 0 ]; then
+      s=$(( $(left "$dl") - 4 )); [ "$s" -le 10 ] || s=10
+      if [ "$s" -gt 0 ]; then
+        cstep "docker-stop-db $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${dbs[@]}"
+      else
+        note "docker-stop-db $inst" skip "clean budget used up"
+      fi
+    fi
+    if [ "${#rest[@]}" -gt 0 ]; then
+      s=$(( $(left "$dl") - 8 )); [ "$s" -le 25 ] || s=25
+      if [ "$s" -gt 0 ]; then
+        cstep "docker-stop $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${rest[@]}"
+      else
+        note "docker-stop $inst" skip "clean budget used up"
+      fi
+    fi
+  fi
+
+  for path in "${mounts[@]}"; do
+    cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- umount "$path" ||
+      stuck+=("$path")
+  done
+
+  # Forced: nothing that still holds a mount survives, then the mount is detached from the tree.
+  if [ "${#stuck[@]}" -gt 0 ]; then
+    if [ "${#ctrs[@]}" -gt 0 ]; then
+      step "docker-kill $inst" 10 incus exec -T "$inst" --env "PATH=$gpath" -- docker kill "${ctrs[@]}" || true
+    fi
+    for path in "${stuck[@]}"; do
+      step "guest-umount-lazy $inst:$path" 5 incus exec -T "$inst" --env "PATH=$gpath" -- umount -l "$path"
+    done
+  fi
+
+  for vol in "${vols[@]}"; do
+    cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" ||
+      step "detach-force $vol from $inst" 10 incus storage volume detach "$vg" "$vol" "$inst"
+  done
+  return 0
+}
+
+# lock_store NAME CRYPT VG QUIESCE: release every attached volume (all VMs in parallel), then close the LUKS device.
+lock_store() {
+  local name="$1" crypt="$2" vg="$3" quiesce="$4"
+  if [ ! -e "/dev/mapper/$crypt" ]; then
+    note "$name" info "$crypt not open, nothing to do"
+    return 0
+  fi
+
+  # Volumes of the pool attached to instances, grouped per instance as "vol|path" lines.
+  local vol inst path
+  local -A per=()
+  for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do
+    for inst in $(inc storage volume show "$vg" "custom/$vol" \
+                  | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
+      per[$inst]+="$vol|$(inc config device get "$inst" "$vol" path)"$'\n'
+    done
+  done
+
+  local m es c
+  if [ "$mode" = check ]; then
+    for inst in "${!per[@]}"; do
+      local mounts=() dbs=() rest=()
+      while IFS='|' read -r vol path; do
+        [ -n "$vol" ] || continue
+        if [ -z "$path" ] && running "$inst"; then path="$(guest_mount "$inst" "$vol")"; fi
+        [ -n "$path" ] && mounts+=("$path")
+        note "$name detach" info "would detach $vol from $inst (${path:-not mounted})"
+      done <<< "${per[$inst]}"
+      if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ] && running "$inst"; then
+        while read -r c; do
+          [ -n "$c" ] || continue
+          if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
+        done < <(guest_containers "$inst" "${mounts[@]}")
+        note "$name containers $inst" info "would stop db: ${dbs[*]:-none}; then: ${rest[*]:-none}"
+      fi
+    done
+    for m in $(store_mounts "$vg"); do note "$name host-umount" info "would umount $m"; done
+    note "$name" info "would close $crypt"
+    return 0
+  fi
+
+  # Clean: release the volumes in every VM at once, unmount on the host, close, all within the budget.
+  local dl pids=()
+  dl=$(( $(date +%s) + (quiesce ? quiesce_secs : clean_secs) ))
+  for inst in "${!per[@]}"; do
+    mapfile -t es < <(printf '%s' "${per[$inst]}")
+    release_instance "$name" "$vg" "$inst" "$dl" "$quiesce" "${es[@]}" &
+    pids+=($!)
+  done
+  [ "${#pids[@]}" -eq 0 ] || wait "${pids[@]}"
+  for m in $(store_mounts "$vg" | sort -r); do
+    cstep "$name host-umount $m" "$dl" umount "$m"
+  done
+  cstep "$name vg-deactivate" "$dl" vgchange -an "$vg"
+  cstep "$name luks-close" "$dl" cryptsetup close "$crypt"
+
+  if [ -e "/dev/mapper/$crypt" ]; then
+    # Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close.
+    for m in $(store_mounts "$vg"); do
+      if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "$name virtiofsd-kill $m" ok ""; fi
+    done
+    for m in $(store_mounts "$vg" | sort -r); do
+      step "$name host-umount-lazy $m" 5 umount -l "$m"
+    done
+    step "$name vg-deactivate-force" 10 vgchange -an "$vg"
+    step "$name luks-close-force" 10 cryptsetup close "$crypt"
+  fi
+
+  if [ -e "/dev/mapper/$crypt" ]; then
+    # Still busy: wipe the key now (suspend drops it from the kernel), then replace the
+    # stack with error targets so nothing waits on the suspended device.
+    step "$name luks-suspend" 10 cryptsetup luksSuspend "$crypt"
+    for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
+      step "$name dm-remove-force $m" 5 dmsetup remove --force "$m"
+    done
+    step "$name dm-remove-force $crypt" 5 dmsetup remove --force "$crypt"
+  fi
+
+  # At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped).
+  if [ ! -e "/dev/mapper/$crypt" ]; then
+    note "$name" ok "$crypt closed"
+  else
+    local attr target
+    attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
+    target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')"
+    if [[ "$attr" == *s* ]]; then
+      note "$name" ok "$crypt still listed but suspended, key wiped"
+    elif [ "$target" = error ]; then
+      note "$name" ok "$crypt replaced by error target, key dropped"
+    else
+      note "$name" fail "$crypt still active: attr $attr, target $target"
+    fi
+  fi
+  return 0
+}
+
+# All three run in parallel so a slow TrueNAS shutdown or container stop does not delay the others.
+lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
+lock_store ashenvale ashenvale_crypt ashenvale_pool 0 &
+lock_store duskwood duskwood_crypt duskwood_pool 1 &
+wait
+
+# Subshell failures do not reach this shell; derive them from the recorded steps.
+if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "duskwood" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
+  failed=1
+fi
+
+put "finished	$(date -Is)"
+if [ "$failed" -eq 0 ]; then put "state	done"; else put "state	failed"; fi
+mv "$tmp" "$status"
+exit "$failed"

hardware-configuration.nix

@@ -22,6 +22,7 @@
 
     # helps isolate PCIe ports if your card shares groups
     "pci_acs_override=downstream,multifunction"
+    "init_on_free=1" # zero freed pages (fde-lock: no key material left behind)
   ];
 
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "usbhid" "sd_mod" ];

master.nix

@@ -24,8 +24,8 @@ in
     ./incus.nix
     ./drives.nix
     ./prometheus.nix
-    #./cron.nix
+    ./cron.nix
-    #./auto-upgrade.nix
+    ./fde-lock.nix
   ];
 
   # This value determines the NixOS release from which the default