feat: fde-lock endpoint for home assistant, init_on_free

cron.nix

@@ -0,0 +1,11 @@
+{ config, pkgs, ... }:
+
+{
+  services.cron = {
+    enable = true;
+    systemCronJobs = [
+      # ISP speed test (unify-admin: scripts/uplink_logger.py), hourly at :05
+      "5 * * * *  stormwind  ${pkgs.python3}/bin/python3 /home/stormwind/isp-speed-test/uplink_logger.py 2>> /home/stormwind/isp-speed-test/uplink_logger.err"
+    ];
+  };
+}

fde-lock.nix

@@ -0,0 +1,80 @@
+{ config, pkgs, ... }:
+
+# Home Assistant endpoint that puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
+# HTTP API (unprivileged) -> trigger file in /run/fde-lock-api -> path unit -> root oneshot.
+# Token: /etc/nixos/secrets/fde-lock-token (not versioned).
+
+let
+  listen = "192.168.0.19";
+  port = 8742;
+  allowed = "192.168.0.14"; # varian (Home Assistant)
+
+  fde-lock = pkgs.writeShellApplication {
+    name = "fde-lock";
+    runtimeInputs = with pkgs; [
+      coreutils util-linux procps gawk gnused
+      lvm2 cryptsetup
+      config.virtualisation.incus.clientPackage
+    ];
+    text = builtins.readFile ./fde-lock/lock.sh;
+  };
+
+  lockService = mode: {
+    description = "Put Nethergarde and ashenvale_crypt to rest (${mode})";
+    # Never (re)started by a rebuild, only by its path unit.
+    restartIfChanged = false;
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${fde-lock}/bin/fde-lock ${mode}";
+      StandardInput = "null";
+      TimeoutStartSec = "6min";
+    };
+  };
+
+  trigger = name: {
+    wantedBy = [ "paths.target" ];
+    pathConfig = {
+      PathChanged = "/run/fde-lock-api/${name}";
+      Unit = "fde-${name}.service";
+    };
+  };
+in
+{
+  systemd.services.fde-lock = lockService "lock";
+  systemd.services.fde-check = lockService "check";
+  systemd.paths.fde-lock = trigger "lock";
+  systemd.paths.fde-check = trigger "check";
+
+  users.users.fde-lock-api = { isSystemUser = true; group = "fde-lock-api"; };
+  users.groups.fde-lock-api = { };
+
+  systemd.tmpfiles.rules = [
+    "d /run/fde-lock-api 0755 fde-lock-api fde-lock-api -"
+    "d /run/fde-lock 0755 root root -"
+  ];
+
+  systemd.services.fde-lock-api = {
+    description = "HTTP trigger for fde-lock (Home Assistant)";
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    environment = {
+      FDE_LOCK_LISTEN = listen;
+      FDE_LOCK_PORT = toString port;
+      FDE_LOCK_ALLOWED = allowed;
+    };
+    serviceConfig = {
+      ExecStart = "${pkgs.python3}/bin/python3 ${./fde-lock/api.py}";
+      User = "fde-lock-api";
+      Group = "fde-lock-api";
+      LoadCredential = "token:/etc/nixos/secrets/fde-lock-token";
+      Restart = "always";
+      RestartSec = 5;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ReadWritePaths = [ "/run/fde-lock-api" ];
+    };
+  };
+}

fde-lock/api.py

@@ -0,0 +1,71 @@
+# HTTP trigger for fde-lock, called by Home Assistant.
+#   POST /lock    lock Nethergarde and ashenvale_crypt (202, runs in the background)
+#   POST /check   dry run, changes nothing
+#   GET  /status  result of the last run
+# Needs "Authorization: Bearer <token>" and a source address from FDE_LOCK_ALLOWED.
+# This process has no privileges: it only touches a trigger file that a root path unit watches.
+
+import hmac
+import json
+import os
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+TOKEN = (Path(os.environ["CREDENTIALS_DIRECTORY"]) / "token").read_text().strip().encode()
+ALLOWED = set(os.environ["FDE_LOCK_ALLOWED"].split())
+TRIGGER_DIR = Path("/run/fde-lock-api")
+STATUS = Path("/run/fde-lock/status")
+
+
+def read_status():
+    if not STATUS.exists():
+        return {"state": "never-run"}
+    result = {"steps": []}
+    for line in STATUS.read_text().splitlines():
+        parts = line.split("\t")
+        if parts[0] == "step" and len(parts) >= 3:
+            result["steps"].append({"name": parts[1], "result": parts[2], "detail": "\t".join(parts[3:])})
+        elif len(parts) == 2:
+            result[parts[0]] = parts[1]
+    return result
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "fde-lock"
+
+    def authorized(self):
+        if self.client_address[0] not in ALLOWED:
+            return False
+        given = self.headers.get("Authorization", "").encode()
+        return hmac.compare_digest(given, b"Bearer " + TOKEN)
+
+    def reply(self, code, body):
+        data = json.dumps(body).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_POST(self):
+        if not self.authorized():
+            return self.reply(403, {"error": "forbidden"})
+        action = {"/lock": "lock", "/check": "check"}.get(self.path)
+        if action is None:
+            return self.reply(404, {"error": "not found"})
+        (TRIGGER_DIR / action).write_text(f"{time.time()}\n")
+        self.reply(202, {"accepted": action})
+
+    def do_GET(self):
+        if not self.authorized():
+            return self.reply(403, {"error": "forbidden"})
+        if self.path != "/status":
+            return self.reply(404, {"error": "not found"})
+        self.reply(200, read_status())
+
+    def log_message(self, fmt, *args):
+        print(f"{self.client_address[0]} {fmt % args}", flush=True)
+
+
+ThreadingHTTPServer((os.environ["FDE_LOCK_LISTEN"], int(os.environ["FDE_LOCK_PORT"])), Handler).serve_forever()

fde-lock/lock.sh

@@ -0,0 +1,165 @@
+# Puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
+#   fde-lock lock    lock both, never prompts, every step time-boxed
+#   fde-lock check   dry run: report what lock would act on, change nothing
+# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).
+
+# writeShellApplication enables errexit; failures are handled per step instead.
+set +o errexit
+
+mode="${1:-}"
+case "$mode" in
+  lock | check) ;;
+  *) echo "usage: fde-lock lock|check" >&2; exit 2 ;;
+esac
+
+status=/run/fde-lock/status
+tmp="$status.tmp"
+truenas_ip=192.168.0.16
+crypt=ashenvale_crypt
+vg=ashenvale_pool
+failed=0
+
+mkdir -p /run/fde-lock
+: > "$tmp"
+put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
+put "mode	$mode"
+put "state	running"
+put "started	$(date -Is)"
+
+# step NAME SECONDS CMD...: run CMD with a hard timeout, no stdin, record result.
+step() {
+  local name="$1" secs="$2" out rc
+  shift 2
+  out="$(timeout --kill-after=5 "$secs" "$@" < /dev/null 2>&1)"
+  rc=$?
+  out="$(printf '%s' "$out" | tr '\t\n' '  ' | cut -c1-300)"
+  if [ "$rc" -eq 0 ]; then
+    put "step	$name	ok	$out"
+  else
+    put "step	$name	fail($rc)	$out"
+  fi
+  echo "$name: rc=$rc $out"
+  return "$rc"
+}
+note() { put "step	$1	info	$2"; echo "$1: $2"; }
+
+running() { [ "$(incus list "^$1\$" -c s -f csv 2> /dev/null)" = RUNNING ]; }
+
+# ---------------------------------------------------------------- Nethergarde
+lock_nethergarde() {
+  # Hard NFS mounts from truenas would block forever once it is gone: detach them first.
+  # findmnt only reads mountinfo, it does not touch the NFS server.
+  local m
+  findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do
+    case "$src" in
+      "$truenas_ip":*)
+        if [ "$mode" = check ]; then note "nfs-umount" "would detach $m"
+        else step "nfs-umount $m" 15 umount -f -l "$m" || true; fi ;;
+    esac
+  done
+
+  if ! running truenas; then
+    note "truenas" "not running, keys not in memory"
+    return 0
+  fi
+
+  if [ "$mode" = check ]; then
+    step "truenas-query" 30 incus exec -T truenas -- midclt call pool.dataset.query \
+      '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}' || failed=1
+    return 0
+  fi
+
+  # Unlocked ZFS keys are zeroed by unload-key; force_umount stops the shares that hold the pool.
+  step "truenas-lock" 120 incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \
+    '{"force_umount": true}' || failed=1
+  step "truenas-locked?" 30 incus exec -T truenas -- midclt call pool.dataset.query \
+    '[["id","=","Nethergarde"]]' '{"select":["id","locked"]}' || true
+
+  # Stop the VM in any case, so no key can stay in its memory (init_on_free zeroes it on the host).
+  step "truenas-stop" 100 incus stop truenas --timeout 90 || true
+  if running truenas; then
+    step "truenas-stop-force" 30 incus stop truenas --force || failed=1
+  fi
+  running truenas && failed=1
+  return 0
+}
+
+# ------------------------------------------------------------ ashenvale_crypt
+ashenvale_mounts() {
+  findmnt -rn -o TARGET,SOURCE | while read -r t src; do
+    case "$src" in "/dev/mapper/$vg-"*) echo "$t" ;; esac
+  done
+}
+
+lock_ashenvale() {
+  if [ ! -e "/dev/mapper/$crypt" ]; then
+    note "ashenvale" "$crypt not open, nothing to do"
+    return 0
+  fi
+
+  # Detach every ashenvale_pool volume from running instances (today: phonecam on varian).
+  local inst vol
+  for vol in $(incus storage volume list "$vg" type=custom -c n -f csv 2> /dev/null); do
+    for inst in $(incus storage volume show "$vg" "custom/$vol" 2> /dev/null \
+                  | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
+      if [ "$mode" = check ]; then note "detach" "would detach $vol from $inst"; continue; fi
+      local path
+      path="$(incus config device get "$inst" "$vol" path 2> /dev/null)"
+      if [ -n "$path" ] && running "$inst"; then
+        step "guest-umount $inst:$path" 30 incus exec -T "$inst" -- umount "$path" \
+          || step "guest-umount-lazy $inst:$path" 15 incus exec -T "$inst" -- umount -l "$path" || true
+      fi
+      step "detach $vol from $inst" 60 incus storage volume detach "$vg" "$vol" "$inst" || true
+    done
+  done
+
+  local m
+  if [ "$mode" = check ]; then
+    for m in $(ashenvale_mounts); do note "host-umount" "would umount $m"; done
+    note "ashenvale" "would close $crypt"
+    return 0
+  fi
+
+  # Whatever still serves the volume (virtiofsd after a failed detach) keeps it busy.
+  for m in $(ashenvale_mounts); do
+    pkill -f -- "--shared-dir=$m( |$)" || true
+  done
+  for m in $(ashenvale_mounts | sort -r); do
+    step "host-umount $m" 30 umount "$m" || step "host-umount-lazy $m" 15 umount -l "$m" || true
+  done
+
+  step "vg-deactivate" 30 vgchange -an "$vg" || true
+  step "luks-close" 30 cryptsetup close "$crypt" || true
+
+  if [ -e "/dev/mapper/$crypt" ]; then
+    # Still busy: drop the key now (suspend wipes it from the kernel), then replace the
+    # stack with error targets so nothing waits on the suspended device.
+    step "luks-suspend" 30 cryptsetup luksSuspend "$crypt" || true
+    for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
+      step "dm-remove-force $m" 15 dmsetup remove --force "$m" || true
+    done
+    step "dm-remove-force $crypt" 15 dmsetup remove --force "$crypt" || true
+  fi
+
+  if [ -e "/dev/mapper/$crypt" ]; then
+    failed=1
+    note "ashenvale" "$crypt still present: $(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
+  fi
+  return 0
+}
+
+# Both locks run in parallel so a slow TrueNAS shutdown does not delay ashenvale.
+lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
+lock_ashenvale
+wait
+
+# Subshell failures do not reach this shell; derive them from the recorded steps.
+if awk -F '\t' '$1 == "step" && $2 ~ /^truenas-(lock|stop-force|query)$/ && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
+  failed=1
+fi
+running truenas && [ "$mode" = lock ] && failed=1
+
+put "finished	$(date -Is)"
+if [ "$failed" -eq 0 ]; then put "state	done"; else put "state	failed"; fi
+mv "$tmp" "$status"
+exit "$failed"

hardware-configuration.nix

@@ -22,6 +22,7 @@
 
     # helps isolate PCIe ports if your card shares groups
     "pci_acs_override=downstream,multifunction"
+    "init_on_free=1" # zero freed pages (fde-lock: no key material left behind)
   ];
 
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "usbhid" "sd_mod" ];

master.nix

@@ -24,7 +24,8 @@ in
     ./incus.nix
     ./drives.nix
     ./prometheus.nix
-    #./cron.nix
+    ./cron.nix
+    ./fde-lock.nix
     #./auto-upgrade.nix
   ];