stormwind-nixfiles/fde-lock/lock.sh

# Puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
#   fde-lock lock    lock both, never prompts, every step time-boxed
#   fde-lock check   dry run: report what lock would act on, change nothing
# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).

# writeShellApplication enables errexit; failures are handled per step instead.
set +o errexit

mode="${1:-}"
case "$mode" in
  lock | check) ;;
  *) echo "usage: fde-lock lock|check" >&2; exit 2 ;;
esac

status=/run/fde-lock/status
tmp="$status.tmp"
truenas_ip=192.168.0.16
crypt=ashenvale_crypt
vg=ashenvale_pool
failed=0

mkdir -p /run/fde-lock
: > "$tmp"
put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
put "mode	$mode"
put "state	running"
put "started	$(date -Is)"

# step NAME SECONDS CMD...: run CMD with a hard timeout, no stdin, record result.
step() {
  local name="$1" secs="$2" out rc
  shift 2
  out="$(timeout --kill-after=5 "$secs" "$@" < /dev/null 2>&1)"
  rc=$?
  out="$(printf '%s' "$out" | tr '\t\n' '  ' | cut -c1-300)"
  if [ "$rc" -eq 0 ]; then
    put "step	$name	ok	$out"
  else
    put "step	$name	fail($rc)	$out"
  fi
  echo "$name: rc=$rc $out"
  return "$rc"
}
note() { put "step	$1	info	$2"; echo "$1: $2"; }

running() { [ "$(incus list "^$1\$" -c s -f csv 2> /dev/null)" = RUNNING ]; }

# ---------------------------------------------------------------- Nethergarde
lock_nethergarde() {
  # Hard NFS mounts from truenas would block forever once it is gone: detach them first.
  # findmnt only reads mountinfo, it does not touch the NFS server.
  local m
  findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do
    case "$src" in
      "$truenas_ip":*)
        if [ "$mode" = check ]; then note "nfs-umount" "would detach $m"
        else step "nfs-umount $m" 15 umount -f -l "$m" || true; fi ;;
    esac
  done

  if ! running truenas; then
    note "truenas" "not running, keys not in memory"
    return 0
  fi

  if [ "$mode" = check ]; then
    step "truenas-query" 30 incus exec -T truenas -- midclt call pool.dataset.query \
      '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}' || failed=1
    return 0
  fi

  # Unlocked ZFS keys are zeroed by unload-key; force_umount stops the shares that hold the pool.
  step "truenas-lock" 120 incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \
    '{"force_umount": true}' || failed=1
  step "truenas-locked?" 30 incus exec -T truenas -- midclt call pool.dataset.query \
    '[["id","=","Nethergarde"]]' '{"select":["id","locked"]}' || true

  # Stop the VM in any case, so no key can stay in its memory (init_on_free zeroes it on the host).
  step "truenas-stop" 100 incus stop truenas --timeout 90 || true
  if running truenas; then
    step "truenas-stop-force" 30 incus stop truenas --force || failed=1
  fi
  running truenas && failed=1
  return 0
}

# ------------------------------------------------------------ ashenvale_crypt
ashenvale_mounts() {
  # findmnt names the source /dev/dm-N (bind mounts add "[/subdir]"), so match both spellings.
  local d devs=" "
  for d in /dev/mapper/"$vg"-*; do
    [ -e "$d" ] && devs+="$d $(realpath "$d") "
  done
  findmnt -rn -o TARGET,SOURCE | while read -r t src; do
    case "$devs" in *" ${src%%\[*} "*) echo "$t" ;; esac
  done
}

lock_ashenvale() {
  if [ ! -e "/dev/mapper/$crypt" ]; then
    note "ashenvale" "$crypt not open, nothing to do"
    return 0
  fi

  # Detach every ashenvale_pool volume from running instances (today: phonecam on varian).
  local inst vol
  for vol in $(incus storage volume list "$vg" type=custom -c n -f csv 2> /dev/null); do
    for inst in $(incus storage volume show "$vg" "custom/$vol" 2> /dev/null \
                  | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
      if [ "$mode" = check ]; then note "detach" "would detach $vol from $inst"; continue; fi
      local path
      path="$(incus config device get "$inst" "$vol" path 2> /dev/null)"
      if [ -n "$path" ] && running "$inst"; then
        step "guest-umount $inst:$path" 30 incus exec -T "$inst" -- umount "$path" \
          || step "guest-umount-lazy $inst:$path" 15 incus exec -T "$inst" -- umount -l "$path" || true
      fi
      step "detach $vol from $inst" 60 incus storage volume detach "$vg" "$vol" "$inst" || true
    done
  done

  local m
  if [ "$mode" = check ]; then
    for m in $(ashenvale_mounts); do note "host-umount" "would umount $m"; done
    note "ashenvale" "would close $crypt"
    return 0
  fi

  # Whatever still serves the volume (virtiofsd after a failed detach) keeps it busy.
  for m in $(ashenvale_mounts); do
    pkill -f -- "--shared-dir=$m( |$)" || true
  done
  for m in $(ashenvale_mounts | sort -r); do
    step "host-umount $m" 30 umount "$m" || step "host-umount-lazy $m" 15 umount -l "$m" || true
  done

  step "vg-deactivate" 30 vgchange -an "$vg" || true
  step "luks-close" 30 cryptsetup close "$crypt" || true

  if [ -e "/dev/mapper/$crypt" ]; then
    # Still busy: drop the key now (suspend wipes it from the kernel), then replace the
    # stack with error targets so nothing waits on the suspended device.
    step "luks-suspend" 30 cryptsetup luksSuspend "$crypt" || true
    for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
      step "dm-remove-force $m" 15 dmsetup remove --force "$m" || true
    done
    step "dm-remove-force $crypt" 15 dmsetup remove --force "$crypt" || true
  fi

  if [ -e "/dev/mapper/$crypt" ]; then
    failed=1
    note "ashenvale" "$crypt still present: $(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
  fi
  return 0
}

# Both locks run in parallel so a slow TrueNAS shutdown does not delay ashenvale.
lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
lock_ashenvale
wait

# Subshell failures do not reach this shell; derive them from the recorded steps.
if awk -F '\t' '$1 == "step" && $2 ~ /^truenas-(lock|stop-force|query)$/ && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
  failed=1
fi
running truenas && [ "$mode" = lock ] && failed=1

put "finished	$(date -Is)"
if [ "$failed" -eq 0 ]; then put "state	done"; else put "state	failed"; fi
mv "$tmp" "$status"
exit "$failed"