stormwind-nixfiles/fde-lock/lock.sh

# Puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest.
#   fde-lock lock    lock all three, never prompts, every step time-boxed
#   fde-lock check   dry run: report what lock would act on, change nothing
# Each part first gets a clean attempt (stop containers, unmount, lock/close, shutdown) with a budget of
# $clean_secs. Whatever fails or is still open after that is forced: truenas is killed,
# a store's users are killed and its key is wiped even if the device stays busy.
# duskwood holds the VMs' /mnt/mariadb and /mnt/appdata: before unmounting, the guest stops every
# container that binds them, databases first. Their names go to $state_dir/duskwood-<vm>.containers.
# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).

# writeShellApplication enables errexit; failures are handled per step instead.
set +o errexit

mode="${1:-}"
case "$mode" in
  lock | check) ;;
  *) echo "usage: fde-lock lock|check" >&2; exit 2 ;;
esac

status=/run/fde-lock/status
tmp="$status.tmp"
state_dir=/var/lib/fde-lock
truenas_ip=192.168.0.16
clean_secs=20
failed=0
# incus exec into a VM starts commands with a PATH that misses the NixOS system profile.
gpath=/run/wrappers/bin:/run/current-system/sw/bin
db_re='mariadb|mysql|postgres|redis|(^|[-_])db([-_0-9]|$)'

mkdir -p /run/fde-lock "$state_dir"
: > "$tmp"
put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
put "mode	$mode"
put "state	running"
put "started	$(date -Is)"

# step NAME SECONDS CMD...: run CMD with a hard timeout, no stdin, record result.
step() {
  local name="$1" secs="$2" out rc
  shift 2
  out="$(timeout --kill-after=3 "$secs" "$@" < /dev/null 2>&1)"
  rc=$?
  out="$(printf '%s' "$out" | tr '\t\n' '  ' | cut -c1-300)"
  if [ "$rc" -eq 0 ]; then
    put "step	$name	ok	$out"
  else
    put "step	$name	fail($rc)	$out"
  fi
  echo "$name: rc=$rc $out"
  return "$rc"
}
note() { put "step	$1	$2	$3"; echo "$1: $2 $3"; }

# Clean steps share one deadline per part; once it has passed they are skipped.
left() { local l=$(( $1 - $(date +%s) )); [ "$l" -gt 0 ] || l=0; echo "$l"; }
cstep() {
  local name="$1" t
  t="$(left "$2")"
  shift 2
  if [ "$t" -eq 0 ]; then note "$name" skip "clean budget used up"; return 1; fi
  step "$name" "$t" "$@"
}

# incus queries must not hang the run either.
inc() { timeout --kill-after=3 10 incus "$@" < /dev/null 2> /dev/null; }
running() { [ "$(inc list "^$1\$" -c s -f csv)" = RUNNING ]; }

# truenas counts as gone only when its qemu process is gone, whatever incus says.
truenas_qemu() { pgrep -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; }
truenas_alive() { running truenas || truenas_qemu > /dev/null; }

# ---------------------------------------------------------------- Nethergarde
lock_nethergarde() {
  # Hard NFS mounts from truenas would block forever once it is gone: detach them first.
  # findmnt only reads mountinfo, it does not touch the NFS server.
  local m
  findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do
    case "$src" in
      "$truenas_ip":*)
        if [ "$mode" = check ]; then note "nfs-umount" info "would detach $m"
        else step "nfs-umount $m" 5 umount -f -l "$m" || true; fi ;;
    esac
  done

  if ! truenas_alive; then
    note "truenas" info "not running, keys not in memory"
    return 0
  fi

  if [ "$mode" = check ]; then
    step "truenas-query" 15 incus exec -T truenas -- midclt call pool.dataset.query \
      '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}'
    note "truenas-qemu" info "pid $(truenas_qemu | tr '\n' ' ')"
    return 0
  fi

  # Clean: unload-key zeroes the ZFS key, then a regular shutdown, both within the budget.
  local dl t
  dl=$(( $(date +%s) + clean_secs ))
  cstep "truenas-lock" "$dl" incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \
    '{"force_umount": true}'
  t="$(left "$dl")"
  if [ "$t" -gt 3 ]; then
    step "truenas-stop" "$t" incus stop truenas --timeout $(( t - 3 ))
  else
    note "truenas-stop" skip "clean budget used up"
  fi

  # Forced: whatever is left of the VM goes, its memory (and every key in it) with it.
  if truenas_alive; then step "truenas-stop-force" 10 incus stop truenas --force; fi
  if truenas_qemu > /dev/null; then step "truenas-kill" 5 pkill -9 -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; fi
  sleep 1
  if truenas_qemu > /dev/null; then
    note "truenas" fail "qemu still alive: $(truenas_qemu | tr '\n' ' ')"
  else
    note "truenas" ok "stopped, memory released"
  fi
  return 0
}

# ------------------------------------------------- LUKS stores (ashenvale, duskwood)
# store_mounts VG: host mount points of the VG's volumes. findmnt names the source /dev/dm-N
# (bind mounts add "[/subdir]"), so match both spellings.
store_mounts() {
  local d devs=" "
  for d in /dev/mapper/"$1"-*; do
    [ -e "$d" ] && devs+="$d $(realpath "$d") "
  done
  findmnt -rn -o TARGET,SOURCE | while read -r t src; do
    case "$devs" in *" ${src%%\[*} "*) echo "$t" ;; esac
  done
}

# guest_mount INST VOL: where the guest mounted a block volume (by-id path from the Incus device name).
guest_mount() {
  # shellcheck disable=SC2016 # $1 expands in the guest shell
  timeout --kill-after=3 10 incus exec -T "$1" --env "PATH=$gpath" -- sh -c \
    'd="$(realpath -e "$1")" && findmnt -rn -o TARGET -S "$d"' sh \
    "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_incus_$2-part1" < /dev/null 2> /dev/null | head -n1
}

# guest_containers INST MOUNT...: running containers in the guest that bind anything below MOUNT.
guest_containers() {
  local inst="$1"
  shift
  # shellcheck disable=SC2016 # $@, $s, $n expand in the guest shell
  timeout --kill-after=3 10 incus exec -T "$inst" --env "PATH=$gpath" -- sh -c '
    docker ps -q | xargs -r docker inspect -f "{{.Name}}{{range .Mounts}} {{.Source}}{{end}}" |
    while read -r n srcs; do
      for s in $srcs; do
        for m in "$@"; do
          case "$s/" in "$m"/*) echo "${n#/}"; continue 3 ;; esac
        done
      done
    done' sh "$@" < /dev/null 2> /dev/null
}

# release_instance NAME VG INST DEADLINE QUIESCE "VOL|PATH"...: free the store's volumes in one VM.
# Clean within DEADLINE: stop the containers that use them (QUIESCE=1, databases first), unmount in
# the guest, detach. Forced after: kill those containers, lazy unmount, detach anyway.
# A stopped VM keeps its devices, so it cannot start while the store is locked.
release_instance() {
  local name="$1" vg="$2" inst="$3" dl="$4" quiesce="$5" e vol path s c
  shift 5
  local vols=() mounts=() ctrs=() dbs=() rest=() stuck=()
  if ! running "$inst"; then
    note "$name $inst" info "not running, devices stay attached"
    return 0
  fi
  for e in "$@"; do
    vol="${e%%|*}"
    path="${e#*|}"
    vols+=("$vol")
    [ -n "$path" ] || path="$(guest_mount "$inst" "$vol")"
    [ -n "$path" ] && mounts+=("$path")
  done

  if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ]; then
    mapfile -t ctrs < <(guest_containers "$inst" "${mounts[@]}")
    # Kept for the unlock; an empty list (second lock in a row) must not overwrite a good one.
    if [ "${#ctrs[@]}" -gt 0 ]; then printf '%s\n' "${ctrs[@]}" > "$state_dir/$name-$inst.containers"; fi
    for c in "${ctrs[@]}"; do
      if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
    done
    # Databases first while the budget is fresh; docker itself kills what outlives -t.
    if [ "${#dbs[@]}" -gt 0 ]; then
      s=$(( $(left "$dl") - 4 )); [ "$s" -le 10 ] || s=10
      if [ "$s" -gt 0 ]; then
        cstep "docker-stop-db $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${dbs[@]}"
      else
        note "docker-stop-db $inst" skip "clean budget used up"
      fi
    fi
    if [ "${#rest[@]}" -gt 0 ]; then
      s=$(( $(left "$dl") - 4 )); [ "$s" -le 5 ] || s=5
      if [ "$s" -gt 0 ]; then
        cstep "docker-stop $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${rest[@]}"
      else
        note "docker-stop $inst" skip "clean budget used up"
      fi
    fi
  fi

  for path in "${mounts[@]}"; do
    cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- umount "$path" ||
      stuck+=("$path")
  done

  # Forced: nothing that still holds a mount survives, then the mount is detached from the tree.
  if [ "${#stuck[@]}" -gt 0 ]; then
    if [ "${#ctrs[@]}" -gt 0 ]; then
      step "docker-kill $inst" 10 incus exec -T "$inst" --env "PATH=$gpath" -- docker kill "${ctrs[@]}" || true
    fi
    for path in "${stuck[@]}"; do
      step "guest-umount-lazy $inst:$path" 5 incus exec -T "$inst" --env "PATH=$gpath" -- umount -l "$path"
    done
  fi

  for vol in "${vols[@]}"; do
    cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" ||
      step "detach-force $vol from $inst" 10 incus storage volume detach "$vg" "$vol" "$inst"
  done
  return 0
}

# lock_store NAME CRYPT VG QUIESCE: release every attached volume (all VMs in parallel), then close the LUKS device.
lock_store() {
  local name="$1" crypt="$2" vg="$3" quiesce="$4"
  if [ ! -e "/dev/mapper/$crypt" ]; then
    note "$name" info "$crypt not open, nothing to do"
    return 0
  fi

  # Volumes of the pool attached to instances, grouped per instance as "vol|path" lines.
  local vol inst path
  local -A per=()
  for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do
    for inst in $(inc storage volume show "$vg" "custom/$vol" \
                  | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
      per[$inst]+="$vol|$(inc config device get "$inst" "$vol" path)"$'\n'
    done
  done

  local m es c
  if [ "$mode" = check ]; then
    for inst in "${!per[@]}"; do
      local mounts=() dbs=() rest=()
      while IFS='|' read -r vol path; do
        [ -n "$vol" ] || continue
        if [ -z "$path" ] && running "$inst"; then path="$(guest_mount "$inst" "$vol")"; fi
        [ -n "$path" ] && mounts+=("$path")
        note "$name detach" info "would detach $vol from $inst (${path:-not mounted})"
      done <<< "${per[$inst]}"
      if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ] && running "$inst"; then
        while read -r c; do
          [ -n "$c" ] || continue
          if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
        done < <(guest_containers "$inst" "${mounts[@]}")
        note "$name containers $inst" info "would stop db: ${dbs[*]:-none}; then: ${rest[*]:-none}"
      fi
    done
    for m in $(store_mounts "$vg"); do note "$name host-umount" info "would umount $m"; done
    note "$name" info "would close $crypt"
    return 0
  fi

  # Clean: release the volumes in every VM at once, unmount on the host, close, all within the budget.
  local dl pids=()
  dl=$(( $(date +%s) + clean_secs ))
  for inst in "${!per[@]}"; do
    mapfile -t es < <(printf '%s' "${per[$inst]}")
    release_instance "$name" "$vg" "$inst" "$dl" "$quiesce" "${es[@]}" &
    pids+=($!)
  done
  [ "${#pids[@]}" -eq 0 ] || wait "${pids[@]}"
  for m in $(store_mounts "$vg" | sort -r); do
    cstep "$name host-umount $m" "$dl" umount "$m"
  done
  cstep "$name vg-deactivate" "$dl" vgchange -an "$vg"
  cstep "$name luks-close" "$dl" cryptsetup close "$crypt"

  if [ -e "/dev/mapper/$crypt" ]; then
    # Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close.
    for m in $(store_mounts "$vg"); do
      if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "$name virtiofsd-kill $m" ok ""; fi
    done
    for m in $(store_mounts "$vg" | sort -r); do
      step "$name host-umount-lazy $m" 5 umount -l "$m"
    done
    step "$name vg-deactivate-force" 10 vgchange -an "$vg"
    step "$name luks-close-force" 10 cryptsetup close "$crypt"
  fi

  if [ -e "/dev/mapper/$crypt" ]; then
    # Still busy: wipe the key now (suspend drops it from the kernel), then replace the
    # stack with error targets so nothing waits on the suspended device.
    step "$name luks-suspend" 10 cryptsetup luksSuspend "$crypt"
    for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
      step "$name dm-remove-force $m" 5 dmsetup remove --force "$m"
    done
    step "$name dm-remove-force $crypt" 5 dmsetup remove --force "$crypt"
  fi

  # At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped).
  if [ ! -e "/dev/mapper/$crypt" ]; then
    note "$name" ok "$crypt closed"
  else
    local attr target
    attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
    target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')"
    if [[ "$attr" == *s* ]]; then
      note "$name" ok "$crypt still listed but suspended, key wiped"
    elif [ "$target" = error ]; then
      note "$name" ok "$crypt replaced by error target, key dropped"
    else
      note "$name" fail "$crypt still active: attr $attr, target $target"
    fi
  fi
  return 0
}

# All three run in parallel so a slow TrueNAS shutdown or container stop does not delay the others.
lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
lock_store ashenvale ashenvale_crypt ashenvale_pool 0 &
lock_store duskwood duskwood_crypt duskwood_pool 1 &
wait

# Subshell failures do not reach this shell; derive them from the recorded steps.
if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "duskwood" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
  failed=1
fi

put "finished	$(date -Is)"
if [ "$failed" -eq 0 ]; then put "state	done"; else put "state	failed"; fi
mv "$tmp" "$status"
exit "$failed"