From 487bb678ef93c444573f449e0174c945dfebbee4 Mon Sep 17 00:00:00 2001 From: RootHost-Stormwind Date: Mon, 2 Nov 2026 02:25:03 +0100 Subject: [PATCH] feat: fde-lock also locks duskwood, stopping its containers first --- fde-lock.nix | 4 +- fde-lock/api.py | 2 +- fde-lock/lock.sh | 213 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 165 insertions(+), 54 deletions(-) diff --git a/fde-lock.nix b/fde-lock.nix index af88d87..ad0bcd8 100644 --- a/fde-lock.nix +++ b/fde-lock.nix @@ -1,6 +1,6 @@ { config, pkgs, ... }: -# Home Assistant endpoint that puts Nethergarde (truenas VM) and ashenvale_crypt to rest. +# Home Assistant endpoint that puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest. # HTTP API (unprivileged) -> trigger file in /run/fde-lock-api -> path unit -> root oneshot. # Token: /etc/nixos/secrets/fde-lock-token (not versioned). @@ -20,7 +20,7 @@ let }; lockService = mode: { - description = "Put Nethergarde and ashenvale_crypt to rest (${mode})"; + description = "Put Nethergarde, ashenvale_crypt and duskwood_crypt to rest (${mode})"; # Never (re)started by a rebuild, only by its path unit. restartIfChanged = false; serviceConfig = { diff --git a/fde-lock/api.py b/fde-lock/api.py index 7b4f4cb..fd9bf16 100644 --- a/fde-lock/api.py +++ b/fde-lock/api.py @@ -1,5 +1,5 @@ # HTTP trigger for fde-lock, called by Home Assistant. -# POST /lock lock Nethergarde and ashenvale_crypt (202, runs in the background) +# POST /lock lock Nethergarde, ashenvale_crypt and duskwood_crypt (202, runs in the background) # POST /check dry run, changes nothing # GET /status result of the last run # Needs "Authorization: Bearer " and a source address from FDE_LOCK_ALLOWED. diff --git a/fde-lock/lock.sh b/fde-lock/lock.sh index 2d1cab7..4b0174c 100644 --- a/fde-lock/lock.sh +++ b/fde-lock/lock.sh @@ -1,9 +1,11 @@ -# Puts Nethergarde (truenas VM) and ashenvale_crypt to rest. -# fde-lock lock lock both, never prompts, every step time-boxed +# Puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest. +# fde-lock lock lock all three, never prompts, every step time-boxed # fde-lock check dry run: report what lock would act on, change nothing -# Each part first gets a clean attempt (unmount, lock/close, shutdown) with a shared budget of +# Each part first gets a clean attempt (stop containers, unmount, lock/close, shutdown) with a budget of # $clean_secs. Whatever fails or is still open after that is forced: truenas is killed, -# ashenvale's users are killed and its key is wiped even if the device stays busy. +# a store's users are killed and its key is wiped even if the device stays busy. +# duskwood holds the VMs' /mnt/mariadb and /mnt/appdata: before unmounting, the guest stops every +# container that binds them, databases first. Their names go to $state_dir/duskwood-.containers. # Status goes to /run/fde-lock/status (one "keyvalue" or "stepnameresultdetail" per line). # writeShellApplication enables errexit; failures are handled per step instead. @@ -17,13 +19,15 @@ esac status=/run/fde-lock/status tmp="$status.tmp" +state_dir=/var/lib/fde-lock truenas_ip=192.168.0.16 -crypt=ashenvale_crypt -vg=ashenvale_pool clean_secs=20 failed=0 +# incus exec into a VM starts commands with a PATH that misses the NixOS system profile. +gpath=/run/wrappers/bin:/run/current-system/sw/bin +db_re='mariadb|mysql|postgres|redis|(^|[-_])db([-_0-9]|$)' -mkdir -p /run/fde-lock +mkdir -p /run/fde-lock "$state_dir" : > "$tmp" put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; } put "mode $mode" @@ -114,11 +118,12 @@ lock_nethergarde() { return 0 } -# ------------------------------------------------------------ ashenvale_crypt -ashenvale_mounts() { - # findmnt names the source /dev/dm-N (bind mounts add "[/subdir]"), so match both spellings. +# ------------------------------------------------- LUKS stores (ashenvale, duskwood) +# store_mounts VG: host mount points of the VG's volumes. findmnt names the source /dev/dm-N +# (bind mounts add "[/subdir]"), so match both spellings. +store_mounts() { local d devs=" " - for d in /dev/mapper/"$vg"-*; do + for d in /dev/mapper/"$1"-*; do [ -e "$d" ] && devs+="$d $(realpath "$d") " done findmnt -rn -o TARGET,SOURCE | while read -r t src; do @@ -126,94 +131,200 @@ ashenvale_mounts() { done } -lock_ashenvale() { +# guest_mount INST VOL: where the guest mounted a block volume (by-id path from the Incus device name). +guest_mount() { + timeout --kill-after=3 10 incus exec -T "$1" --env "PATH=$gpath" -- sh -c \ + 'd="$(realpath -e "$1")" && findmnt -rn -o TARGET -S "$d"' sh \ + "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_incus_$2-part1" < /dev/null 2> /dev/null | head -n1 +} + +# guest_containers INST MOUNT...: running containers in the guest that bind anything below MOUNT. +guest_containers() { + local inst="$1" + shift + timeout --kill-after=3 10 incus exec -T "$inst" --env "PATH=$gpath" -- sh -c ' + docker ps -q | xargs -r docker inspect -f "{{.Name}}{{range .Mounts}} {{.Source}}{{end}}" | + while read -r n srcs; do + for s in $srcs; do + for m in "$@"; do + case "$s/" in "$m"/*) echo "${n#/}"; continue 3 ;; esac + done + done + done' sh "$@" < /dev/null 2> /dev/null +} + +# release_instance NAME VG INST DEADLINE QUIESCE "VOL|PATH"...: free the store's volumes in one VM. +# Clean within DEADLINE: stop the containers that use them (QUIESCE=1, databases first), unmount in +# the guest, detach. Forced after: kill those containers, lazy unmount, detach anyway. +# A stopped VM keeps its devices, so it cannot start while the store is locked. +release_instance() { + local name="$1" vg="$2" inst="$3" dl="$4" quiesce="$5" e vol path s c + shift 5 + local vols=() mounts=() ctrs=() dbs=() rest=() stuck=() + if ! running "$inst"; then + note "$name $inst" info "not running, devices stay attached" + return 0 + fi + for e in "$@"; do + vol="${e%%|*}" + path="${e#*|}" + vols+=("$vol") + [ -n "$path" ] || path="$(guest_mount "$inst" "$vol")" + [ -n "$path" ] && mounts+=("$path") + done + + if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ]; then + mapfile -t ctrs < <(guest_containers "$inst" "${mounts[@]}") + # Kept for the unlock; an empty list (second lock in a row) must not overwrite a good one. + if [ "${#ctrs[@]}" -gt 0 ]; then printf '%s\n' "${ctrs[@]}" > "$state_dir/$name-$inst.containers"; fi + for c in "${ctrs[@]}"; do + if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi + done + # Databases first while the budget is fresh; docker itself kills what outlives -t. + if [ "${#dbs[@]}" -gt 0 ]; then + s=$(( $(left "$dl") - 4 )); [ "$s" -le 10 ] || s=10 + if [ "$s" -gt 0 ]; then + cstep "docker-stop-db $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${dbs[@]}" + else + note "docker-stop-db $inst" skip "clean budget used up" + fi + fi + if [ "${#rest[@]}" -gt 0 ]; then + s=$(( $(left "$dl") - 4 )); [ "$s" -le 5 ] || s=5 + if [ "$s" -gt 0 ]; then + cstep "docker-stop $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${rest[@]}" + else + note "docker-stop $inst" skip "clean budget used up" + fi + fi + fi + + for path in "${mounts[@]}"; do + cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- umount "$path" || + stuck+=("$path") + done + + # Forced: nothing that still holds a mount survives, then the mount is detached from the tree. + if [ "${#stuck[@]}" -gt 0 ]; then + if [ "${#ctrs[@]}" -gt 0 ]; then + step "docker-kill $inst" 10 incus exec -T "$inst" --env "PATH=$gpath" -- docker kill "${ctrs[@]}" || true + fi + for path in "${stuck[@]}"; do + step "guest-umount-lazy $inst:$path" 5 incus exec -T "$inst" --env "PATH=$gpath" -- umount -l "$path" + done + fi + + for vol in "${vols[@]}"; do + cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" || + step "detach-force $vol from $inst" 10 incus storage volume detach "$vg" "$vol" "$inst" + done + return 0 +} + +# lock_store NAME CRYPT VG QUIESCE: release every attached volume (all VMs in parallel), then close the LUKS device. +lock_store() { + local name="$1" crypt="$2" vg="$3" quiesce="$4" if [ ! -e "/dev/mapper/$crypt" ]; then - note "ashenvale" info "$crypt not open, nothing to do" + note "$name" info "$crypt not open, nothing to do" return 0 fi - # Volumes of the pool attached to instances (today: phonecam on varian), as "vol inst path". - local vol inst path attached="" + # Volumes of the pool attached to instances, grouped per instance as "vol|path" lines. + local vol inst path + local -A per=() for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do for inst in $(inc storage volume show "$vg" "custom/$vol" \ | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do - attached+="$vol $inst $(inc config device get "$inst" "$vol" path)"$'\n' + per[$inst]+="$vol|$(inc config device get "$inst" "$vol" path)"$'\n' done done - local m + local m es c if [ "$mode" = check ]; then - while read -r vol inst path; do - [ -n "$vol" ] && note "detach" info "would detach $vol from $inst ($path)" - done <<< "$attached" - for m in $(ashenvale_mounts); do note "host-umount" info "would umount $m"; done - note "ashenvale" info "would close $crypt" + for inst in "${!per[@]}"; do + local mounts=() dbs=() rest=() + while IFS='|' read -r vol path; do + [ -n "$vol" ] || continue + if [ -z "$path" ] && running "$inst"; then path="$(guest_mount "$inst" "$vol")"; fi + [ -n "$path" ] && mounts+=("$path") + note "$name detach" info "would detach $vol from $inst (${path:-not mounted})" + done <<< "${per[$inst]}" + if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ] && running "$inst"; then + while read -r c; do + [ -n "$c" ] || continue + if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi + done < <(guest_containers "$inst" "${mounts[@]}") + note "$name containers $inst" info "would stop db: ${dbs[*]:-none}; then: ${rest[*]:-none}" + fi + done + for m in $(store_mounts "$vg"); do note "$name host-umount" info "would umount $m"; done + note "$name" info "would close $crypt" return 0 fi - # Clean: unmount in the guest, detach, unmount on the host, close, all within the budget. - local dl + # Clean: release the volumes in every VM at once, unmount on the host, close, all within the budget. + local dl pids=() dl=$(( $(date +%s) + clean_secs )) - while read -r vol inst path; do - [ -n "$vol" ] || continue - if [ -n "$path" ] && running "$inst"; then - cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" -- umount "$path" - fi - cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" - done <<< "$attached" - for m in $(ashenvale_mounts | sort -r); do - cstep "host-umount $m" "$dl" umount "$m" + for inst in "${!per[@]}"; do + mapfile -t es < <(printf '%s' "${per[$inst]}") + release_instance "$name" "$vg" "$inst" "$dl" "$quiesce" "${es[@]}" & + pids+=($!) done - cstep "vg-deactivate" "$dl" vgchange -an "$vg" - cstep "luks-close" "$dl" cryptsetup close "$crypt" + [ "${#pids[@]}" -eq 0 ] || wait "${pids[@]}" + for m in $(store_mounts "$vg" | sort -r); do + cstep "$name host-umount $m" "$dl" umount "$m" + done + cstep "$name vg-deactivate" "$dl" vgchange -an "$vg" + cstep "$name luks-close" "$dl" cryptsetup close "$crypt" if [ -e "/dev/mapper/$crypt" ]; then # Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close. - for m in $(ashenvale_mounts); do - if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "virtiofsd-kill $m" ok ""; fi + for m in $(store_mounts "$vg"); do + if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "$name virtiofsd-kill $m" ok ""; fi done - for m in $(ashenvale_mounts | sort -r); do - step "host-umount-lazy $m" 5 umount -l "$m" + for m in $(store_mounts "$vg" | sort -r); do + step "$name host-umount-lazy $m" 5 umount -l "$m" done - step "vg-deactivate-force" 10 vgchange -an "$vg" - step "luks-close-force" 10 cryptsetup close "$crypt" + step "$name vg-deactivate-force" 10 vgchange -an "$vg" + step "$name luks-close-force" 10 cryptsetup close "$crypt" fi if [ -e "/dev/mapper/$crypt" ]; then # Still busy: wipe the key now (suspend drops it from the kernel), then replace the # stack with error targets so nothing waits on the suspended device. - step "luks-suspend" 10 cryptsetup luksSuspend "$crypt" + step "$name luks-suspend" 10 cryptsetup luksSuspend "$crypt" for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do - step "dm-remove-force $m" 5 dmsetup remove --force "$m" + step "$name dm-remove-force $m" 5 dmsetup remove --force "$m" done - step "dm-remove-force $crypt" 5 dmsetup remove --force "$crypt" + step "$name dm-remove-force $crypt" 5 dmsetup remove --force "$crypt" fi # At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped). if [ ! -e "/dev/mapper/$crypt" ]; then - note "ashenvale" ok "$crypt closed" + note "$name" ok "$crypt closed" else local attr target attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)" target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')" if [[ "$attr" == *s* ]]; then - note "ashenvale" ok "$crypt still listed but suspended, key wiped" + note "$name" ok "$crypt still listed but suspended, key wiped" elif [ "$target" = error ]; then - note "ashenvale" ok "$crypt replaced by error target, key dropped" + note "$name" ok "$crypt replaced by error target, key dropped" else - note "ashenvale" fail "$crypt still active: attr $attr, target $target" + note "$name" fail "$crypt still active: attr $attr, target $target" fi fi return 0 } -# Both locks run in parallel so a slow TrueNAS shutdown does not delay ashenvale. +# All three run in parallel so a slow TrueNAS shutdown or container stop does not delay the others. lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 & -lock_ashenvale +lock_store ashenvale ashenvale_crypt ashenvale_pool 0 & +lock_store duskwood duskwood_crypt duskwood_pool 1 & wait # Subshell failures do not reach this shell; derive them from the recorded steps. -if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then +if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "duskwood" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then failed=1 fi