From 1e4bec6bfe7a9ef7528d8b64656033bffaebaa84 Mon Sep 17 00:00:00 2001 From: RootHost-Stormwind Date: Sun, 1 Nov 2026 18:05:38 +0100 Subject: [PATCH] fix: fde-lock tries clean for 20 s, then forces truenas kill and luks wipe --- fde-lock.nix | 2 +- fde-lock/lock.sh | 157 +++++++++++++++++++++++++++++++---------------- 2 files changed, 106 insertions(+), 53 deletions(-) diff --git a/fde-lock.nix b/fde-lock.nix index cc3c54d..af88d87 100644 --- a/fde-lock.nix +++ b/fde-lock.nix @@ -27,7 +27,7 @@ let Type = "oneshot"; ExecStart = "${fde-lock}/bin/fde-lock ${mode}"; StandardInput = "null"; - TimeoutStartSec = "6min"; + TimeoutStartSec = "4min"; }; }; diff --git a/fde-lock/lock.sh b/fde-lock/lock.sh index 8c83738..2d1cab7 100644 --- a/fde-lock/lock.sh +++ b/fde-lock/lock.sh @@ -1,6 +1,9 @@ # Puts Nethergarde (truenas VM) and ashenvale_crypt to rest. # fde-lock lock lock both, never prompts, every step time-boxed # fde-lock check dry run: report what lock would act on, change nothing +# Each part first gets a clean attempt (unmount, lock/close, shutdown) with a shared budget of +# $clean_secs. Whatever fails or is still open after that is forced: truenas is killed, +# ashenvale's users are killed and its key is wiped even if the device stays busy. # Status goes to /run/fde-lock/status (one "keyvalue" or "stepnameresultdetail" per line). # writeShellApplication enables errexit; failures are handled per step instead. @@ -17,6 +20,7 @@ tmp="$status.tmp" truenas_ip=192.168.0.16 crypt=ashenvale_crypt vg=ashenvale_pool +clean_secs=20 failed=0 mkdir -p /run/fde-lock @@ -30,7 +34,7 @@ put "started $(date -Is)" step() { local name="$1" secs="$2" out rc shift 2 - out="$(timeout --kill-after=5 "$secs" "$@" < /dev/null 2>&1)" + out="$(timeout --kill-after=3 "$secs" "$@" < /dev/null 2>&1)" rc=$? out="$(printf '%s' "$out" | tr '\t\n' ' ' | cut -c1-300)" if [ "$rc" -eq 0 ]; then @@ -41,9 +45,25 @@ step() { echo "$name: rc=$rc $out" return "$rc" } -note() { put "step $1 info $2"; echo "$1: $2"; } +note() { put "step $1 $2 $3"; echo "$1: $2 $3"; } -running() { [ "$(incus list "^$1\$" -c s -f csv 2> /dev/null)" = RUNNING ]; } +# Clean steps share one deadline per part; once it has passed they are skipped. +left() { local l=$(( $1 - $(date +%s) )); [ "$l" -gt 0 ] || l=0; echo "$l"; } +cstep() { + local name="$1" t + t="$(left "$2")" + shift 2 + if [ "$t" -eq 0 ]; then note "$name" skip "clean budget used up"; return 1; fi + step "$name" "$t" "$@" +} + +# incus queries must not hang the run either. +inc() { timeout --kill-after=3 10 incus "$@" < /dev/null 2> /dev/null; } +running() { [ "$(inc list "^$1\$" -c s -f csv)" = RUNNING ]; } + +# truenas counts as gone only when its qemu process is gone, whatever incus says. +truenas_qemu() { pgrep -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; } +truenas_alive() { running truenas || truenas_qemu > /dev/null; } # ---------------------------------------------------------------- Nethergarde lock_nethergarde() { @@ -53,34 +73,44 @@ lock_nethergarde() { findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do case "$src" in "$truenas_ip":*) - if [ "$mode" = check ]; then note "nfs-umount" "would detach $m" - else step "nfs-umount $m" 15 umount -f -l "$m" || true; fi ;; + if [ "$mode" = check ]; then note "nfs-umount" info "would detach $m" + else step "nfs-umount $m" 5 umount -f -l "$m" || true; fi ;; esac done - if ! running truenas; then - note "truenas" "not running, keys not in memory" + if ! truenas_alive; then + note "truenas" info "not running, keys not in memory" return 0 fi if [ "$mode" = check ]; then - step "truenas-query" 30 incus exec -T truenas -- midclt call pool.dataset.query \ - '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}' || failed=1 + step "truenas-query" 15 incus exec -T truenas -- midclt call pool.dataset.query \ + '[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}' + note "truenas-qemu" info "pid $(truenas_qemu | tr '\n' ' ')" return 0 fi - # Unlocked ZFS keys are zeroed by unload-key; force_umount stops the shares that hold the pool. - step "truenas-lock" 120 incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \ - '{"force_umount": true}' || failed=1 - step "truenas-locked?" 30 incus exec -T truenas -- midclt call pool.dataset.query \ - '[["id","=","Nethergarde"]]' '{"select":["id","locked"]}' || true + # Clean: unload-key zeroes the ZFS key, then a regular shutdown, both within the budget. + local dl t + dl=$(( $(date +%s) + clean_secs )) + cstep "truenas-lock" "$dl" incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \ + '{"force_umount": true}' + t="$(left "$dl")" + if [ "$t" -gt 3 ]; then + step "truenas-stop" "$t" incus stop truenas --timeout $(( t - 3 )) + else + note "truenas-stop" skip "clean budget used up" + fi - # Stop the VM in any case, so no key can stay in its memory (init_on_free zeroes it on the host). - step "truenas-stop" 100 incus stop truenas --timeout 90 || true - if running truenas; then - step "truenas-stop-force" 30 incus stop truenas --force || failed=1 + # Forced: whatever is left of the VM goes, its memory (and every key in it) with it. + if truenas_alive; then step "truenas-stop-force" 10 incus stop truenas --force; fi + if truenas_qemu > /dev/null; then step "truenas-kill" 5 pkill -9 -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; fi + sleep 1 + if truenas_qemu > /dev/null; then + note "truenas" fail "qemu still alive: $(truenas_qemu | tr '\n' ' ')" + else + note "truenas" ok "stopped, memory released" fi - running truenas && failed=1 return 0 } @@ -98,57 +128,81 @@ ashenvale_mounts() { lock_ashenvale() { if [ ! -e "/dev/mapper/$crypt" ]; then - note "ashenvale" "$crypt not open, nothing to do" + note "ashenvale" info "$crypt not open, nothing to do" return 0 fi - # Detach every ashenvale_pool volume from running instances (today: phonecam on varian). - local inst vol - for vol in $(incus storage volume list "$vg" type=custom -c n -f csv 2> /dev/null); do - for inst in $(incus storage volume show "$vg" "custom/$vol" 2> /dev/null \ + # Volumes of the pool attached to instances (today: phonecam on varian), as "vol inst path". + local vol inst path attached="" + for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do + for inst in $(inc storage volume show "$vg" "custom/$vol" \ | sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do - if [ "$mode" = check ]; then note "detach" "would detach $vol from $inst"; continue; fi - local path - path="$(incus config device get "$inst" "$vol" path 2> /dev/null)" - if [ -n "$path" ] && running "$inst"; then - step "guest-umount $inst:$path" 30 incus exec -T "$inst" -- umount "$path" \ - || step "guest-umount-lazy $inst:$path" 15 incus exec -T "$inst" -- umount -l "$path" || true - fi - step "detach $vol from $inst" 60 incus storage volume detach "$vg" "$vol" "$inst" || true + attached+="$vol $inst $(inc config device get "$inst" "$vol" path)"$'\n' done done local m if [ "$mode" = check ]; then - for m in $(ashenvale_mounts); do note "host-umount" "would umount $m"; done - note "ashenvale" "would close $crypt" + while read -r vol inst path; do + [ -n "$vol" ] && note "detach" info "would detach $vol from $inst ($path)" + done <<< "$attached" + for m in $(ashenvale_mounts); do note "host-umount" info "would umount $m"; done + note "ashenvale" info "would close $crypt" return 0 fi - # Whatever still serves the volume (virtiofsd after a failed detach) keeps it busy. - for m in $(ashenvale_mounts); do - pkill -f -- "--shared-dir=$m( |$)" || true - done + # Clean: unmount in the guest, detach, unmount on the host, close, all within the budget. + local dl + dl=$(( $(date +%s) + clean_secs )) + while read -r vol inst path; do + [ -n "$vol" ] || continue + if [ -n "$path" ] && running "$inst"; then + cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" -- umount "$path" + fi + cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" + done <<< "$attached" for m in $(ashenvale_mounts | sort -r); do - step "host-umount $m" 30 umount "$m" || step "host-umount-lazy $m" 15 umount -l "$m" || true + cstep "host-umount $m" "$dl" umount "$m" done - - step "vg-deactivate" 30 vgchange -an "$vg" || true - step "luks-close" 30 cryptsetup close "$crypt" || true + cstep "vg-deactivate" "$dl" vgchange -an "$vg" + cstep "luks-close" "$dl" cryptsetup close "$crypt" if [ -e "/dev/mapper/$crypt" ]; then - # Still busy: drop the key now (suspend wipes it from the kernel), then replace the - # stack with error targets so nothing waits on the suspended device. - step "luks-suspend" 30 cryptsetup luksSuspend "$crypt" || true - for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do - step "dm-remove-force $m" 15 dmsetup remove --force "$m" || true + # Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close. + for m in $(ashenvale_mounts); do + if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "virtiofsd-kill $m" ok ""; fi done - step "dm-remove-force $crypt" 15 dmsetup remove --force "$crypt" || true + for m in $(ashenvale_mounts | sort -r); do + step "host-umount-lazy $m" 5 umount -l "$m" + done + step "vg-deactivate-force" 10 vgchange -an "$vg" + step "luks-close-force" 10 cryptsetup close "$crypt" fi if [ -e "/dev/mapper/$crypt" ]; then - failed=1 - note "ashenvale" "$crypt still present: $(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)" + # Still busy: wipe the key now (suspend drops it from the kernel), then replace the + # stack with error targets so nothing waits on the suspended device. + step "luks-suspend" 10 cryptsetup luksSuspend "$crypt" + for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do + step "dm-remove-force $m" 5 dmsetup remove --force "$m" + done + step "dm-remove-force $crypt" 5 dmsetup remove --force "$crypt" + fi + + # At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped). + if [ ! -e "/dev/mapper/$crypt" ]; then + note "ashenvale" ok "$crypt closed" + else + local attr target + attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)" + target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')" + if [[ "$attr" == *s* ]]; then + note "ashenvale" ok "$crypt still listed but suspended, key wiped" + elif [ "$target" = error ]; then + note "ashenvale" ok "$crypt replaced by error target, key dropped" + else + note "ashenvale" fail "$crypt still active: attr $attr, target $target" + fi fi return 0 } @@ -159,10 +213,9 @@ lock_ashenvale wait # Subshell failures do not reach this shell; derive them from the recorded steps. -if awk -F '\t' '$1 == "step" && $2 ~ /^truenas-(lock|stop-force|query)$/ && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then +if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then failed=1 fi -running truenas && [ "$mode" = lock ] && failed=1 put "finished $(date -Is)" if [ "$failed" -eq 0 ]; then put "state done"; else put "state failed"; fi