224 lines
8.2 KiB
Bash
224 lines
8.2 KiB
Bash
# Puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
|
|
# fde-lock lock lock both, never prompts, every step time-boxed
|
|
# fde-lock check dry run: report what lock would act on, change nothing
|
|
# Each part first gets a clean attempt (unmount, lock/close, shutdown) with a shared budget of
|
|
# $clean_secs. Whatever fails or is still open after that is forced: truenas is killed,
|
|
# ashenvale's users are killed and its key is wiped even if the device stays busy.
|
|
# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).
|
|
|
|
# writeShellApplication enables errexit; failures are handled per step instead.
|
|
set +o errexit
|
|
|
|
mode="${1:-}"
|
|
case "$mode" in
|
|
lock | check) ;;
|
|
*) echo "usage: fde-lock lock|check" >&2; exit 2 ;;
|
|
esac
|
|
|
|
status=/run/fde-lock/status
|
|
tmp="$status.tmp"
|
|
truenas_ip=192.168.0.16
|
|
crypt=ashenvale_crypt
|
|
vg=ashenvale_pool
|
|
clean_secs=20
|
|
failed=0
|
|
|
|
mkdir -p /run/fde-lock
|
|
: > "$tmp"
|
|
put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
|
|
put "mode $mode"
|
|
put "state running"
|
|
put "started $(date -Is)"
|
|
|
|
# step NAME SECONDS CMD...: run CMD with a hard timeout, no stdin, record result.
|
|
step() {
|
|
local name="$1" secs="$2" out rc
|
|
shift 2
|
|
out="$(timeout --kill-after=3 "$secs" "$@" < /dev/null 2>&1)"
|
|
rc=$?
|
|
out="$(printf '%s' "$out" | tr '\t\n' ' ' | cut -c1-300)"
|
|
if [ "$rc" -eq 0 ]; then
|
|
put "step $name ok $out"
|
|
else
|
|
put "step $name fail($rc) $out"
|
|
fi
|
|
echo "$name: rc=$rc $out"
|
|
return "$rc"
|
|
}
|
|
note() { put "step $1 $2 $3"; echo "$1: $2 $3"; }
|
|
|
|
# Clean steps share one deadline per part; once it has passed they are skipped.
|
|
left() { local l=$(( $1 - $(date +%s) )); [ "$l" -gt 0 ] || l=0; echo "$l"; }
|
|
cstep() {
|
|
local name="$1" t
|
|
t="$(left "$2")"
|
|
shift 2
|
|
if [ "$t" -eq 0 ]; then note "$name" skip "clean budget used up"; return 1; fi
|
|
step "$name" "$t" "$@"
|
|
}
|
|
|
|
# incus queries must not hang the run either.
|
|
inc() { timeout --kill-after=3 10 incus "$@" < /dev/null 2> /dev/null; }
|
|
running() { [ "$(inc list "^$1\$" -c s -f csv)" = RUNNING ]; }
|
|
|
|
# truenas counts as gone only when its qemu process is gone, whatever incus says.
|
|
truenas_qemu() { pgrep -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; }
|
|
truenas_alive() { running truenas || truenas_qemu > /dev/null; }
|
|
|
|
# ---------------------------------------------------------------- Nethergarde
|
|
lock_nethergarde() {
|
|
# Hard NFS mounts from truenas would block forever once it is gone: detach them first.
|
|
# findmnt only reads mountinfo, it does not touch the NFS server.
|
|
local m
|
|
findmnt -rn -t nfs,nfs4 -o TARGET,SOURCE | while read -r m src; do
|
|
case "$src" in
|
|
"$truenas_ip":*)
|
|
if [ "$mode" = check ]; then note "nfs-umount" info "would detach $m"
|
|
else step "nfs-umount $m" 5 umount -f -l "$m" || true; fi ;;
|
|
esac
|
|
done
|
|
|
|
if ! truenas_alive; then
|
|
note "truenas" info "not running, keys not in memory"
|
|
return 0
|
|
fi
|
|
|
|
if [ "$mode" = check ]; then
|
|
step "truenas-query" 15 incus exec -T truenas -- midclt call pool.dataset.query \
|
|
'[["id","=","Nethergarde"]]' '{"select":["id","encrypted","key_format","locked"]}'
|
|
note "truenas-qemu" info "pid $(truenas_qemu | tr '\n' ' ')"
|
|
return 0
|
|
fi
|
|
|
|
# Clean: unload-key zeroes the ZFS key, then a regular shutdown, both within the budget.
|
|
local dl t
|
|
dl=$(( $(date +%s) + clean_secs ))
|
|
cstep "truenas-lock" "$dl" incus exec -T truenas -- midclt call -j pool.dataset.lock Nethergarde \
|
|
'{"force_umount": true}'
|
|
t="$(left "$dl")"
|
|
if [ "$t" -gt 3 ]; then
|
|
step "truenas-stop" "$t" incus stop truenas --timeout $(( t - 3 ))
|
|
else
|
|
note "truenas-stop" skip "clean budget used up"
|
|
fi
|
|
|
|
# Forced: whatever is left of the VM goes, its memory (and every key in it) with it.
|
|
if truenas_alive; then step "truenas-stop-force" 10 incus stop truenas --force; fi
|
|
if truenas_qemu > /dev/null; then step "truenas-kill" 5 pkill -9 -f -- "qemu-system[^ ]* .*-name truenas( |,|$)"; fi
|
|
sleep 1
|
|
if truenas_qemu > /dev/null; then
|
|
note "truenas" fail "qemu still alive: $(truenas_qemu | tr '\n' ' ')"
|
|
else
|
|
note "truenas" ok "stopped, memory released"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ------------------------------------------------------------ ashenvale_crypt
|
|
ashenvale_mounts() {
|
|
# findmnt names the source /dev/dm-N (bind mounts add "[/subdir]"), so match both spellings.
|
|
local d devs=" "
|
|
for d in /dev/mapper/"$vg"-*; do
|
|
[ -e "$d" ] && devs+="$d $(realpath "$d") "
|
|
done
|
|
findmnt -rn -o TARGET,SOURCE | while read -r t src; do
|
|
case "$devs" in *" ${src%%\[*} "*) echo "$t" ;; esac
|
|
done
|
|
}
|
|
|
|
lock_ashenvale() {
|
|
if [ ! -e "/dev/mapper/$crypt" ]; then
|
|
note "ashenvale" info "$crypt not open, nothing to do"
|
|
return 0
|
|
fi
|
|
|
|
# Volumes of the pool attached to instances (today: phonecam on varian), as "vol inst path".
|
|
local vol inst path attached=""
|
|
for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do
|
|
for inst in $(inc storage volume show "$vg" "custom/$vol" \
|
|
| sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
|
|
attached+="$vol $inst $(inc config device get "$inst" "$vol" path)"$'\n'
|
|
done
|
|
done
|
|
|
|
local m
|
|
if [ "$mode" = check ]; then
|
|
while read -r vol inst path; do
|
|
[ -n "$vol" ] && note "detach" info "would detach $vol from $inst ($path)"
|
|
done <<< "$attached"
|
|
for m in $(ashenvale_mounts); do note "host-umount" info "would umount $m"; done
|
|
note "ashenvale" info "would close $crypt"
|
|
return 0
|
|
fi
|
|
|
|
# Clean: unmount in the guest, detach, unmount on the host, close, all within the budget.
|
|
local dl
|
|
dl=$(( $(date +%s) + clean_secs ))
|
|
while read -r vol inst path; do
|
|
[ -n "$vol" ] || continue
|
|
if [ -n "$path" ] && running "$inst"; then
|
|
cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" -- umount "$path"
|
|
fi
|
|
cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst"
|
|
done <<< "$attached"
|
|
for m in $(ashenvale_mounts | sort -r); do
|
|
cstep "host-umount $m" "$dl" umount "$m"
|
|
done
|
|
cstep "vg-deactivate" "$dl" vgchange -an "$vg"
|
|
cstep "luks-close" "$dl" cryptsetup close "$crypt"
|
|
|
|
if [ -e "/dev/mapper/$crypt" ]; then
|
|
# Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close.
|
|
for m in $(ashenvale_mounts); do
|
|
if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "virtiofsd-kill $m" ok ""; fi
|
|
done
|
|
for m in $(ashenvale_mounts | sort -r); do
|
|
step "host-umount-lazy $m" 5 umount -l "$m"
|
|
done
|
|
step "vg-deactivate-force" 10 vgchange -an "$vg"
|
|
step "luks-close-force" 10 cryptsetup close "$crypt"
|
|
fi
|
|
|
|
if [ -e "/dev/mapper/$crypt" ]; then
|
|
# Still busy: wipe the key now (suspend drops it from the kernel), then replace the
|
|
# stack with error targets so nothing waits on the suspended device.
|
|
step "luks-suspend" 10 cryptsetup luksSuspend "$crypt"
|
|
for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
|
|
step "dm-remove-force $m" 5 dmsetup remove --force "$m"
|
|
done
|
|
step "dm-remove-force $crypt" 5 dmsetup remove --force "$crypt"
|
|
fi
|
|
|
|
# At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped).
|
|
if [ ! -e "/dev/mapper/$crypt" ]; then
|
|
note "ashenvale" ok "$crypt closed"
|
|
else
|
|
local attr target
|
|
attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
|
|
target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')"
|
|
if [[ "$attr" == *s* ]]; then
|
|
note "ashenvale" ok "$crypt still listed but suspended, key wiped"
|
|
elif [ "$target" = error ]; then
|
|
note "ashenvale" ok "$crypt replaced by error target, key dropped"
|
|
else
|
|
note "ashenvale" fail "$crypt still active: attr $attr, target $target"
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Both locks run in parallel so a slow TrueNAS shutdown does not delay ashenvale.
|
|
lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
|
|
lock_ashenvale
|
|
wait
|
|
|
|
# Subshell failures do not reach this shell; derive them from the recorded steps.
|
|
if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
|
|
failed=1
|
|
fi
|
|
|
|
put "finished $(date -Is)"
|
|
if [ "$failed" -eq 0 ]; then put "state done"; else put "state failed"; fi
|
|
mv "$tmp" "$status"
|
|
exit "$failed"
|