feat: fde-lock also locks duskwood, stopping its containers first
This commit is contained in:
RootHost-Stormwind committed
1 parent
9890583d8a
commit
487bb678ef
3 files changed
+165
-54
No files matched your search
+2
-2
@@ -1,6 +1,6 @@
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
# Home Assistant endpoint that puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
|
||||
# Home Assistant endpoint that puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest.
|
||||
# HTTP API (unprivileged) -> trigger file in /run/fde-lock-api -> path unit -> root oneshot.
|
||||
# Token: /etc/nixos/secrets/fde-lock-token (not versioned).
|
||||
|
||||
@@ -20,7 +20,7 @@ let
|
||||
};
|
||||
|
||||
lockService = mode: {
|
||||
description = "Put Nethergarde and ashenvale_crypt to rest (${mode})";
|
||||
description = "Put Nethergarde, ashenvale_crypt and duskwood_crypt to rest (${mode})";
|
||||
# Never (re)started by a rebuild, only by its path unit.
|
||||
restartIfChanged = false;
|
||||
serviceConfig = {
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# HTTP trigger for fde-lock, called by Home Assistant.
|
||||
# POST /lock lock Nethergarde and ashenvale_crypt (202, runs in the background)
|
||||
# POST /lock lock Nethergarde, ashenvale_crypt and duskwood_crypt (202, runs in the background)
|
||||
# POST /check dry run, changes nothing
|
||||
# GET /status result of the last run
|
||||
# Needs "Authorization: Bearer <token>" and a source address from FDE_LOCK_ALLOWED.
|
||||
|
||||
+162
-51
@@ -1,9 +1,11 @@
|
||||
# Puts Nethergarde (truenas VM) and ashenvale_crypt to rest.
|
||||
# fde-lock lock lock both, never prompts, every step time-boxed
|
||||
# Puts Nethergarde (truenas VM), ashenvale_crypt and duskwood_crypt to rest.
|
||||
# fde-lock lock lock all three, never prompts, every step time-boxed
|
||||
# fde-lock check dry run: report what lock would act on, change nothing
|
||||
# Each part first gets a clean attempt (unmount, lock/close, shutdown) with a shared budget of
|
||||
# Each part first gets a clean attempt (stop containers, unmount, lock/close, shutdown) with a budget of
|
||||
# $clean_secs. Whatever fails or is still open after that is forced: truenas is killed,
|
||||
# ashenvale's users are killed and its key is wiped even if the device stays busy.
|
||||
# a store's users are killed and its key is wiped even if the device stays busy.
|
||||
# duskwood holds the VMs' /mnt/mariadb and /mnt/appdata: before unmounting, the guest stops every
|
||||
# container that binds them, databases first. Their names go to $state_dir/duskwood-<vm>.containers.
|
||||
# Status goes to /run/fde-lock/status (one "key<TAB>value" or "step<TAB>name<TAB>result<TAB>detail" per line).
|
||||
|
||||
# writeShellApplication enables errexit; failures are handled per step instead.
|
||||
@@ -17,13 +19,15 @@ esac
|
||||
|
||||
status=/run/fde-lock/status
|
||||
tmp="$status.tmp"
|
||||
state_dir=/var/lib/fde-lock
|
||||
truenas_ip=192.168.0.16
|
||||
crypt=ashenvale_crypt
|
||||
vg=ashenvale_pool
|
||||
clean_secs=20
|
||||
failed=0
|
||||
# incus exec into a VM starts commands with a PATH that misses the NixOS system profile.
|
||||
gpath=/run/wrappers/bin:/run/current-system/sw/bin
|
||||
db_re='mariadb|mysql|postgres|redis|(^|[-_])db([-_0-9]|$)'
|
||||
|
||||
mkdir -p /run/fde-lock
|
||||
mkdir -p /run/fde-lock "$state_dir"
|
||||
: > "$tmp"
|
||||
put() { printf '%s\n' "$*" >> "$tmp"; cp "$tmp" "$status"; }
|
||||
put "mode $mode"
|
||||
@@ -114,11 +118,12 @@ lock_nethergarde() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ ashenvale_crypt
|
||||
ashenvale_mounts() {
|
||||
# findmnt names the source /dev/dm-N (bind mounts add "[/subdir]"), so match both spellings.
|
||||
# ------------------------------------------------- LUKS stores (ashenvale, duskwood)
|
||||
# store_mounts VG: host mount points of the VG's volumes. findmnt names the source /dev/dm-N
|
||||
# (bind mounts add "[/subdir]"), so match both spellings.
|
||||
store_mounts() {
|
||||
local d devs=" "
|
||||
for d in /dev/mapper/"$vg"-*; do
|
||||
for d in /dev/mapper/"$1"-*; do
|
||||
[ -e "$d" ] && devs+="$d $(realpath "$d") "
|
||||
done
|
||||
findmnt -rn -o TARGET,SOURCE | while read -r t src; do
|
||||
@@ -126,94 +131,200 @@ ashenvale_mounts() {
|
||||
done
|
||||
}
|
||||
|
||||
lock_ashenvale() {
|
||||
# guest_mount INST VOL: where the guest mounted a block volume (by-id path from the Incus device name).
|
||||
guest_mount() {
|
||||
timeout --kill-after=3 10 incus exec -T "$1" --env "PATH=$gpath" -- sh -c \
|
||||
'd="$(realpath -e "$1")" && findmnt -rn -o TARGET -S "$d"' sh \
|
||||
"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_incus_$2-part1" < /dev/null 2> /dev/null | head -n1
|
||||
}
|
||||
|
||||
# guest_containers INST MOUNT...: running containers in the guest that bind anything below MOUNT.
|
||||
guest_containers() {
|
||||
local inst="$1"
|
||||
shift
|
||||
timeout --kill-after=3 10 incus exec -T "$inst" --env "PATH=$gpath" -- sh -c '
|
||||
docker ps -q | xargs -r docker inspect -f "{{.Name}}{{range .Mounts}} {{.Source}}{{end}}" |
|
||||
while read -r n srcs; do
|
||||
for s in $srcs; do
|
||||
for m in "$@"; do
|
||||
case "$s/" in "$m"/*) echo "${n#/}"; continue 3 ;; esac
|
||||
done
|
||||
done
|
||||
done' sh "$@" < /dev/null 2> /dev/null
|
||||
}
|
||||
|
||||
# release_instance NAME VG INST DEADLINE QUIESCE "VOL|PATH"...: free the store's volumes in one VM.
|
||||
# Clean within DEADLINE: stop the containers that use them (QUIESCE=1, databases first), unmount in
|
||||
# the guest, detach. Forced after: kill those containers, lazy unmount, detach anyway.
|
||||
# A stopped VM keeps its devices, so it cannot start while the store is locked.
|
||||
release_instance() {
|
||||
local name="$1" vg="$2" inst="$3" dl="$4" quiesce="$5" e vol path s c
|
||||
shift 5
|
||||
local vols=() mounts=() ctrs=() dbs=() rest=() stuck=()
|
||||
if ! running "$inst"; then
|
||||
note "$name $inst" info "not running, devices stay attached"
|
||||
return 0
|
||||
fi
|
||||
for e in "$@"; do
|
||||
vol="${e%%|*}"
|
||||
path="${e#*|}"
|
||||
vols+=("$vol")
|
||||
[ -n "$path" ] || path="$(guest_mount "$inst" "$vol")"
|
||||
[ -n "$path" ] && mounts+=("$path")
|
||||
done
|
||||
|
||||
if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ]; then
|
||||
mapfile -t ctrs < <(guest_containers "$inst" "${mounts[@]}")
|
||||
# Kept for the unlock; an empty list (second lock in a row) must not overwrite a good one.
|
||||
if [ "${#ctrs[@]}" -gt 0 ]; then printf '%s\n' "${ctrs[@]}" > "$state_dir/$name-$inst.containers"; fi
|
||||
for c in "${ctrs[@]}"; do
|
||||
if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
|
||||
done
|
||||
# Databases first while the budget is fresh; docker itself kills what outlives -t.
|
||||
if [ "${#dbs[@]}" -gt 0 ]; then
|
||||
s=$(( $(left "$dl") - 4 )); [ "$s" -le 10 ] || s=10
|
||||
if [ "$s" -gt 0 ]; then
|
||||
cstep "docker-stop-db $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${dbs[@]}"
|
||||
else
|
||||
note "docker-stop-db $inst" skip "clean budget used up"
|
||||
fi
|
||||
fi
|
||||
if [ "${#rest[@]}" -gt 0 ]; then
|
||||
s=$(( $(left "$dl") - 4 )); [ "$s" -le 5 ] || s=5
|
||||
if [ "$s" -gt 0 ]; then
|
||||
cstep "docker-stop $inst" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- docker stop -t "$s" "${rest[@]}"
|
||||
else
|
||||
note "docker-stop $inst" skip "clean budget used up"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
for path in "${mounts[@]}"; do
|
||||
cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" --env "PATH=$gpath" -- umount "$path" ||
|
||||
stuck+=("$path")
|
||||
done
|
||||
|
||||
# Forced: nothing that still holds a mount survives, then the mount is detached from the tree.
|
||||
if [ "${#stuck[@]}" -gt 0 ]; then
|
||||
if [ "${#ctrs[@]}" -gt 0 ]; then
|
||||
step "docker-kill $inst" 10 incus exec -T "$inst" --env "PATH=$gpath" -- docker kill "${ctrs[@]}" || true
|
||||
fi
|
||||
for path in "${stuck[@]}"; do
|
||||
step "guest-umount-lazy $inst:$path" 5 incus exec -T "$inst" --env "PATH=$gpath" -- umount -l "$path"
|
||||
done
|
||||
fi
|
||||
|
||||
for vol in "${vols[@]}"; do
|
||||
cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst" ||
|
||||
step "detach-force $vol from $inst" 10 incus storage volume detach "$vg" "$vol" "$inst"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# lock_store NAME CRYPT VG QUIESCE: release every attached volume (all VMs in parallel), then close the LUKS device.
|
||||
lock_store() {
|
||||
local name="$1" crypt="$2" vg="$3" quiesce="$4"
|
||||
if [ ! -e "/dev/mapper/$crypt" ]; then
|
||||
note "ashenvale" info "$crypt not open, nothing to do"
|
||||
note "$name" info "$crypt not open, nothing to do"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Volumes of the pool attached to instances (today: phonecam on varian), as "vol inst path".
|
||||
local vol inst path attached=""
|
||||
# Volumes of the pool attached to instances, grouped per instance as "vol|path" lines.
|
||||
local vol inst path
|
||||
local -A per=()
|
||||
for vol in $(inc storage volume list "$vg" type=custom -c n -f csv); do
|
||||
for inst in $(inc storage volume show "$vg" "custom/$vol" \
|
||||
| sed -n 's#^ *- /1.0/instances/\([^?]*\).*#\1#p'); do
|
||||
attached+="$vol $inst $(inc config device get "$inst" "$vol" path)"$'\n'
|
||||
per[$inst]+="$vol|$(inc config device get "$inst" "$vol" path)"$'\n'
|
||||
done
|
||||
done
|
||||
|
||||
local m
|
||||
local m es c
|
||||
if [ "$mode" = check ]; then
|
||||
while read -r vol inst path; do
|
||||
[ -n "$vol" ] && note "detach" info "would detach $vol from $inst ($path)"
|
||||
done <<< "$attached"
|
||||
for m in $(ashenvale_mounts); do note "host-umount" info "would umount $m"; done
|
||||
note "ashenvale" info "would close $crypt"
|
||||
for inst in "${!per[@]}"; do
|
||||
local mounts=() dbs=() rest=()
|
||||
while IFS='|' read -r vol path; do
|
||||
[ -n "$vol" ] || continue
|
||||
if [ -z "$path" ] && running "$inst"; then path="$(guest_mount "$inst" "$vol")"; fi
|
||||
[ -n "$path" ] && mounts+=("$path")
|
||||
note "$name detach" info "would detach $vol from $inst (${path:-not mounted})"
|
||||
done <<< "${per[$inst]}"
|
||||
if [ "$quiesce" = 1 ] && [ "${#mounts[@]}" -gt 0 ] && running "$inst"; then
|
||||
while read -r c; do
|
||||
[ -n "$c" ] || continue
|
||||
if [[ "$c" =~ $db_re ]]; then dbs+=("$c"); else rest+=("$c"); fi
|
||||
done < <(guest_containers "$inst" "${mounts[@]}")
|
||||
note "$name containers $inst" info "would stop db: ${dbs[*]:-none}; then: ${rest[*]:-none}"
|
||||
fi
|
||||
done
|
||||
for m in $(store_mounts "$vg"); do note "$name host-umount" info "would umount $m"; done
|
||||
note "$name" info "would close $crypt"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Clean: unmount in the guest, detach, unmount on the host, close, all within the budget.
|
||||
local dl
|
||||
# Clean: release the volumes in every VM at once, unmount on the host, close, all within the budget.
|
||||
local dl pids=()
|
||||
dl=$(( $(date +%s) + clean_secs ))
|
||||
while read -r vol inst path; do
|
||||
[ -n "$vol" ] || continue
|
||||
if [ -n "$path" ] && running "$inst"; then
|
||||
cstep "guest-umount $inst:$path" "$dl" incus exec -T "$inst" -- umount "$path"
|
||||
fi
|
||||
cstep "detach $vol from $inst" "$dl" incus storage volume detach "$vg" "$vol" "$inst"
|
||||
done <<< "$attached"
|
||||
for m in $(ashenvale_mounts | sort -r); do
|
||||
cstep "host-umount $m" "$dl" umount "$m"
|
||||
for inst in "${!per[@]}"; do
|
||||
mapfile -t es < <(printf '%s' "${per[$inst]}")
|
||||
release_instance "$name" "$vg" "$inst" "$dl" "$quiesce" "${es[@]}" &
|
||||
pids+=($!)
|
||||
done
|
||||
cstep "vg-deactivate" "$dl" vgchange -an "$vg"
|
||||
cstep "luks-close" "$dl" cryptsetup close "$crypt"
|
||||
[ "${#pids[@]}" -eq 0 ] || wait "${pids[@]}"
|
||||
for m in $(store_mounts "$vg" | sort -r); do
|
||||
cstep "$name host-umount $m" "$dl" umount "$m"
|
||||
done
|
||||
cstep "$name vg-deactivate" "$dl" vgchange -an "$vg"
|
||||
cstep "$name luks-close" "$dl" cryptsetup close "$crypt"
|
||||
|
||||
if [ -e "/dev/mapper/$crypt" ]; then
|
||||
# Forced: kill whatever still serves the volume (virtiofsd), detach the mounts, retry the close.
|
||||
for m in $(ashenvale_mounts); do
|
||||
if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "virtiofsd-kill $m" ok ""; fi
|
||||
for m in $(store_mounts "$vg"); do
|
||||
if pkill -9 -f -- "--shared-dir=$m( |$)"; then note "$name virtiofsd-kill $m" ok ""; fi
|
||||
done
|
||||
for m in $(ashenvale_mounts | sort -r); do
|
||||
step "host-umount-lazy $m" 5 umount -l "$m"
|
||||
for m in $(store_mounts "$vg" | sort -r); do
|
||||
step "$name host-umount-lazy $m" 5 umount -l "$m"
|
||||
done
|
||||
step "vg-deactivate-force" 10 vgchange -an "$vg"
|
||||
step "luks-close-force" 10 cryptsetup close "$crypt"
|
||||
step "$name vg-deactivate-force" 10 vgchange -an "$vg"
|
||||
step "$name luks-close-force" 10 cryptsetup close "$crypt"
|
||||
fi
|
||||
|
||||
if [ -e "/dev/mapper/$crypt" ]; then
|
||||
# Still busy: wipe the key now (suspend drops it from the kernel), then replace the
|
||||
# stack with error targets so nothing waits on the suspended device.
|
||||
step "luks-suspend" 10 cryptsetup luksSuspend "$crypt"
|
||||
step "$name luks-suspend" 10 cryptsetup luksSuspend "$crypt"
|
||||
for m in $(dmsetup ls 2> /dev/null | awk -v p="$vg-" 'index($1, p) == 1 {print $1}'); do
|
||||
step "dm-remove-force $m" 5 dmsetup remove --force "$m"
|
||||
step "$name dm-remove-force $m" 5 dmsetup remove --force "$m"
|
||||
done
|
||||
step "dm-remove-force $crypt" 5 dmsetup remove --force "$crypt"
|
||||
step "$name dm-remove-force $crypt" 5 dmsetup remove --force "$crypt"
|
||||
fi
|
||||
|
||||
# At rest = device gone, or still listed but suspended (key wiped) or an error target (crypt table dropped).
|
||||
if [ ! -e "/dev/mapper/$crypt" ]; then
|
||||
note "ashenvale" ok "$crypt closed"
|
||||
note "$name" ok "$crypt closed"
|
||||
else
|
||||
local attr target
|
||||
attr="$(dmsetup info -c --noheadings -o attr "$crypt" 2>&1)"
|
||||
target="$(dmsetup table "$crypt" 2> /dev/null | awk '{print $3; exit}')"
|
||||
if [[ "$attr" == *s* ]]; then
|
||||
note "ashenvale" ok "$crypt still listed but suspended, key wiped"
|
||||
note "$name" ok "$crypt still listed but suspended, key wiped"
|
||||
elif [ "$target" = error ]; then
|
||||
note "ashenvale" ok "$crypt replaced by error target, key dropped"
|
||||
note "$name" ok "$crypt replaced by error target, key dropped"
|
||||
else
|
||||
note "ashenvale" fail "$crypt still active: attr $attr, target $target"
|
||||
note "$name" fail "$crypt still active: attr $attr, target $target"
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Both locks run in parallel so a slow TrueNAS shutdown does not delay ashenvale.
|
||||
# All three run in parallel so a slow TrueNAS shutdown or container stop does not delay the others.
|
||||
lock_nethergarde > /run/fde-lock/nethergarde.log 2>&1 &
|
||||
lock_ashenvale
|
||||
lock_store ashenvale ashenvale_crypt ashenvale_pool 0 &
|
||||
lock_store duskwood duskwood_crypt duskwood_pool 1 &
|
||||
wait
|
||||
|
||||
# Subshell failures do not reach this shell; derive them from the recorded steps.
|
||||
if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
|
||||
if awk -F '\t' '$1 == "step" && ($2 == "truenas" || $2 == "ashenvale" || $2 == "duskwood" || $2 == "truenas-query") && $3 ~ /^fail/ {f = 1} END {exit !f}' "$tmp"; then
|
||||
failed=1
|
||||
fi
|
||||
|
||||
|
||||
Reference in new issue
Block a user