diff --git a/scripts/container-doctor.sh b/scripts/container-doctor.sh index 322c8e94..e16795af 100755 --- a/scripts/container-doctor.sh +++ b/scripts/container-doctor.sh @@ -383,10 +383,22 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]])) # ── Fix 8: Rootless netns egress lost ──────────────────────── # Rootless podman uses pasta to give containers internet egress. If pasta's -# tap vanishes (host link flap, mount churn), the rootless-netns keeps inter- -# container traffic working but silently loses outbound. Bitcoin IBD stalls -# at 0 peers; package pulls fail. The only reliable repair is a stop-all/ -# start-all cycle so pasta + aardvark-dns rebuild the netns from scratch. +# tap vanishes (host link flap, mount churn, pasta dying during a boot-time +# restart storm), the rootless-netns keeps inter-container traffic working +# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls +# fail. The repair must rebuild the netns from scratch: merely cycling the +# containers reuses the existing (broken) netns because its holders +# (aardvark-dns, podman's pause process) survive — observed on shorty-s +# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers +# every timer run for ~an hour without ever restoring egress. So: stop the +# containers, kill the netns holders, `podman system migrate`, clear the +# stale netns state, then start everything back up. +# +# Destructive-action latch: cycling the whole fleet is a last resort. After +# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly) +# until a run observes egress healthy again, which resets the counter. +NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures" +NETNS_CYCLE_MAX=3 fix_rootless_netns_egress() { # Needs root for nsenter. When doctor runs as the rootless container owner, # a failed nsenter probe is a permissions artifact, not evidence of broken @@ -410,16 +422,28 @@ fix_rootless_netns_egress() { # Probe egress from inside the rootless-netns. One probe is noisy; # require two consecutive failures 10s apart to rule out transients. if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then + rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch return 1 # first probe succeeded fi sleep 10 aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1) [ -z "$aardvark_pid" ] && return 1 if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then + rm -f "$NETNS_CYCLE_STATE" return 1 # recovered on its own fi - log "Rootless-netns egress is broken (host online, container netns unreachable) — cycling" + # Latch: don't keep bouncing the fleet when the rebuild demonstrably + # isn't fixing it. + local failures + failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0) + case "$failures" in *[!0-9]*|"") failures=0;; esac + if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then + log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)" + return 1 + fi + + log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns" local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman" local running @@ -435,6 +459,19 @@ fix_rootless_netns_egress() { $PODMANCMD stop --all --time 30 >/dev/null 2>&1 sleep 5 + # Tear the broken netns down for real: kill its holders and drop the + # stale state so the first container start rebuilds pasta + aardvark-dns + # from scratch. Without this, podman re-enters the old netns and the + # missing pasta tap never comes back. + log " Rebuilding rootless netns (killing holders, clearing state)..." + pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null + pkill -U "$archi_uid" -x pasta 2>/dev/null + pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null + pkill -U "$archi_uid" -x slirp4netns 2>/dev/null + sleep 2 + $PODMANCMD system migrate >/dev/null 2>&1 + rm -rf "/run/user/$archi_uid/containers/networks" + log " Starting containers back up..." for c in $running; do $PODMANCMD start "$c" >/dev/null 2>&1 & @@ -445,8 +482,11 @@ fix_rootless_netns_egress() { aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1) if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then log " Rootless-netns egress restored ($count containers cycled)" + rm -f "$NETNS_CYCLE_STATE" else - log " WARN: egress still broken after cycle — may need manual intervention" + failures=$((failures + 1)) + echo "$failures" > "$NETNS_CYCLE_STATE" + log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention" fi return 0 }