fix(doctor): actually rebuild the rootless netns, and latch after repeated failures

The netns-egress repair only stopped/started containers, which reuses the
existing netns — its holders (aardvark-dns, podman pause) survive, so a
missing pasta tap was never rebuilt. On shorty-s (2026-07-10) this cycled all
35 containers every timer run for ~an hour, bouncing bitcoind/LND/BTCPay the
whole time, with 'egress still broken after cycle' after every pass.

Now the cycle kills the netns holders, runs podman system migrate, and clears
/run/user/<uid>/containers/networks so the first container start recreates
pasta + aardvark-dns from scratch (the sequence that actually fixed the node).
A failure counter (/var/lib/archipelago/doctor-netns-cycle-failures) stops the
fleet-wide cycling after 3 consecutive failed rebuilds until egress is
observed healthy again.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/container-doctor.sh

@@ -383,10 +383,22 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
 
 # ── Fix 8: Rootless netns egress lost ────────────────────────
 # Rootless podman uses pasta to give containers internet egress. If pasta's
-# tap vanishes (host link flap, mount churn), the rootless-netns keeps inter-
+# tap vanishes (host link flap, mount churn, pasta dying during a boot-time
-# container traffic working but silently loses outbound. Bitcoin IBD stalls
+# restart storm), the rootless-netns keeps inter-container traffic working
-# at 0 peers; package pulls fail. The only reliable repair is a stop-all/
+# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls
-# start-all cycle so pasta + aardvark-dns rebuild the netns from scratch.
+# fail. The repair must rebuild the netns from scratch: merely cycling the
+# containers reuses the existing (broken) netns because its holders
+# (aardvark-dns, podman's pause process) survive — observed on shorty-s
+# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers
+# every timer run for ~an hour without ever restoring egress. So: stop the
+# containers, kill the netns holders, `podman system migrate`, clear the
+# stale netns state, then start everything back up.
+#
+# Destructive-action latch: cycling the whole fleet is a last resort. After
+# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly)
+# until a run observes egress healthy again, which resets the counter.
+NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures"
+NETNS_CYCLE_MAX=3
 fix_rootless_netns_egress() {
     # Needs root for nsenter. When doctor runs as the rootless container owner,
     # a failed nsenter probe is a permissions artifact, not evidence of broken
@@ -410,16 +422,28 @@ fix_rootless_netns_egress() {
     # Probe egress from inside the rootless-netns. One probe is noisy;
     # require two consecutive failures 10s apart to rule out transients.
     if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
+        rm -f "$NETNS_CYCLE_STATE"   # healthy again — re-arm the latch
         return 1   # first probe succeeded
     fi
     sleep 10
     aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
     [ -z "$aardvark_pid" ] && return 1
     if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
+        rm -f "$NETNS_CYCLE_STATE"
         return 1   # recovered on its own
     fi
 
-    log "Rootless-netns egress is broken (host online, container netns unreachable) — cycling"
+    # Latch: don't keep bouncing the fleet when the rebuild demonstrably
+    # isn't fixing it.
+    local failures
+    failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0)
+    case "$failures" in *[!0-9]*|"") failures=0;; esac
+    if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then
+        log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)"
+        return 1
+    fi
+
+    log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns"
 
     local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman"
     local running
@@ -435,6 +459,19 @@ fix_rootless_netns_egress() {
     $PODMANCMD stop --all --time 30 >/dev/null 2>&1
     sleep 5
 
+    # Tear the broken netns down for real: kill its holders and drop the
+    # stale state so the first container start rebuilds pasta + aardvark-dns
+    # from scratch. Without this, podman re-enters the old netns and the
+    # missing pasta tap never comes back.
+    log "  Rebuilding rootless netns (killing holders, clearing state)..."
+    pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null
+    pkill -U "$archi_uid" -x pasta 2>/dev/null
+    pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null
+    pkill -U "$archi_uid" -x slirp4netns 2>/dev/null
+    sleep 2
+    $PODMANCMD system migrate >/dev/null 2>&1
+    rm -rf "/run/user/$archi_uid/containers/networks"
+
     log "  Starting containers back up..."
     for c in $running; do
         $PODMANCMD start "$c" >/dev/null 2>&1 &
@@ -445,8 +482,11 @@ fix_rootless_netns_egress() {
     aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
     if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
         log "  Rootless-netns egress restored ($count containers cycled)"
+        rm -f "$NETNS_CYCLE_STATE"
     else
-        log "  WARN: egress still broken after cycle — may need manual intervention"
+        failures=$((failures + 1))
+        echo "$failures" > "$NETNS_CYCLE_STATE"
+        log "  WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention"
     fi
     return 0
 }