From f91c1f33db3c3204f993bf5a95e05d3f8d546569 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 30 Sep 2026 17:34:11 -0400 Subject: [PATCH] fix: keep apps running when network diagnostics fail --- docker/bitcoin-ui/Dockerfile | 2 +- docker/cuprate-ui/Dockerfile | 2 +- docker/electrs-ui/Dockerfile | 2 +- docker/fedimint-ui/Dockerfile | 2 +- docker/fips-ui/Dockerfile | 2 +- docker/lnd-ui/Dockerfile | 2 +- scripts/check-app-build-contexts.py | 2 + scripts/container-doctor.sh | 127 +++++--------------- tests/regression/app-build-contexts.py | 6 + tests/regression/container-doctor-egress.sh | 53 ++++++++ tests/release/run.sh | 1 + 11 files changed, 96 insertions(+), 105 deletions(-) create mode 100644 tests/regression/container-doctor-egress.sh diff --git a/docker/bitcoin-ui/Dockerfile b/docker/bitcoin-ui/Dockerfile index 0fa32db0..ede1e63c 100644 --- a/docker/bitcoin-ui/Dockerfile +++ b/docker/bitcoin-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine # Static site content. COPY index.html /usr/share/nginx/html/ COPY tailwind.css /usr/share/nginx/html/ diff --git a/docker/cuprate-ui/Dockerfile b/docker/cuprate-ui/Dockerfile index 0bf1b297..589025c7 100644 --- a/docker/cuprate-ui/Dockerfile +++ b/docker/cuprate-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine # Static site content. COPY index.html /usr/share/nginx/html/ COPY 50x.html /usr/share/nginx/html/ diff --git a/docker/electrs-ui/Dockerfile b/docker/electrs-ui/Dockerfile index fdfe0e5e..f1f513b4 100644 --- a/docker/electrs-ui/Dockerfile +++ b/docker/electrs-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine COPY index.html /usr/share/nginx/html/ COPY 50x.html /usr/share/nginx/html/ COPY qrcode.js /usr/share/nginx/html/ diff --git a/docker/fedimint-ui/Dockerfile b/docker/fedimint-ui/Dockerfile index 7cb19290..71cb459b 100644 --- a/docker/fedimint-ui/Dockerfile +++ b/docker/fedimint-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine COPY index.html /usr/share/nginx/html/index.html COPY nginx.conf /etc/nginx/conf.d/default.conf diff --git a/docker/fips-ui/Dockerfile b/docker/fips-ui/Dockerfile index e531d196..a4a84a96 100644 --- a/docker/fips-ui/Dockerfile +++ b/docker/fips-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine # Static site content. COPY index.html /usr/share/nginx/html/ # diff --git a/docker/lnd-ui/Dockerfile b/docker/lnd-ui/Dockerfile index 13b9401d..c9ffb1da 100644 --- a/docker/lnd-ui/Dockerfile +++ b/docker/lnd-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine +FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine # Copy the HTML file COPY index.html /usr/share/nginx/html/ diff --git a/scripts/check-app-build-contexts.py b/scripts/check-app-build-contexts.py index 1f3e1fd8..10d52c02 100644 --- a/scripts/check-app-build-contexts.py +++ b/scripts/check-app-build-contexts.py @@ -27,6 +27,8 @@ def check(root: Path) -> int: dockerfile = (context / build.get('dockerfile', 'Dockerfile')).resolve() if not dockerfile.is_relative_to(context) or not dockerfile.is_file(): raise ValueError(f'{app["id"]}: missing or out-of-context Dockerfile: {dockerfile}') + if 'git.tx1138.com/' in dockerfile.read_text(): + raise ValueError(f'{app["id"]}: Dockerfile references retired registry git.tx1138.com') count += 1 return count diff --git a/scripts/container-doctor.sh b/scripts/container-doctor.sh index 05ceb149..4ac29c0d 100755 --- a/scripts/container-doctor.sh +++ b/scripts/container-doctor.sh @@ -32,6 +32,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" FIXES_APPLIED=0 CHECKS_PASSED=0 +CHECKS_WARNED=0 +WARNING_NAMES=() FIX_NAMES=() log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; } @@ -83,7 +85,13 @@ run_fix() { FIXES_APPLIED=$((FIXES_APPLIED + 1)) FIX_NAMES+=("$name") else - CHECKS_PASSED=$((CHECKS_PASSED + 1)) + local status=$? + if [ "$status" = 1 ]; then + CHECKS_PASSED=$((CHECKS_PASSED + 1)) + else + CHECKS_WARNED=$((CHECKS_WARNED + 1)) + WARNING_NAMES+=("$name") + fi fi } @@ -446,114 +454,33 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]])) [ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1 } -# ── Fix 8: Rootless netns egress lost ──────────────────────── -# Rootless podman uses pasta to give containers internet egress. If pasta's -# tap vanishes (host link flap, mount churn, pasta dying during a boot-time -# restart storm), the rootless-netns keeps inter-container traffic working -# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls -# fail. The repair must rebuild the netns from scratch: merely cycling the -# containers reuses the existing (broken) netns because its holders -# (aardvark-dns, podman's pause process) survive — observed on a test node -# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers -# every timer run for ~an hour without ever restoring egress. So: stop the -# containers, kill the netns holders, `podman system migrate`, clear the -# stale netns state, then start everything back up. -# -# Destructive-action latch: cycling the whole fleet is a last resort. After -# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly) -# until a run observes egress healthy again, which resets the counter. -NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures" -NETNS_CYCLE_MAX=3 -fix_rootless_netns_egress() { - # Needs root for nsenter. When doctor runs as the rootless container owner, - # a failed nsenter probe is a permissions artifact, not evidence of broken - # egress; do not cycle the fleet from that context. +# ── Check 8: Rootless network egress (diagnostic only) ────── +# A single external endpoint or nsenter failure cannot establish that the +# containers have lost connectivity. In particular, entering only the network +# namespace can fail for rootless user namespaces. Never stop apps, kill network +# helpers, migrate Podman, or remove network state in response to this probe. +# Return 1 for healthy/not applicable and 2 for an inconclusive warning. +check_rootless_netns_egress() { [ "$(id -u)" = "0" ] || return 1 - - local archi_uid + local archi_uid aardvark_pid archi_uid=$(id -u archipelago 2>/dev/null) || return 1 - - # Locate the rootless-netns via aardvark-dns (it lives inside it). - local aardvark_pid aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1) - [ -z "$aardvark_pid" ] && return 1 # no rootless network active + [ -n "$aardvark_pid" ] || return 1 - # Host precheck: if the host itself can't reach the internet, no point - # cycling containers — this is an upstream problem. if ! timeout 3 bash -c '/dev/null; then - return 1 + log "WARNING: host connectivity probe failed; external endpoint may be unavailable. Apps left running." + return 2 fi - - # Probe egress from inside the rootless-netns. One probe is noisy; - # require two consecutive failures 10s apart to rule out transients. if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then - rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch - return 1 # first probe succeeded + return 1 fi sleep 10 - aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1) - [ -z "$aardvark_pid" ] && return 1 - if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then - rm -f "$NETNS_CYCLE_STATE" - return 1 # recovered on its own - fi - - # Latch: don't keep bouncing the fleet when the rebuild demonstrably - # isn't fixing it. - local failures - failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0) - case "$failures" in *[!0-9]*|"") failures=0;; esac - if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then - log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)" - return 1 - fi - - log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns" - - local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman" - local running - running=$($PODMANCMD ps --format '{{.Names}}' 2>/dev/null) - if [ -z "$running" ]; then - log " No running containers to cycle — skipping" - return 1 - fi - - local count - count=$(echo "$running" | wc -l) - log " Stopping $count running containers (graceful, 30s)..." - $PODMANCMD stop --all --time 30 >/dev/null 2>&1 - sleep 5 - - # Tear the broken netns down for real: kill its holders and drop the - # stale state so the first container start rebuilds pasta + aardvark-dns - # from scratch. Without this, podman re-enters the old netns and the - # missing pasta tap never comes back. - log " Rebuilding rootless netns (killing holders, clearing state)..." - pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null - pkill -U "$archi_uid" -x pasta 2>/dev/null - pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null - pkill -U "$archi_uid" -x slirp4netns 2>/dev/null - sleep 2 - $PODMANCMD system migrate >/dev/null 2>&1 - rm -rf "/run/user/$archi_uid/containers/networks" - - log " Starting containers back up..." - for c in $running; do - $PODMANCMD start "$c" >/dev/null 2>&1 & - done - wait - sleep 5 - aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1) if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '/dev/null; then - log " Rootless-netns egress restored ($count containers cycled)" - rm -f "$NETNS_CYCLE_STATE" - else - failures=$((failures + 1)) - echo "$failures" > "$NETNS_CYCLE_STATE" - log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention" + return 1 fi - return 0 + log "WARNING: rootless network probe inconclusive (endpoint, connectivity, or namespace access). Inspect affected apps before repair. Apps left running." + return 2 } # ── Fix 9: Restart stopped core containers ────────────────── @@ -731,7 +658,7 @@ run_fix "tor-permissions" fix_tor_permissions run_fix "searxng" fix_searxng run_fix "bitcoin-txindex" fix_bitcoin_txindex run_fix "exit-127" fix_exit_127 -run_fix "netns-egress" fix_rootless_netns_egress +run_fix "netns-egress" check_rootless_netns_egress run_fix "stopped-core" fix_stopped_core_containers run_fix "rootless-ports" fix_missing_rootless_ports run_fix "npm-public-hosts" fix_npm_public_hosts @@ -740,7 +667,9 @@ run_fix "catatonit" fix_missing_catatonit run_fix "dialout" fix_archipelago_dialout echo "" -if [ $FIXES_APPLIED -gt 0 ]; then +if [ "$CHECKS_WARNED" -gt 0 ]; then + log "Done: $CHECKS_WARNED unresolved warnings (${WARNING_NAMES[*]}), $FIXES_APPLIED fixes applied, $CHECKS_PASSED checks passed" +elif [ $FIXES_APPLIED -gt 0 ]; then log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed" else log "Done: all $CHECKS_PASSED checks passed — no fixes needed" diff --git a/tests/regression/app-build-contexts.py b/tests/regression/app-build-contexts.py index 6a1be14b..83793be1 100644 --- a/tests/regression/app-build-contexts.py +++ b/tests/regression/app-build-contexts.py @@ -40,6 +40,12 @@ class BuildPayloadTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'out-of-payload'): contexts.check(self.root) + def test_retired_registry_rejected(self): + target = self.root / 'docker/lnd-ui/Dockerfile' + target.write_text('FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine\n') + with self.assertRaisesRegex(ValueError, 'lnd-ui.*retired registry'): + contexts.check(self.root) + def test_empty_payload_rejected(self): shutil.rmtree(self.root / 'apps') with self.assertRaisesRegex(ValueError, 'No app manifests'): diff --git a/tests/regression/container-doctor-egress.sh b/tests/regression/container-doctor-egress.sh new file mode 100644 index 00000000..c2c8b077 --- /dev/null +++ b/tests/regression/container-doctor-egress.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Simulate failures without namespaces, network access, or real repair commands. +set -euo pipefail +source "$(dirname "$0")/../../scripts/container-doctor.sh" +id() { if [[ "$*" == '-u archipelago' ]]; then echo 1000; else echo "${TEST_UID:-0}"; fi; } +pgrep() { if [[ "$HAS_NETWORK" == 1 ]]; then echo 123; else return 1; fi; } +sleep() { :; } +# Any mutation fails the test immediately, including within command substitution. +tripwire() { echo 'FAIL: diagnostic attempted a mutation' >&2; exit 99; } +podman() { tripwire; } +podman_rootless() { tripwire; } +sudo() { tripwire; } +systemctl() { tripwire; } +pkill() { tripwire; } +kill() { tripwire; } +rm() { tripwire; } +mkdir() { tripwire; } +timeout() { + if [[ "$2" == bash ]]; then return "$HOST_STATUS"; fi + [[ "$2" == nsenter ]] || exit 98 + PROBES=$((PROBES + 1)) + if [[ "$PROBES" == 1 ]]; then return "$FIRST_STATUS"; fi + return "$SECOND_STATUS" +} +check_case() { + local label=$1 expected=$2 expected_probes=$3 + PROBES=0 + local status=0 + check_rootless_netns_egress > /dev/null || status=$? + [[ "$status" == "$expected" && "$PROBES" == "$expected_probes" ]] || { + echo "FAIL: $label status=$status probes=$PROBES"; exit 1; + } + echo "PASS: $label" +} +HAS_NETWORK=1 HOST_STATUS=0 FIRST_STATUS=0 SECOND_STATUS=0 +check_case healthy 1 1 +TEST_UID=1000 check_case rootless-caller 1 0 +HAS_NETWORK=0 check_case no-network 1 0 +HOST_STATUS=1 check_case host-offline 2 0 +FIRST_STATUS=1 check_case transient-recovery 1 2 +FIRST_STATUS=1 SECOND_STATUS=1 check_case repeated-egress-failure 2 2 +FIRST_STATUS=126 SECOND_STATUS=126 check_case namespace-access-failure 2 2 +# Failure must remain an unresolved warning on every scheduled invocation. +FIRST_STATUS=1 SECOND_STATUS=1 +for attempt in 1 2 3 4 5; do + PROBES=0 + run_fix netns-egress check_rootless_netns_egress > /dev/null +done +[[ "$CHECKS_WARNED" == 5 && "$FIXES_APPLIED" == 0 && "$CHECKS_PASSED" == 0 ]] +FIRST_STATUS=0 PROBES=0 +run_fix netns-egress check_rootless_netns_egress > /dev/null +[[ "$CHECKS_PASSED" == 1 && "$FIXES_APPLIED" == 0 ]] +echo 'PASS: repeated failure warnings never trigger repair or report a successful check' diff --git a/tests/release/run.sh b/tests/release/run.sh index 3ea8ff68..1c04af14 100755 --- a/tests/release/run.sh +++ b/tests/release/run.sh @@ -74,6 +74,7 @@ stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml - stage "app-build-contexts" python3 tests/regression/app-build-contexts.py stage "manifest-shell" python3 scripts/check-manifest-shell.py stage "npm-tunnel-migration" python3 -m unittest discover -s scripts/tests -p test_repair_npm_tunnel.py +stage "doctor-egress" bash tests/regression/container-doctor-egress.sh stage "doctor-ports" bash tests/regression/container-doctor-ports.sh stage "bitcoin-pruning" python3 tests/regression/bitcoin-prune-entrypoint.py stage "lnd-ui-readiness" node --test tests/regression/lnd-ui-readiness.cjs