lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 10 Wiki Activity

fix: keep apps running when network diagnostics fail

Browse Source
This commit is contained in:
archipelago
2026-09-30 17:34:11 -04:00
parent 02b840f2d1
commit f91c1f33db
11 changed files with 96 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/bitcoin-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY tailwind.css /usr/share/nginx/html/
docker/cuprate-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
docker/electrs-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
COPY qrcode.js /usr/share/nginx/html/
docker/fedimint-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/index.html
COPY nginx.conf /etc/nginx/conf.d/default.conf
docker/fips-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
#
docker/lnd-ui/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Copy the HTML file
COPY index.html /usr/share/nginx/html/
scripts/check-app-build-contexts.py
+2
View File
@@ -27,6 +27,8 @@ def check(root: Path) -> int:
dockerfile = (context / build.get('dockerfile', 'Dockerfile')).resolve()
if not dockerfile.is_relative_to(context) or not dockerfile.is_file():
raise ValueError(f'{app["id"]}: missing or out-of-context Dockerfile: {dockerfile}')
if 'git.tx1138.com/' in dockerfile.read_text():
raise ValueError(f'{app["id"]}: Dockerfile references retired registry git.tx1138.com')
count += 1
return count
scripts/container-doctor.sh
+28 -99
View File
@@ -32,6 +32,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
FIXES_APPLIED=0
CHECKS_PASSED=0
CHECKS_WARNED=0
WARNING_NAMES=()
FIX_NAMES=()
log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
@@ -83,7 +85,13 @@ run_fix() {
FIXES_APPLIED=$((FIXES_APPLIED + 1))
FIX_NAMES+=("$name")
else
CHECKS_PASSED=$((CHECKS_PASSED + 1))
local status=$?
if [ "$status" = 1 ]; then
CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
CHECKS_WARNED=$((CHECKS_WARNED + 1))
WARNING_NAMES+=("$name")
fi
fi
}
@@ -446,114 +454,33 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
[ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1
}
# ── Fix 8: Rootless netns egress lost ────────────────────────
# Rootless podman uses pasta to give containers internet egress. If pasta's
# tap vanishes (host link flap, mount churn, pasta dying during a boot-time
# restart storm), the rootless-netns keeps inter-container traffic working
# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls
# fail. The repair must rebuild the netns from scratch: merely cycling the
# containers reuses the existing (broken) netns because its holders
# (aardvark-dns, podman's pause process) survive — observed on a test node
# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers
# every timer run for ~an hour without ever restoring egress. So: stop the
# containers, kill the netns holders, `podman system migrate`, clear the
# stale netns state, then start everything back up.
#
# Destructive-action latch: cycling the whole fleet is a last resort. After
# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly)
# until a run observes egress healthy again, which resets the counter.
NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures"
NETNS_CYCLE_MAX=3
fix_rootless_netns_egress() {
# Needs root for nsenter. When doctor runs as the rootless container owner,
# a failed nsenter probe is a permissions artifact, not evidence of broken
# egress; do not cycle the fleet from that context.
# ── Check 8: Rootless network egress (diagnostic only) ──────
# A single external endpoint or nsenter failure cannot establish that the
# containers have lost connectivity. In particular, entering only the network
# namespace can fail for rootless user namespaces. Never stop apps, kill network
# helpers, migrate Podman, or remove network state in response to this probe.
# Return 1 for healthy/not applicable and 2 for an inconclusive warning.
check_rootless_netns_egress() {
[ "$(id -u)" = "0" ] || return 1
local archi_uid
local archi_uid aardvark_pid
archi_uid=$(id -u archipelago 2>/dev/null) || return 1
# Locate the rootless-netns via aardvark-dns (it lives inside it).
local aardvark_pid
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1 # no rootless network active
[ -n "$aardvark_pid" ] || return 1
# Host precheck: if the host itself can't reach the internet, no point
# cycling containers — this is an upstream problem.
if ! timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
return 1
log "WARNING: host connectivity probe failed; external endpoint may be unavailable. Apps left running."
return 2
fi
# Probe egress from inside the rootless-netns. One probe is noisy;
# require two consecutive failures 10s apart to rule out transients.
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch
return 1 # first probe succeeded
return 1
fi
sleep 10
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE"
return 1 # recovered on its own
fi
# Latch: don't keep bouncing the fleet when the rebuild demonstrably
# isn't fixing it.
local failures
failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0)
case "$failures" in *[!0-9]*|"") failures=0;; esac
if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then
log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)"
return 1
fi
log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns"
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman"
local running
running=$($PODMANCMD ps --format '{{.Names}}' 2>/dev/null)
if [ -z "$running" ]; then
log " No running containers to cycle — skipping"
return 1
fi
local count
count=$(echo "$running" | wc -l)
log " Stopping $count running containers (graceful, 30s)..."
$PODMANCMD stop --all --time 30 >/dev/null 2>&1
sleep 5
# Tear the broken netns down for real: kill its holders and drop the
# stale state so the first container start rebuilds pasta + aardvark-dns
# from scratch. Without this, podman re-enters the old netns and the
# missing pasta tap never comes back.
log " Rebuilding rootless netns (killing holders, clearing state)..."
pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null
pkill -U "$archi_uid" -x pasta 2>/dev/null
pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null
pkill -U "$archi_uid" -x slirp4netns 2>/dev/null
sleep 2
$PODMANCMD system migrate >/dev/null 2>&1
rm -rf "/run/user/$archi_uid/containers/networks"
log " Starting containers back up..."
for c in $running; do
$PODMANCMD start "$c" >/dev/null 2>&1 &
done
wait
sleep 5
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
log " Rootless-netns egress restored ($count containers cycled)"
rm -f "$NETNS_CYCLE_STATE"
else
failures=$((failures + 1))
echo "$failures" > "$NETNS_CYCLE_STATE"
log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention"
return 1
fi
return 0
log "WARNING: rootless network probe inconclusive (endpoint, connectivity, or namespace access). Inspect affected apps before repair. Apps left running."
return 2
}
# ── Fix 9: Restart stopped core containers ──────────────────
@@ -731,7 +658,7 @@ run_fix "tor-permissions" fix_tor_permissions
run_fix "searxng" fix_searxng
run_fix "bitcoin-txindex" fix_bitcoin_txindex
run_fix "exit-127" fix_exit_127
run_fix "netns-egress" fix_rootless_netns_egress
run_fix "netns-egress" check_rootless_netns_egress
run_fix "stopped-core" fix_stopped_core_containers
run_fix "rootless-ports" fix_missing_rootless_ports
run_fix "npm-public-hosts" fix_npm_public_hosts
@@ -740,7 +667,9 @@ run_fix "catatonit" fix_missing_catatonit
run_fix "dialout" fix_archipelago_dialout
echo ""
if [ $FIXES_APPLIED -gt 0 ]; then
if [ "$CHECKS_WARNED" -gt 0 ]; then
log "Done: $CHECKS_WARNED unresolved warnings (${WARNING_NAMES[*]}), $FIXES_APPLIED fixes applied, $CHECKS_PASSED checks passed"
elif [ $FIXES_APPLIED -gt 0 ]; then
log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed"
else
log "Done: all $CHECKS_PASSED checks passed — no fixes needed"
tests/regression/app-build-contexts.py
+6
View File
@@ -40,6 +40,12 @@ class BuildPayloadTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'out-of-payload'):
contexts.check(self.root)
def test_retired_registry_rejected(self):
target = self.root / 'docker/lnd-ui/Dockerfile'
target.write_text('FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine\n')
with self.assertRaisesRegex(ValueError, 'lnd-ui.*retired registry'):
contexts.check(self.root)
def test_empty_payload_rejected(self):
shutil.rmtree(self.root / 'apps')
with self.assertRaisesRegex(ValueError, 'No app manifests'):
tests/regression/container-doctor-egress.sh
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Simulate failures without namespaces, network access, or real repair commands.
set -euo pipefail
source "$(dirname "$0")/../../scripts/container-doctor.sh"
id() { if [[ "$*" == '-u archipelago' ]]; then echo 1000; else echo "${TEST_UID:-0}"; fi; }
pgrep() { if [[ "$HAS_NETWORK" == 1 ]]; then echo 123; else return 1; fi; }
sleep() { :; }
# Any mutation fails the test immediately, including within command substitution.
tripwire() { echo 'FAIL: diagnostic attempted a mutation' >&2; exit 99; }
podman() { tripwire; }
podman_rootless() { tripwire; }
sudo() { tripwire; }
systemctl() { tripwire; }
pkill() { tripwire; }
kill() { tripwire; }
rm() { tripwire; }
mkdir() { tripwire; }
timeout() {
if [[ "$2" == bash ]]; then return "$HOST_STATUS"; fi
[[ "$2" == nsenter ]] || exit 98
PROBES=$((PROBES + 1))
if [[ "$PROBES" == 1 ]]; then return "$FIRST_STATUS"; fi
return "$SECOND_STATUS"
}
check_case() {
local label=$1 expected=$2 expected_probes=$3
PROBES=0
local status=0
check_rootless_netns_egress > /dev/null || status=$?
[[ "$status" == "$expected" && "$PROBES" == "$expected_probes" ]] || {
echo "FAIL: $label status=$status probes=$PROBES"; exit 1;
}
echo "PASS: $label"
}
HAS_NETWORK=1 HOST_STATUS=0 FIRST_STATUS=0 SECOND_STATUS=0
check_case healthy 1 1
TEST_UID=1000 check_case rootless-caller 1 0
HAS_NETWORK=0 check_case no-network 1 0
HOST_STATUS=1 check_case host-offline 2 0
FIRST_STATUS=1 check_case transient-recovery 1 2
FIRST_STATUS=1 SECOND_STATUS=1 check_case repeated-egress-failure 2 2
FIRST_STATUS=126 SECOND_STATUS=126 check_case namespace-access-failure 2 2
# Failure must remain an unresolved warning on every scheduled invocation.
FIRST_STATUS=1 SECOND_STATUS=1
for attempt in 1 2 3 4 5; do
PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
done
[[ "$CHECKS_WARNED" == 5 && "$FIXES_APPLIED" == 0 && "$CHECKS_PASSED" == 0 ]]
FIRST_STATUS=0 PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
[[ "$CHECKS_PASSED" == 1 && "$FIXES_APPLIED" == 0 ]]
echo 'PASS: repeated failure warnings never trigger repair or report a successful check'
tests/release/run.sh
+1
View File
@@ -74,6 +74,7 @@ stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml -
stage "app-build-contexts" python3 tests/regression/app-build-contexts.py
stage "manifest-shell" python3 scripts/check-manifest-shell.py
stage "npm-tunnel-migration" python3 -m unittest discover -s scripts/tests -p test_repair_npm_tunnel.py
stage "doctor-egress" bash tests/regression/container-doctor-egress.sh
stage "doctor-ports" bash tests/regression/container-doctor-ports.sh
stage "bitcoin-pruning" python3 tests/regression/bitcoin-prune-entrypoint.py
stage "lnd-ui-readiness" node --test tests/regression/lnd-ui-readiness.cjs