The frontend nginx used a literal proxy_pass host with no resolver, so it pinned mempool-api's IP at worker startup. When the backend restarts (gate, OTA, crash, reboot re-IPAM) podman reassigns its IP and nginx keeps proxying to the dead one -> /api hangs, websocket 502s, UI shows 'offline' until a manual nginx reload. Same stale-upstream-IP class as the netbird 502. Fix: mempool-frontend:v3.0.1 rewrites the generated nginx-mempool.conf to re-resolve the backend per-request via 'resolver' + a variable proxy_pass. Resolver address is read from /etc/resolv.conf (podman aardvark-dns answers on the network gateway, not Docker's 127.0.0.11). Per-location path mapping preserved (ws -> '/', /api/v1 identity via no-URI, /api/ -> /api/v1/ rewrite). Proven on .228: backend IP change now auto-recovers with no reload; the literal-host control still 502s. Migrated the manifest off the retired tx1138 registry to vps2. Also: mempool.bats #74 waited only 180s post-restart (the slow path) and called an undefined 'fail' helper (status 127). Bumped to 300s to match the passing parity probes and emit a real failure instead. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
181 lines
6.9 KiB
Bash
181 lines
6.9 KiB
Bash
#!/usr/bin/env bats
|
|
# tests/lifecycle/bats/mempool.bats
|
|
#
|
|
# Lifecycle tests for the mempool stack:
|
|
# - mempool (legacy install path; the frontend container)
|
|
# - mempool-api (orchestrator-managed; the backend api)
|
|
# - archy-mempool-db (orchestrator-managed; the mariadb)
|
|
# - archy-mempool-web (orchestrator-managed; the proxy/static layer)
|
|
#
|
|
# The mempool stack is split between the legacy install path (mempool itself)
|
|
# and orchestrator-managed sub-containers — see uses_orchestrator_install_flow
|
|
# in install.rs. Tests here treat them as one stack at the package.install/stop
|
|
# level, addressed by id "mempool". UI URL coverage is in ui-coverage.bats.
|
|
|
|
load '../lib/rpc.bash'
|
|
|
|
setup_file() {
|
|
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
|
|
export ARCHY_FORCE_LOGIN=1
|
|
rpc_login
|
|
unset ARCHY_FORCE_LOGIN
|
|
}
|
|
|
|
teardown_file() {
|
|
rpc_logout_local
|
|
}
|
|
|
|
mempool_components=(
|
|
"mempool-api"
|
|
"archy-mempool-db"
|
|
)
|
|
|
|
mempool_optional_components=(
|
|
"mempool"
|
|
"archy-mempool-web"
|
|
)
|
|
|
|
mempool_skip_if_absent() {
|
|
for c in "${mempool_components[@]}"; do
|
|
podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1 && return 0
|
|
done
|
|
skip "mempool stack not installed"
|
|
}
|
|
|
|
@test "container-list includes the core mempool components" {
|
|
run rpc_result container-list
|
|
[ "$status" -eq 0 ]
|
|
local found=0
|
|
for c in "${mempool_components[@]}"; do
|
|
if echo "$output" | jq -e --arg n "$c" '.[] | select(.name == $n)' >/dev/null; then
|
|
found=$((found + 1))
|
|
fi
|
|
done
|
|
(( found > 0 )) || skip "mempool stack not installed"
|
|
}
|
|
|
|
@test "every present mempool component reports a valid state" {
|
|
run rpc_result container-list
|
|
[ "$status" -eq 0 ]
|
|
local present=0
|
|
for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
|
|
local state
|
|
state=$(echo "$output" | jq -r --arg n "$c" '.[] | select(.name == $n) | .state')
|
|
[[ -n "$state" ]] || continue
|
|
present=$((present + 1))
|
|
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] \
|
|
|| fail "invalid state for $c: $state"
|
|
done
|
|
(( present > 0 )) || skip "mempool stack not installed"
|
|
}
|
|
|
|
@test "no orphan mempool-related containers beyond the known set" {
|
|
local total known
|
|
total=$(podman ps -a --format '{{.Names}}' \
|
|
| grep -Ec '^(mempool|archy-mempool)' || true)
|
|
known=$(podman ps -a --format '{{.Names}}' \
|
|
| grep -Ec '^(mempool|mempool-api|archy-mempool-db|archy-mempool-web)$' || true)
|
|
[ "$total" -eq "$known" ]
|
|
}
|
|
|
|
# ────────────────────────────────────────────────────────────────────
|
|
# Destructive tier — operate on the package id "mempool" which the
|
|
# legacy install path treats as the whole stack
|
|
# ────────────────────────────────────────────────────────────────────
|
|
|
|
@test "package.stop transitions mempool stack to stopped" {
|
|
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
|
|
mempool_skip_if_absent
|
|
|
|
run rpc_result package.stop '{"id":"mempool"}'
|
|
[ "$status" -eq 0 ]
|
|
|
|
# The frontend container is the user-visible target; supporting
|
|
# services may stay running depending on orchestrator policy.
|
|
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
|
|
run wait_for_container_status mempool stopped 60
|
|
[ "$status" -eq 0 ]
|
|
fi
|
|
}
|
|
|
|
@test "package.start brings mempool stack back to running" {
|
|
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
|
|
mempool_skip_if_absent
|
|
|
|
run rpc_result package.start '{"id":"mempool"}'
|
|
[ "$status" -eq 0 ]
|
|
|
|
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
|
|
run wait_for_container_status mempool running 180
|
|
[ "$status" -eq 0 ]
|
|
fi
|
|
}
|
|
|
|
@test "package.restart leaves mempool stack in running state" {
|
|
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
|
|
mempool_skip_if_absent
|
|
|
|
run rpc_result package.restart '{"id":"mempool"}'
|
|
[ "$status" -eq 0 ]
|
|
|
|
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
|
|
run wait_for_container_status mempool running 180
|
|
[ "$status" -eq 0 ]
|
|
fi
|
|
}
|
|
|
|
@test "mempool api backend remains queryable when stack is up" {
|
|
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
|
|
mempool_skip_if_absent
|
|
|
|
# mempool-api on :8999 — same probe required-stack.bats uses for parity.
|
|
# This case runs immediately after package.restart, so mempool-api has just
|
|
# dropped + must re-establish its electrs/bitcoin connection (it reports
|
|
# "offline" in the frontend during this window). Give it the same recovery
|
|
# budget the passing parity probes use (required-stack-destructive: 240s,
|
|
# package-update-smoke: 300s) — 180s was too tight for the post-restart path.
|
|
local deadline=$(( $(date +%s) + 300 ))
|
|
while (( $(date +%s) < deadline )); do
|
|
if curl -fsS -m 5 "http://127.0.0.1:8999/api/v1/backend-info" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 3
|
|
done
|
|
# NB: bats-assert's `fail` is not loaded in this file (only ../lib/rpc.bash),
|
|
# so emit + return non-zero directly rather than calling an undefined helper.
|
|
echo "mempool-api never responded on :8999 within 300s" >&2
|
|
return 1
|
|
}
|
|
|
|
# ────────────────────────────────────────────────────────────────────
|
|
# Cascade-destructive tier
|
|
# ────────────────────────────────────────────────────────────────────
|
|
|
|
@test "package.uninstall removes the mempool stack" {
|
|
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
|
|
mempool_skip_if_absent
|
|
|
|
run rpc_result package.uninstall '{"id":"mempool","preserve_data":true}'
|
|
[ "$status" -eq 0 ]
|
|
|
|
for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
|
|
if podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1; then
|
|
run wait_for_container_status "$c" absent 120
|
|
[ "$status" -eq 0 ] || fail "mempool component $c not removed by uninstall"
|
|
fi
|
|
done
|
|
}
|
|
|
|
@test "package.install restores the mempool stack" {
|
|
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
|
|
|
|
run rpc_result package.install '{"manifest_path":"mempool/manifest.yaml"}'
|
|
[ "$status" -eq 0 ]
|
|
|
|
# At minimum the core orchestrator-managed components must come back.
|
|
for c in "${mempool_components[@]}"; do
|
|
run wait_for_container_status "$c" running 240
|
|
[ "$status" -eq 0 ] || fail "mempool component $c never reached running after reinstall"
|
|
done
|
|
}
|