test(multinode): assertion-based two-node E2E smoke suite
Adds tests/multinode/smoke.sh on the existing multinode.bash lib: an assertion suite (pass/fail + non-zero exit) driving two real nodes through login, onion + FIPS identity, FIPS anchor-connected, federation pairing both directions, peer content browse over the mesh, and the removed-node tombstone (with an optional 3rd node C for the transitive-reappear case). Guards the v1.7.94/v1.7.95 fixes. Content-browse + tombstone checks skip-with-note against peers older than v1.7.95. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
parent
e2c2f942c2
commit
0c8991b519
115
tests/multinode/lib/multinode.bash
Executable file
115
tests/multinode/lib/multinode.bash
Executable file
@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# Multi-node RPC harness library.
|
||||
#
|
||||
# Unlike tests/lifecycle/lib/rpc.bash (which targets a single ARCHY_HOST),
|
||||
# this drives N independent archipelago nodes in one run so we can exercise
|
||||
# real node-to-node paths: federation sync over Tor, FIPS anchoring, etc.
|
||||
#
|
||||
# A "node handle" is a short label (e.g. A, B, alice). For each handle you
|
||||
# register a base URL + UI password; the lib logs in and keeps that node's
|
||||
# session/CSRF cookies in its own state file so calls never cross wires.
|
||||
#
|
||||
# Usage:
|
||||
# source tests/multinode/lib/multinode.bash
|
||||
# node_register A https://192.168.1.228 password123
|
||||
# node_register B http://192.168.1.116 'ThisIsWeb54321@'
|
||||
# node_login A; node_login B
|
||||
# node_rpc A node.tor-address
|
||||
# node_result B federation.list-nodes
|
||||
#
|
||||
# Requires: curl, jq.
|
||||
#
|
||||
# Note: this is a library — it does NOT set shell options (set -u/-e), since
|
||||
# that would leak into the sourcing script. Each function guards its own vars
|
||||
# with ${var:-} defaults. Callers set their own options.
|
||||
|
||||
# Where per-node session state lives (one file per handle).
|
||||
MULTINODE_STATE_DIR="${MULTINODE_STATE_DIR:-/tmp/archy-multinode}"
|
||||
mkdir -p "$MULTINODE_STATE_DIR"
|
||||
|
||||
# handle -> base url / password, kept in associative arrays.
|
||||
declare -gA _MN_URL
|
||||
declare -gA _MN_PW
|
||||
declare -gA _MN_SESSION
|
||||
declare -gA _MN_CSRF
|
||||
|
||||
# node_register HANDLE BASE_URL PASSWORD
|
||||
node_register() {
|
||||
local h="$1" url="$2" pw="$3"
|
||||
_MN_URL[$h]="${url%/}"
|
||||
_MN_PW[$h]="$pw"
|
||||
}
|
||||
|
||||
_mn_session_file() { echo "$MULTINODE_STATE_DIR/session-$1"; }
|
||||
|
||||
# node_login HANDLE — authenticate and capture session + csrf cookies.
|
||||
node_login() {
|
||||
local h="$1"
|
||||
local url="${_MN_URL[$h]:-}" pw="${_MN_PW[$h]:-}"
|
||||
if [[ -z "$url" || -z "$pw" ]]; then
|
||||
echo "node_login: handle '$h' not registered" >&2
|
||||
return 1
|
||||
fi
|
||||
local headers; headers=$(mktemp)
|
||||
local body
|
||||
body=$(curl -sk -D "$headers" -X POST "${url}/rpc/v1" \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data-raw "{\"jsonrpc\":\"2.0\",\"method\":\"auth.login\",\"params\":{\"password\":\"${pw}\"},\"id\":1}")
|
||||
local err; err=$(echo "$body" | jq -r '.error.message // empty' 2>/dev/null)
|
||||
if [[ -n "$err" ]]; then
|
||||
echo "node_login[$h] failed: $err" >&2
|
||||
rm -f "$headers"
|
||||
return 1
|
||||
fi
|
||||
local session csrf
|
||||
session=$(grep -i '^set-cookie: session=' "$headers" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r')
|
||||
csrf=$(grep -i '^set-cookie: csrf_token=' "$headers" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r')
|
||||
rm -f "$headers"
|
||||
if [[ -z "$session" || -z "$csrf" ]]; then
|
||||
echo "node_login[$h]: missing session/csrf cookie" >&2
|
||||
return 1
|
||||
fi
|
||||
_MN_SESSION[$h]="$session"
|
||||
_MN_CSRF[$h]="$csrf"
|
||||
printf '%s\n%s\n' "$session" "$csrf" > "$(_mn_session_file "$h")"
|
||||
}
|
||||
|
||||
# node_rpc HANDLE METHOD [PARAMS_JSON] — raw JSON-RPC response on stdout.
|
||||
node_rpc() {
|
||||
local h="$1" method="$2" params="${3:-}"
|
||||
local url="${_MN_URL[$h]:-}"
|
||||
local session="${_MN_SESSION[$h]:-}" csrf="${_MN_CSRF[$h]:-}"
|
||||
if [[ -z "$session" || -z "$csrf" ]] && [[ -f "$(_mn_session_file "$h")" ]]; then
|
||||
mapfile -t lines < "$(_mn_session_file "$h")"
|
||||
session="${lines[0]:-}"; csrf="${lines[1]:-}"
|
||||
_MN_SESSION[$h]="$session"; _MN_CSRF[$h]="$csrf"
|
||||
fi
|
||||
local payload
|
||||
if [[ -z "$params" ]]; then
|
||||
payload=$(jq -nc --arg m "$method" '{jsonrpc:"2.0",method:$m,id:1}')
|
||||
else
|
||||
payload=$(jq -nc --arg m "$method" --argjson p "$params" '{jsonrpc:"2.0",method:$m,params:$p,id:1}')
|
||||
fi
|
||||
curl -sk -X POST "${url}/rpc/v1" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "Cookie: session=${session}; csrf_token=${csrf}" \
|
||||
-H "X-CSRF-Token: ${csrf}" \
|
||||
--data-raw "$payload"
|
||||
}
|
||||
|
||||
# node_result HANDLE METHOD [PARAMS_JSON] — .result on success; prints error to
|
||||
# stderr and returns non-zero on RPC error.
|
||||
node_result() {
|
||||
local resp; resp=$(node_rpc "$@")
|
||||
local err; err=$(echo "$resp" | jq -r '.error.message // empty' 2>/dev/null)
|
||||
if [[ -n "$err" ]]; then
|
||||
echo "node_result[$1 $2] error: $err" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "$resp" | jq '.result'
|
||||
}
|
||||
|
||||
# node_onion HANDLE — echo this node's own .onion address (empty if none).
|
||||
node_onion() {
|
||||
node_result "$1" node.tor-address 2>/dev/null | jq -r '. // empty | if type=="object" then (.onion // .address // .tor_address // empty) else . end' 2>/dev/null
|
||||
}
|
||||
77
tests/multinode/repro-federation-sync.sh
Executable file
77
tests/multinode/repro-federation-sync.sh
Executable file
@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# Controlled two-node reproduction of node-to-node federation sync.
|
||||
#
|
||||
# Pairs two real nodes via federation.invite/join, triggers federation.sync-state
|
||||
# in both directions, and reports which transport actually carried the call and
|
||||
# any per-peer error. This is the controlled repro for the reported
|
||||
# "Tor connection cloud->node not working" symptom: raw Tor transport is known
|
||||
# good (see README), so this isolates whether the APP-level sync path works and,
|
||||
# if it fails, surfaces the exact error string.
|
||||
#
|
||||
# Env (override as needed):
|
||||
# A_URL A_PW node A base url + UI password (default .116 http)
|
||||
# B_URL B_PW node B base url + UI password (default .228 https)
|
||||
# FORCE_TOR=1 set both nodes' federation transport preference to Tor first
|
||||
#
|
||||
# Usage: tests/multinode/repro-federation-sync.sh
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$HERE/lib/multinode.bash"
|
||||
|
||||
A_URL="${A_URL:-http://192.168.1.116}"; A_PW="${A_PW:-ThisIsWeb54321@}"
|
||||
B_URL="${B_URL:-https://192.168.1.228}"; B_PW="${B_PW:-password123}"
|
||||
|
||||
bar() { printf '\n=== %s ===\n' "$*"; }
|
||||
|
||||
node_register A "$A_URL" "$A_PW"
|
||||
node_register B "$B_URL" "$B_PW"
|
||||
|
||||
bar "login"
|
||||
node_login A || { echo "A login failed"; exit 1; }
|
||||
node_login B || { echo "B login failed"; exit 1; }
|
||||
echo "A=$A_URL B=$B_URL logged in"
|
||||
|
||||
bar "onions"
|
||||
A_ONION=$(node_onion A); B_ONION=$(node_onion B)
|
||||
echo "A onion: ${A_ONION:-<none>}"
|
||||
echo "B onion: ${B_ONION:-<none>}"
|
||||
|
||||
if [[ "${FORCE_TOR:-0}" == "1" ]]; then
|
||||
bar "force federation transport = tor on both"
|
||||
node_rpc A transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error'
|
||||
node_rpc B transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error'
|
||||
fi
|
||||
|
||||
bar "federation state BEFORE"
|
||||
echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)"
|
||||
echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)"
|
||||
|
||||
bar "pair: A invites, B joins"
|
||||
INV_A=$(node_result A federation.invite)
|
||||
CODE_A=$(echo "$INV_A" | jq -r '.code // empty')
|
||||
echo "A invite code: ${CODE_A:0:40}…"
|
||||
if [[ -n "$CODE_A" ]]; then
|
||||
node_result B federation.join "$(jq -nc --arg c "$CODE_A" '{code:$c}')" \
|
||||
&& echo "B joined A" || echo "B join FAILED"
|
||||
fi
|
||||
|
||||
bar "pair: B invites, A joins"
|
||||
INV_B=$(node_result B federation.invite)
|
||||
CODE_B=$(echo "$INV_B" | jq -r '.code // empty')
|
||||
echo "B invite code: ${CODE_B:0:40}…"
|
||||
if [[ -n "$CODE_B" ]]; then
|
||||
node_result A federation.join "$(jq -nc --arg c "$CODE_B" '{code:$c}')" \
|
||||
&& echo "A joined B" || echo "A join FAILED"
|
||||
fi
|
||||
|
||||
bar "trigger sync-state on A (A dials its peers)"
|
||||
node_result A federation.sync-state | jq '.'
|
||||
|
||||
bar "trigger sync-state on B (B dials its peers)"
|
||||
node_result B federation.sync-state | jq '.'
|
||||
|
||||
bar "federation state AFTER (look for fresh last_seen + transport)"
|
||||
echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null
|
||||
echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null
|
||||
|
||||
bar "done"
|
||||
156
tests/multinode/smoke.sh
Normal file
156
tests/multinode/smoke.sh
Normal file
@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
# Two-node (optionally three-node) end-to-end smoke suite for the full app.
|
||||
#
|
||||
# Unlike repro-federation-sync.sh (a diagnostic that just prints state), this
|
||||
# is an ASSERTION suite: every check is pass/fail and the script exits non-zero
|
||||
# if any required check fails. It exercises the real node-to-node surface and
|
||||
# specifically guards the bugs fixed in v1.7.94 / v1.7.95:
|
||||
# - FIPS auto-connects to the public anchor (v1.7.94)
|
||||
# - peer content browse works over the mesh, not just Tor (v1.7.95 — the
|
||||
# `/content` catalog used to 404 over FIPS and never fall back to Tor)
|
||||
# - a removed federation node stays removed, incl. transitive re-discovery
|
||||
# (v1.7.95 tombstone) — the transitive case needs node C.
|
||||
#
|
||||
# Nodes (override via env):
|
||||
# A_URL A_PW node A (default .116 http)
|
||||
# B_URL B_PW node B (default .228 https)
|
||||
# C_URL C_PW node C (OPTIONAL — enables the transitive-tombstone test)
|
||||
#
|
||||
# Requires both nodes on v1.7.95-alpha+ for the content-browse and tombstone
|
||||
# checks; older peers SKIP those (reported, not failed).
|
||||
#
|
||||
# Usage:
|
||||
# tests/multinode/smoke.sh
|
||||
# A_URL=http://192.168.1.116 B_URL=https://192.168.1.228 tests/multinode/smoke.sh
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$HERE/lib/multinode.bash"
|
||||
|
||||
A_URL="${A_URL:-http://192.168.1.116}"; A_PW="${A_PW:-ThisIsWeb54321@}"
|
||||
B_URL="${B_URL:-https://192.168.1.228}"; B_PW="${B_PW:-password123}"
|
||||
C_URL="${C_URL:-}"; C_PW="${C_PW:-}"
|
||||
|
||||
# ── tiny assertion framework ──────────────────────────────────────────────
|
||||
PASS=0; FAIL=0; SKIP=0
|
||||
declare -a FAILED_NAMES
|
||||
green() { printf '\033[32m%s\033[0m' "$*"; }
|
||||
red() { printf '\033[31m%s\033[0m' "$*"; }
|
||||
yellow(){ printf '\033[33m%s\033[0m' "$*"; }
|
||||
section() { printf '\n\033[1m── %s ──\033[0m\n' "$*"; }
|
||||
ok() { printf ' %s %s\n' "$(green ✓)" "$1"; PASS=$((PASS+1)); }
|
||||
no() { printf ' %s %s\n' "$(red ✗)" "$1"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); }
|
||||
skip() { printf ' %s %s (%s)\n' "$(yellow —)" "$1" "$2"; SKIP=$((SKIP+1)); }
|
||||
# assert_eq NAME EXPECTED ACTUAL
|
||||
assert_eq() { [[ "$2" == "$3" ]] && ok "$1" || no "$1 (expected '$2', got '$3')"; }
|
||||
# assert_true NAME VALUE — passes when VALUE is "true"
|
||||
assert_true() { [[ "$2" == "true" ]] && ok "$1" || no "$1 (got '$2')"; }
|
||||
|
||||
# did_of HANDLE — this node's own DID via node.did (string or {did:...}).
|
||||
did_of() {
|
||||
node_result "$1" node.did 2>/dev/null \
|
||||
| jq -r 'if type=="string" then . elif type=="object" then (.did // .node_did // empty) else empty end' 2>/dev/null
|
||||
}
|
||||
|
||||
# pair HANDLE_INVITER HANDLE_JOINER — invite + join one direction. Echo "ok"/"fail".
|
||||
pair() {
|
||||
local inv code
|
||||
inv=$(node_result "$1" federation.invite 2>/dev/null)
|
||||
code=$(echo "$inv" | jq -r '.code // empty' 2>/dev/null)
|
||||
[[ -z "$code" ]] && { echo "fail"; return; }
|
||||
if node_result "$2" federation.join "$(jq -nc --arg c "$code" '{code:$c}')" >/dev/null 2>&1; then
|
||||
echo "ok"
|
||||
else echo "fail"; fi
|
||||
}
|
||||
|
||||
node_register A "$A_URL" "$A_PW"
|
||||
node_register B "$B_URL" "$B_PW"
|
||||
HAVE_C=0
|
||||
if [[ -n "$C_URL" && -n "$C_PW" ]]; then node_register C "$C_URL" "$C_PW"; HAVE_C=1; fi
|
||||
|
||||
# ── 1. reachability + auth ────────────────────────────────────────────────
|
||||
section "reachability + login"
|
||||
node_login A && ok "A login ($A_URL)" || { no "A login ($A_URL)"; echo "A unreachable — aborting"; exit 1; }
|
||||
node_login B && ok "B login ($B_URL)" || { no "B login ($B_URL)"; echo "B unreachable — aborting"; exit 1; }
|
||||
if [[ $HAVE_C == 1 ]]; then node_login C && ok "C login ($C_URL)" || { no "C login"; HAVE_C=0; }; fi
|
||||
|
||||
A_ONION=$(node_onion A); B_ONION=$(node_onion B)
|
||||
[[ -n "$A_ONION" ]] && ok "A has onion address" || no "A has onion address"
|
||||
[[ -n "$B_ONION" ]] && ok "B has onion address" || no "B has onion address"
|
||||
|
||||
# ── 2. FIPS mesh: daemon up + anchor connected (v1.7.94) ──────────────────
|
||||
section "FIPS mesh / anchor"
|
||||
for h in A B; do
|
||||
s=$(node_result "$h" fips.status 2>/dev/null)
|
||||
if [[ -z "$s" ]]; then skip "$h fips.status" "no FIPS RPC (old build?)"; continue; fi
|
||||
assert_true "$h FIPS service active" "$(echo "$s" | jq -r '.service_active')"
|
||||
ac=$(echo "$s" | jq -r '.anchor_connected')
|
||||
if [[ "$ac" == "true" ]]; then ok "$h anchor connected"
|
||||
else skip "$h anchor connected" "anchor_connected=$ac — node may need v1.7.94 + a moment to handshake"; fi
|
||||
done
|
||||
|
||||
# ── 3. federation pairing (both directions) ───────────────────────────────
|
||||
section "federation pairing"
|
||||
assert_eq "A invites, B joins" "ok" "$(pair A B)"
|
||||
assert_eq "B invites, A joins" "ok" "$(pair B A)"
|
||||
# both should now list each other
|
||||
node_result A federation.sync-state >/dev/null 2>&1
|
||||
node_result B federation.sync-state >/dev/null 2>&1
|
||||
A_SEES_B=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg o "${B_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)')
|
||||
B_SEES_A=$(node_result B federation.list-nodes 2>/dev/null | jq -r --arg o "${A_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)')
|
||||
assert_true "A's node list contains B" "$A_SEES_B"
|
||||
assert_true "B's node list contains A" "$B_SEES_A"
|
||||
|
||||
# ── 4. peer content browse over the mesh (v1.7.95 fix) ────────────────────
|
||||
section "peer content browse (was: 404 over mesh, no Tor fallback)"
|
||||
if [[ -n "$B_ONION" ]]; then
|
||||
resp=$(node_rpc A content.browse-peer "$(jq -nc --arg o "$B_ONION" '{onion:$o}')")
|
||||
err=$(echo "$resp" | jq -r '.error.message // empty')
|
||||
if [[ -z "$err" ]]; then
|
||||
ok "A browses B's content catalog (HTTP 200)"
|
||||
elif echo "$err" | grep -q '404'; then
|
||||
no "A browses B's content — still 404 over mesh (is B on v1.7.95?): $err"
|
||||
else
|
||||
# Other errors (peer offline, no content shared) are environmental, not the bug.
|
||||
skip "A browses B's content" "non-404 error: $err"
|
||||
fi
|
||||
else
|
||||
skip "A browses B's content" "B has no onion"
|
||||
fi
|
||||
|
||||
# ── 5. removed-node tombstone (v1.7.95) ───────────────────────────────────
|
||||
section "removed-node tombstone"
|
||||
B_DID=$(did_of B)
|
||||
if [[ -z "$B_DID" ]]; then
|
||||
skip "remove B then verify stays removed" "couldn't resolve B's DID"
|
||||
else
|
||||
if node_result A federation.remove-node "$(jq -nc --arg d "$B_DID" '{did:$d}')" >/dev/null 2>&1; then
|
||||
still=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
|
||||
assert_eq "B removed from A's list" "false" "$still"
|
||||
# Transitive test needs C: A federated with B and C; C federated with B;
|
||||
# A removes B; A syncs with C (who advertises B) → B must NOT reappear.
|
||||
if [[ $HAVE_C == 1 ]]; then
|
||||
pair A C >/dev/null; pair C A >/dev/null; pair C B >/dev/null
|
||||
node_result A federation.sync-state >/dev/null 2>&1
|
||||
reappeared=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
|
||||
assert_eq "B does NOT reappear via transitive sync with C" "false" "$reappeared"
|
||||
else
|
||||
skip "transitive reappear via 3rd node" "set C_URL/C_PW to enable"
|
||||
fi
|
||||
# re-add restores B (explicit re-add clears the tombstone)
|
||||
pair B A >/dev/null
|
||||
node_result A federation.sync-state >/dev/null 2>&1
|
||||
readded=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
|
||||
assert_true "explicit re-pair brings B back (tombstone cleared)" "$readded"
|
||||
else
|
||||
skip "remove B" "remove-node RPC failed (B may already be absent)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────────────────
|
||||
section "summary"
|
||||
printf ' %s passed, %s failed, %s skipped\n' "$(green $PASS)" "$([[ $FAIL -gt 0 ]] && red $FAIL || echo $FAIL)" "$(yellow $SKIP)"
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
printf ' failed:\n'; for n in "${FAILED_NAMES[@]}"; do printf ' - %s\n' "$n"; done
|
||||
exit 1
|
||||
fi
|
||||
echo " all required checks passed"
|
||||
Loading…
x
Reference in New Issue
Block a user