diff --git a/tests/multinode/lib/multinode.bash b/tests/multinode/lib/multinode.bash new file mode 100755 index 00000000..00734ed4 --- /dev/null +++ b/tests/multinode/lib/multinode.bash @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Multi-node RPC harness library. +# +# Unlike tests/lifecycle/lib/rpc.bash (which targets a single ARCHY_HOST), +# this drives N independent archipelago nodes in one run so we can exercise +# real node-to-node paths: federation sync over Tor, FIPS anchoring, etc. +# +# A "node handle" is a short label (e.g. A, B, alice). For each handle you +# register a base URL + UI password; the lib logs in and keeps that node's +# session/CSRF cookies in its own state file so calls never cross wires. +# +# Usage: +# source tests/multinode/lib/multinode.bash +# node_register A https://192.168.1.228 password123 +# node_register B http://192.168.1.116 'ThisIsWeb54321@' +# node_login A; node_login B +# node_rpc A node.tor-address +# node_result B federation.list-nodes +# +# Requires: curl, jq. +# +# Note: this is a library — it does NOT set shell options (set -u/-e), since +# that would leak into the sourcing script. Each function guards its own vars +# with ${var:-} defaults. Callers set their own options. + +# Where per-node session state lives (one file per handle). +MULTINODE_STATE_DIR="${MULTINODE_STATE_DIR:-/tmp/archy-multinode}" +mkdir -p "$MULTINODE_STATE_DIR" + +# handle -> base url / password, kept in associative arrays. +declare -gA _MN_URL +declare -gA _MN_PW +declare -gA _MN_SESSION +declare -gA _MN_CSRF + +# node_register HANDLE BASE_URL PASSWORD +node_register() { + local h="$1" url="$2" pw="$3" + _MN_URL[$h]="${url%/}" + _MN_PW[$h]="$pw" +} + +_mn_session_file() { echo "$MULTINODE_STATE_DIR/session-$1"; } + +# node_login HANDLE — authenticate and capture session + csrf cookies. +node_login() { + local h="$1" + local url="${_MN_URL[$h]:-}" pw="${_MN_PW[$h]:-}" + if [[ -z "$url" || -z "$pw" ]]; then + echo "node_login: handle '$h' not registered" >&2 + return 1 + fi + local headers; headers=$(mktemp) + local body + body=$(curl -sk -D "$headers" -X POST "${url}/rpc/v1" \ + -H 'Content-Type: application/json' \ + --data-raw "{\"jsonrpc\":\"2.0\",\"method\":\"auth.login\",\"params\":{\"password\":\"${pw}\"},\"id\":1}") + local err; err=$(echo "$body" | jq -r '.error.message // empty' 2>/dev/null) + if [[ -n "$err" ]]; then + echo "node_login[$h] failed: $err" >&2 + rm -f "$headers" + return 1 + fi + local session csrf + session=$(grep -i '^set-cookie: session=' "$headers" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r') + csrf=$(grep -i '^set-cookie: csrf_token=' "$headers" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r') + rm -f "$headers" + if [[ -z "$session" || -z "$csrf" ]]; then + echo "node_login[$h]: missing session/csrf cookie" >&2 + return 1 + fi + _MN_SESSION[$h]="$session" + _MN_CSRF[$h]="$csrf" + printf '%s\n%s\n' "$session" "$csrf" > "$(_mn_session_file "$h")" +} + +# node_rpc HANDLE METHOD [PARAMS_JSON] — raw JSON-RPC response on stdout. +node_rpc() { + local h="$1" method="$2" params="${3:-}" + local url="${_MN_URL[$h]:-}" + local session="${_MN_SESSION[$h]:-}" csrf="${_MN_CSRF[$h]:-}" + if [[ -z "$session" || -z "$csrf" ]] && [[ -f "$(_mn_session_file "$h")" ]]; then + mapfile -t lines < "$(_mn_session_file "$h")" + session="${lines[0]:-}"; csrf="${lines[1]:-}" + _MN_SESSION[$h]="$session"; _MN_CSRF[$h]="$csrf" + fi + local payload + if [[ -z "$params" ]]; then + payload=$(jq -nc --arg m "$method" '{jsonrpc:"2.0",method:$m,id:1}') + else + payload=$(jq -nc --arg m "$method" --argjson p "$params" '{jsonrpc:"2.0",method:$m,params:$p,id:1}') + fi + curl -sk -X POST "${url}/rpc/v1" \ + -H 'Content-Type: application/json' \ + -H "Cookie: session=${session}; csrf_token=${csrf}" \ + -H "X-CSRF-Token: ${csrf}" \ + --data-raw "$payload" +} + +# node_result HANDLE METHOD [PARAMS_JSON] — .result on success; prints error to +# stderr and returns non-zero on RPC error. +node_result() { + local resp; resp=$(node_rpc "$@") + local err; err=$(echo "$resp" | jq -r '.error.message // empty' 2>/dev/null) + if [[ -n "$err" ]]; then + echo "node_result[$1 $2] error: $err" >&2 + return 1 + fi + echo "$resp" | jq '.result' +} + +# node_onion HANDLE — echo this node's own .onion address (empty if none). +node_onion() { + node_result "$1" node.tor-address 2>/dev/null | jq -r '. // empty | if type=="object" then (.onion // .address // .tor_address // empty) else . end' 2>/dev/null +} diff --git a/tests/multinode/repro-federation-sync.sh b/tests/multinode/repro-federation-sync.sh new file mode 100755 index 00000000..9492204f --- /dev/null +++ b/tests/multinode/repro-federation-sync.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Controlled two-node reproduction of node-to-node federation sync. +# +# Pairs two real nodes via federation.invite/join, triggers federation.sync-state +# in both directions, and reports which transport actually carried the call and +# any per-peer error. This is the controlled repro for the reported +# "Tor connection cloud->node not working" symptom: raw Tor transport is known +# good (see README), so this isolates whether the APP-level sync path works and, +# if it fails, surfaces the exact error string. +# +# Env (override as needed): +# A_URL A_PW node A base url + UI password (default .116 http) +# B_URL B_PW node B base url + UI password (default .228 https) +# FORCE_TOR=1 set both nodes' federation transport preference to Tor first +# +# Usage: tests/multinode/repro-federation-sync.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$HERE/lib/multinode.bash" + +A_URL="${A_URL:-http://192.168.1.116}"; A_PW="${A_PW:-ThisIsWeb54321@}" +B_URL="${B_URL:-https://192.168.1.228}"; B_PW="${B_PW:-password123}" + +bar() { printf '\n=== %s ===\n' "$*"; } + +node_register A "$A_URL" "$A_PW" +node_register B "$B_URL" "$B_PW" + +bar "login" +node_login A || { echo "A login failed"; exit 1; } +node_login B || { echo "B login failed"; exit 1; } +echo "A=$A_URL B=$B_URL logged in" + +bar "onions" +A_ONION=$(node_onion A); B_ONION=$(node_onion B) +echo "A onion: ${A_ONION:-}" +echo "B onion: ${B_ONION:-}" + +if [[ "${FORCE_TOR:-0}" == "1" ]]; then + bar "force federation transport = tor on both" + node_rpc A transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error' + node_rpc B transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error' +fi + +bar "federation state BEFORE" +echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)" +echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)" + +bar "pair: A invites, B joins" +INV_A=$(node_result A federation.invite) +CODE_A=$(echo "$INV_A" | jq -r '.code // empty') +echo "A invite code: ${CODE_A:0:40}…" +if [[ -n "$CODE_A" ]]; then + node_result B federation.join "$(jq -nc --arg c "$CODE_A" '{code:$c}')" \ + && echo "B joined A" || echo "B join FAILED" +fi + +bar "pair: B invites, A joins" +INV_B=$(node_result B federation.invite) +CODE_B=$(echo "$INV_B" | jq -r '.code // empty') +echo "B invite code: ${CODE_B:0:40}…" +if [[ -n "$CODE_B" ]]; then + node_result A federation.join "$(jq -nc --arg c "$CODE_B" '{code:$c}')" \ + && echo "A joined B" || echo "A join FAILED" +fi + +bar "trigger sync-state on A (A dials its peers)" +node_result A federation.sync-state | jq '.' + +bar "trigger sync-state on B (B dials its peers)" +node_result B federation.sync-state | jq '.' + +bar "federation state AFTER (look for fresh last_seen + transport)" +echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null +echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null + +bar "done" diff --git a/tests/multinode/smoke.sh b/tests/multinode/smoke.sh new file mode 100644 index 00000000..16aaca0e --- /dev/null +++ b/tests/multinode/smoke.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Two-node (optionally three-node) end-to-end smoke suite for the full app. +# +# Unlike repro-federation-sync.sh (a diagnostic that just prints state), this +# is an ASSERTION suite: every check is pass/fail and the script exits non-zero +# if any required check fails. It exercises the real node-to-node surface and +# specifically guards the bugs fixed in v1.7.94 / v1.7.95: +# - FIPS auto-connects to the public anchor (v1.7.94) +# - peer content browse works over the mesh, not just Tor (v1.7.95 — the +# `/content` catalog used to 404 over FIPS and never fall back to Tor) +# - a removed federation node stays removed, incl. transitive re-discovery +# (v1.7.95 tombstone) — the transitive case needs node C. +# +# Nodes (override via env): +# A_URL A_PW node A (default .116 http) +# B_URL B_PW node B (default .228 https) +# C_URL C_PW node C (OPTIONAL — enables the transitive-tombstone test) +# +# Requires both nodes on v1.7.95-alpha+ for the content-browse and tombstone +# checks; older peers SKIP those (reported, not failed). +# +# Usage: +# tests/multinode/smoke.sh +# A_URL=http://192.168.1.116 B_URL=https://192.168.1.228 tests/multinode/smoke.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$HERE/lib/multinode.bash" + +A_URL="${A_URL:-http://192.168.1.116}"; A_PW="${A_PW:-ThisIsWeb54321@}" +B_URL="${B_URL:-https://192.168.1.228}"; B_PW="${B_PW:-password123}" +C_URL="${C_URL:-}"; C_PW="${C_PW:-}" + +# ── tiny assertion framework ────────────────────────────────────────────── +PASS=0; FAIL=0; SKIP=0 +declare -a FAILED_NAMES +green() { printf '\033[32m%s\033[0m' "$*"; } +red() { printf '\033[31m%s\033[0m' "$*"; } +yellow(){ printf '\033[33m%s\033[0m' "$*"; } +section() { printf '\n\033[1m── %s ──\033[0m\n' "$*"; } +ok() { printf ' %s %s\n' "$(green ✓)" "$1"; PASS=$((PASS+1)); } +no() { printf ' %s %s\n' "$(red ✗)" "$1"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); } +skip() { printf ' %s %s (%s)\n' "$(yellow —)" "$1" "$2"; SKIP=$((SKIP+1)); } +# assert_eq NAME EXPECTED ACTUAL +assert_eq() { [[ "$2" == "$3" ]] && ok "$1" || no "$1 (expected '$2', got '$3')"; } +# assert_true NAME VALUE — passes when VALUE is "true" +assert_true() { [[ "$2" == "true" ]] && ok "$1" || no "$1 (got '$2')"; } + +# did_of HANDLE — this node's own DID via node.did (string or {did:...}). +did_of() { + node_result "$1" node.did 2>/dev/null \ + | jq -r 'if type=="string" then . elif type=="object" then (.did // .node_did // empty) else empty end' 2>/dev/null +} + +# pair HANDLE_INVITER HANDLE_JOINER — invite + join one direction. Echo "ok"/"fail". +pair() { + local inv code + inv=$(node_result "$1" federation.invite 2>/dev/null) + code=$(echo "$inv" | jq -r '.code // empty' 2>/dev/null) + [[ -z "$code" ]] && { echo "fail"; return; } + if node_result "$2" federation.join "$(jq -nc --arg c "$code" '{code:$c}')" >/dev/null 2>&1; then + echo "ok" + else echo "fail"; fi +} + +node_register A "$A_URL" "$A_PW" +node_register B "$B_URL" "$B_PW" +HAVE_C=0 +if [[ -n "$C_URL" && -n "$C_PW" ]]; then node_register C "$C_URL" "$C_PW"; HAVE_C=1; fi + +# ── 1. reachability + auth ──────────────────────────────────────────────── +section "reachability + login" +node_login A && ok "A login ($A_URL)" || { no "A login ($A_URL)"; echo "A unreachable — aborting"; exit 1; } +node_login B && ok "B login ($B_URL)" || { no "B login ($B_URL)"; echo "B unreachable — aborting"; exit 1; } +if [[ $HAVE_C == 1 ]]; then node_login C && ok "C login ($C_URL)" || { no "C login"; HAVE_C=0; }; fi + +A_ONION=$(node_onion A); B_ONION=$(node_onion B) +[[ -n "$A_ONION" ]] && ok "A has onion address" || no "A has onion address" +[[ -n "$B_ONION" ]] && ok "B has onion address" || no "B has onion address" + +# ── 2. FIPS mesh: daemon up + anchor connected (v1.7.94) ────────────────── +section "FIPS mesh / anchor" +for h in A B; do + s=$(node_result "$h" fips.status 2>/dev/null) + if [[ -z "$s" ]]; then skip "$h fips.status" "no FIPS RPC (old build?)"; continue; fi + assert_true "$h FIPS service active" "$(echo "$s" | jq -r '.service_active')" + ac=$(echo "$s" | jq -r '.anchor_connected') + if [[ "$ac" == "true" ]]; then ok "$h anchor connected" + else skip "$h anchor connected" "anchor_connected=$ac — node may need v1.7.94 + a moment to handshake"; fi +done + +# ── 3. federation pairing (both directions) ─────────────────────────────── +section "federation pairing" +assert_eq "A invites, B joins" "ok" "$(pair A B)" +assert_eq "B invites, A joins" "ok" "$(pair B A)" +# both should now list each other +node_result A federation.sync-state >/dev/null 2>&1 +node_result B federation.sync-state >/dev/null 2>&1 +A_SEES_B=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg o "${B_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)') +B_SEES_A=$(node_result B federation.list-nodes 2>/dev/null | jq -r --arg o "${A_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)') +assert_true "A's node list contains B" "$A_SEES_B" +assert_true "B's node list contains A" "$B_SEES_A" + +# ── 4. peer content browse over the mesh (v1.7.95 fix) ──────────────────── +section "peer content browse (was: 404 over mesh, no Tor fallback)" +if [[ -n "$B_ONION" ]]; then + resp=$(node_rpc A content.browse-peer "$(jq -nc --arg o "$B_ONION" '{onion:$o}')") + err=$(echo "$resp" | jq -r '.error.message // empty') + if [[ -z "$err" ]]; then + ok "A browses B's content catalog (HTTP 200)" + elif echo "$err" | grep -q '404'; then + no "A browses B's content — still 404 over mesh (is B on v1.7.95?): $err" + else + # Other errors (peer offline, no content shared) are environmental, not the bug. + skip "A browses B's content" "non-404 error: $err" + fi +else + skip "A browses B's content" "B has no onion" +fi + +# ── 5. removed-node tombstone (v1.7.95) ─────────────────────────────────── +section "removed-node tombstone" +B_DID=$(did_of B) +if [[ -z "$B_DID" ]]; then + skip "remove B then verify stays removed" "couldn't resolve B's DID" +else + if node_result A federation.remove-node "$(jq -nc --arg d "$B_DID" '{did:$d}')" >/dev/null 2>&1; then + still=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)') + assert_eq "B removed from A's list" "false" "$still" + # Transitive test needs C: A federated with B and C; C federated with B; + # A removes B; A syncs with C (who advertises B) → B must NOT reappear. + if [[ $HAVE_C == 1 ]]; then + pair A C >/dev/null; pair C A >/dev/null; pair C B >/dev/null + node_result A federation.sync-state >/dev/null 2>&1 + reappeared=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)') + assert_eq "B does NOT reappear via transitive sync with C" "false" "$reappeared" + else + skip "transitive reappear via 3rd node" "set C_URL/C_PW to enable" + fi + # re-add restores B (explicit re-add clears the tombstone) + pair B A >/dev/null + node_result A federation.sync-state >/dev/null 2>&1 + readded=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)') + assert_true "explicit re-pair brings B back (tombstone cleared)" "$readded" + else + skip "remove B" "remove-node RPC failed (B may already be absent)" + fi +fi + +# ── summary ─────────────────────────────────────────────────────────────── +section "summary" +printf ' %s passed, %s failed, %s skipped\n' "$(green $PASS)" "$([[ $FAIL -gt 0 ]] && red $FAIL || echo $FAIL)" "$(yellow $SKIP)" +if [[ $FAIL -gt 0 ]]; then + printf ' failed:\n'; for n in "${FAILED_NAMES[@]}"; do printf ' - %s\n' "$n"; done + exit 1 +fi +echo " all required checks passed"