archy/scripts/deploy-to-target.sh

#!/bin/bash
#
# Deploy Archipelago code to the HP ProDesk target
#
# Usage:
#   ./scripts/deploy-to-target.sh                # Sync and rebuild
#   ./scripts/deploy-to-target.sh --quick        # Sync only, no rebuild
#   ./scripts/deploy-to-target.sh --live         # Deploy to live system (default: 192.168.1.228)
#   ./scripts/deploy-to-target.sh --both         # Deploy to 228, then copy to 198 + 253
#   ./scripts/deploy-to-target.sh --frontend-only  # Frontend-only deploy (skip Rust build + container rebuilds)
#   ./scripts/deploy-to-target.sh --demo           # Demo mode: Bitcoin pruning enabled (smaller disk)
#   ./scripts/deploy-to-target.sh --dry-run --live # Show what would be deployed without executing
#   ./scripts/deploy-to-target.sh --tailscale        # Deploy to all 3 Tailscale alpha tester nodes
#   ./scripts/deploy-to-target.sh --tailscale-node=arch2  # Deploy to a specific Tailscale node
#

set -eo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

# Load deploy config (password etc.) - deploy-config.sh is gitignored
[ -f "$SCRIPT_DIR/deploy-config.sh" ] && . "$SCRIPT_DIR/deploy-config.sh"

# Source pinned image versions (single source of truth)
[ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh"

# Source shared utility library
[ -f "$SCRIPT_DIR/lib/common.sh" ] && . "$SCRIPT_DIR/lib/common.sh"

# Configuration
TARGET_HOST="${ARCHIPELAGO_TARGET:-archipelago@192.168.1.228}"
TARGET_DIR="/home/archipelago/archy"
SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -i $SSH_KEY"

DEPLOY_START=$(date +%s)
timestamp() { echo "[$(date +%H:%M:%S)]"; }

echo "╔════════════════════════════════════════════════════════════════╗"
echo "║  Deploying to Archipelago Target                              ║"
echo "╚════════════════════════════════════════════════════════════════╝"
echo ""
echo "$(timestamp) Target: $TARGET_HOST"
echo ""

# Parse arguments
QUICK=false
LIVE=false
BOTH=false
FRONTEND_ONLY=false
DEMO=false
DRY_RUN=false
CANARY=false
TAILSCALE=false
TAILSCALE_NODE=""
FLEET=false
RESET_MESH=false
for arg in "$@"; do
    case $arg in
        --quick) QUICK=true ;;
        --live) LIVE=true ;;
        --both) BOTH=true ;;
        --frontend-only) FRONTEND_ONLY=true; LIVE=true ;;
        --demo) DEMO=true ;;
        --dry-run) DRY_RUN=true ;;
        --canary) CANARY=true ;;
        --tailscale) TAILSCALE=true ;;
        --tailscale-node=*) TAILSCALE_NODE="${arg#*=}" ;;
        --fleet) FLEET=true ;;
        --all) FLEET=true ;;
        --reset-mesh) RESET_MESH=true ;;
    esac
done

# Fleet deploy: .228 → .198 → all 3 Tailscale nodes (all 5 servers)
if [ "$FLEET" = true ]; then
    echo "╔════════════════════════════════════════════════════════════════╗"
    echo "║  FLEET DEPLOY — All nodes                                    ║"
    echo "╚════════════════════════════════════════════════════════════════╝"
    echo ""
    echo "Phase 1: Build + deploy to .228 (primary build server)"
    # Try LAN first, fall back to Tailscale IP
    if ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1; then
        "$0" --live || { echo "FAILED: .228 deploy"; exit 1; }
    elif [ -n "${TAILSCALE_PRIMARY:-}" ] || tailscale status >/dev/null 2>&1; then
        TS_PRIMARY="${TAILSCALE_PRIMARY:-$(tailscale status 2>/dev/null | grep 'archipelago-0' | awk '{print $1}')}"
        if [ -n "$TS_PRIMARY" ]; then
            echo "   LAN unreachable — using Tailscale IP $TS_PRIMARY"
            ARCHIPELAGO_TARGET="archipelago@${TS_PRIMARY}" "$0" --live || { echo "FAILED: .228 deploy via Tailscale"; exit 1; }
        else
            echo "FAILED: .228 unreachable on LAN or Tailscale"; exit 1
        fi
    else
        echo "FAILED: .228 unreachable"; exit 1
    fi
    echo ""
    echo "Phase 2: Copy to .198 + .253 (LAN secondaries — skip if unreachable)"
    "$0" --both 2>/dev/null || echo "   LAN secondaries unreachable, skipping"
    echo ""
    echo "Phase 3: Deploy to all Tailscale nodes (Arch 1/2/3)"
    "$SCRIPT_DIR/deploy-tailscale.sh" --all || { echo "WARNING: Some Tailscale nodes failed"; }
    echo ""
    echo "════════════════════════════════════════════════════════════════"
    echo "Fleet deploy complete."
    exit 0
fi

# Tailscale deploy: delegate to deploy-tailscale.sh
if [ "$TAILSCALE" = true ]; then
    echo "Deploying to all Tailscale nodes..."
    exec "$SCRIPT_DIR/deploy-tailscale.sh" --all
fi
if [ -n "$TAILSCALE_NODE" ]; then
    echo "Deploying to Tailscale node: $TAILSCALE_NODE"
    exec "$SCRIPT_DIR/deploy-tailscale.sh" "$TAILSCALE_NODE"
fi

# Deploy locking — prevent concurrent deploys to the same target
TARGET_IP_FOR_LOCK="$(echo "$TARGET_HOST" | cut -d@ -f2)"
LOCK_DIR="/tmp/archipelago-deploy-${TARGET_IP_FOR_LOCK}.lock"
# Check for stale lock (older than 30 minutes)
if [ -d "$LOCK_DIR" ]; then
    LOCK_STAMP="$LOCK_DIR/pid"
    if [ -f "$LOCK_STAMP" ]; then
        # macOS uses stat -f %m, Linux uses stat -c %Y
        if stat -c %Y "$LOCK_STAMP" >/dev/null 2>&1; then
            LOCK_MTIME=$(stat -c %Y "$LOCK_STAMP")
        else
            LOCK_MTIME=$(stat -f %m "$LOCK_STAMP")
        fi
        LOCK_AGE=$(( $(date +%s) - ${LOCK_MTIME:-0} ))
        if [ "$LOCK_AGE" -gt 1800 ]; then
            echo "$(timestamp) WARNING: Removing stale lock (${LOCK_AGE}s old)"
            rm -rf "$LOCK_DIR"
        fi
    fi
fi
# mkdir is atomic — fails if directory already exists
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
    echo "ERROR: Deploy already in progress for $TARGET_HOST (lock: $LOCK_DIR)"
    exit 1
fi
echo $$ > "$LOCK_DIR"/pid
# Temp directory for intermediate files (cleaned up on exit)
TMPDIR="/tmp/archipelago-deploy-$$"
mkdir -p "$TMPDIR"
# Clean up lock and temp files on exit (normal, error, or signal)
cleanup_deploy() { rm -rf "$LOCK_DIR" "$TMPDIR"; }
trap cleanup_deploy EXIT

# Dry run mode: show what would be deployed without executing
if [[ "$DRY_RUN" == "true" ]]; then
    echo "═══ DRY RUN MODE — no changes will be made ═══"
    echo ""
    echo "Target: $TARGET_HOST"
    echo "Project: $PROJECT_DIR"
    echo "Mode: $(
        [[ "$BOTH" == "true" ]] && echo "both (.228 + .198)" || \
        [[ "$LIVE" == "true" ]] && echo "live (.228)" || \
        echo "dev (sync + build)"
    )"
    echo ""
    echo "Files that would be synced:"
    rsync -avn --exclude '.git' --exclude 'target' --exclude 'node_modules' \
        --exclude 'dist' --exclude 'web/dist' --exclude '*.iso' \
        --exclude 'image-recipe/_archived/build' --exclude 'image-recipe/_archived/results' \
        --exclude '.codex-target-*' --exclude '.codex-tmp' --exclude 'uploads' \
        "$PROJECT_DIR/" "$TARGET_HOST:$TARGET_DIR/" 2>/dev/null | \
        grep -E '^[<>]|^deleting' | head -50 || echo "  (rsync check failed — SSH may be unavailable)"
    echo ""
    echo "Frontend build: $(
        [[ "$QUICK" == "true" ]] && echo "SKIP (--quick)" || echo "vue-tsc + vite build"
    )"
    echo "Backend build: $(
        [[ "$FRONTEND_ONLY" == "true" ]] && echo "SKIP (--frontend-only)" || \
        [[ "$QUICK" == "true" ]] && echo "SKIP (--quick)" || echo "cargo build --release"
    )"
    echo "Live deploy: $(
        [[ "$LIVE" == "true" || "$BOTH" == "true" ]] && echo "YES — binary + frontend + nginx + systemd" || echo "NO"
    )"
    echo ""
    echo "═══ DRY RUN COMPLETE — nothing was changed ═══"
    exit 0
fi

# Section timing helper
section_start() { SECTION_START=$(date +%s); }
section_end() {
    local elapsed=$(($(date +%s) - SECTION_START))
    echo "   (${elapsed}s)"
}

# ── Progress bar ──────────────────────────────────────────────
CURRENT_STEP=0
BAR_WIDTH=30

calculate_total_steps() {
    local total=4  # SSH, prereqs, health, git state

    if [[ "$QUICK" == "true" ]]; then
        total=$((total + 1))  # sync only
        echo $total; return
    fi

    total=$((total + 1))  # sync code
    total=$((total + 1))  # frontend build

    if [[ "$FRONTEND_ONLY" != "true" ]]; then
        total=$((total + 1))  # backend build
    fi

    if [[ "$LIVE" == "true" ]]; then
        total=$((total + 14))  # rollback, frontend, AIUI, nginx, systemd, claude proxy, dev mode, data dirs, nostr-provider, filebrowser, manifest, restart, HTTPS, health check
        if [[ "$FRONTEND_ONLY" != "true" ]]; then
            total=$((total + 1))   # deploy backend binary
            total=$((total + 16))  # container rebuilds
        fi
        total=$((total + 3))  # UFW, IndeedHub fix, container doctor
    fi

    echo $total
}

TOTAL_STEPS=$(calculate_total_steps)

progress() {
    CURRENT_STEP=$((CURRENT_STEP + 1))
    local pct=$((CURRENT_STEP * 100 / TOTAL_STEPS))
    local filled=$((pct * BAR_WIDTH / 100))
    local empty=$((BAR_WIDTH - filled))
    local bar
    bar=$(printf '%*s' "$filled" '' | tr ' ' '█')$(printf '%*s' "$empty" '' | tr ' ' '░')
    printf "\033[1;36m━━━ [%s] %3d%% (%d/%d)\033[0m %s\n" "$bar" "$pct" "$CURRENT_STEP" "$TOTAL_STEPS" "$1"
}
# ─────────────────────────────────────────────────────────────

# SSH connectivity pre-flight check
progress "Checking SSH connectivity"
if ! ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1; then
    echo "   ERROR: Cannot connect to $TARGET_HOST"
    echo "   Check that the server is on and reachable."
    exit 1
fi
echo "   Connected."

# Disk space pre-flight — abort if target is dangerously full
DISK_PCT=$(ssh $SSH_OPTS "$TARGET_HOST" "df / | tail -1 | awk '{print \$(NF-1)}' | tr -d '%'" 2>/dev/null)
if [ -n "$DISK_PCT" ] && [ "$DISK_PCT" -gt 85 ] 2>/dev/null; then
  echo "ERROR: Target disk at ${DISK_PCT}% — need <85% for safe deploy. Free space and retry."
  exit 1
fi

# Install prerequisites if missing (rsync for code sync, python3 for Claude API proxy)
progress "Checking prerequisites"
ssh $SSH_OPTS "$TARGET_HOST" '
  NEED_INSTALL=""
  command -v rsync >/dev/null 2>&1 || NEED_INSTALL="$NEED_INSTALL rsync"
  command -v python3 >/dev/null 2>&1 || NEED_INSTALL="$NEED_INSTALL python3"
  if ! command -v node >/dev/null 2>&1 || ! command -v npm >/dev/null 2>&1; then
    echo "   Node.js/npm not found — installing..."
    curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - 2>&1 | tail -3
    NEED_INSTALL="$NEED_INSTALL nodejs"
  fi
  if [ -n "$NEED_INSTALL" ]; then
    echo "   Installing:$NEED_INSTALL"
    sudo apt-get update -qq && sudo apt-get install -y -qq $NEED_INSTALL 2>&1 | tail -3
  else
    echo "   All prerequisites present"
  fi
' 2>&1

# Pre-deploy health check (informational — warns but does not block)
progress "Pre-deploy health check"
TARGET_IP_ONLY="$(echo "$TARGET_HOST" | cut -d@ -f2)"
PRE_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || { echo "WARNING: Pre-deploy health check failed for $TARGET_IP_ONLY" >&2; echo "000"; })
if [ "$PRE_HEALTH" = "200" ]; then
    echo "   Server health: OK (200)"
else
    echo "   ⚠️  Server health: $PRE_HEALTH (may be down or unhealthy — deploying anyway)"
fi
echo ""

# Git state check — detect uncommitted changes and record deploy version
progress "Checking git state"
DEPLOY_COMMIT=$(git -C "$PROJECT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
DEPLOY_COMMIT_FULL=$(git -C "$PROJECT_DIR" rev-parse HEAD 2>/dev/null || echo "unknown")
DEPLOY_BRANCH=$(git -C "$PROJECT_DIR" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
DIRTY_FILES=$(git -C "$PROJECT_DIR" status --porcelain 2>/dev/null | grep -v '^??' | grep -v '\.claude/memory/' || true)
DEPLOY_DIRTY=false

echo "$(timestamp) Git state: $DEPLOY_BRANCH @ $DEPLOY_COMMIT"
if [ -n "$DIRTY_FILES" ]; then
    DEPLOY_DIRTY=true
    DIRTY_COUNT=$(echo "$DIRTY_FILES" | wc -l | tr -d ' ')
    echo "   ⚠️  WARNING: $DIRTY_COUNT uncommitted change(s) — deploying working directory, NOT last commit"
    echo "$DIRTY_FILES" | head -10 | sed 's/^/   /'
    [ "$DIRTY_COUNT" -gt 10 ] && echo "   ... and $((DIRTY_COUNT - 10)) more"
    echo ""
    echo "   To deploy clean: commit or stash changes first"
    echo "   Continuing in 3 seconds... (Ctrl+C to abort)"
    sleep 3
else
    echo "   Working tree clean — deploying commit $DEPLOY_COMMIT"
fi
echo ""

# When --canary: deploy to 198 first, verify health, then deploy to 228
if [ "$CANARY" = true ]; then
    echo "🐤 Canary deploy: .198 first, then .228 if healthy..."
    echo ""

    # Deploy to .228 (builds code), then copy to .198
    "$0" --both

    # Verify .198 is healthy before declaring success
    echo ""
    echo "🐤 Canary check: verifying .198 health..."
    CANARY_OK=false
    for i in $(seq 1 12); do
        sleep 5
        CANARY_HEALTH=$(curl -s --max-time 5 "http://192.168.1.198/health" 2>/dev/null || { echo "WARNING: Canary health check failed for 192.168.1.198" >&2; echo ""; })
        if [ "$CANARY_HEALTH" = "OK" ]; then
            echo "   ✅ Canary .198 healthy after $((i * 5))s"
            CANARY_OK=true
            break
        fi
    done

    if [ "$CANARY_OK" != "true" ]; then
        echo "   ❌ Canary .198 FAILED health check after 60s"
        echo "   ⚠️  .228 was also deployed. Check both servers."
        exit 1
    fi

    echo "🐤 Canary deploy complete — both nodes healthy"
    exit 0
fi

# ── deploy_secondary: copy built binary+frontend from .228 to a secondary node ──
# Usage: deploy_secondary <user@ip> <short_label>  (e.g. deploy_secondary archipelago@192.168.1.198 198)
deploy_secondary() {
    local SEC_TARGET="$1"
    local SEC_LABEL="$2"
    local SEC_IP="${SEC_TARGET#*@}"

    echo ""
    echo "📤 Copying to $SEC_IP (no rsync/cargo on that node)..."

    scp $SSH_OPTS /tmp/archipelago-both "$SEC_TARGET:/tmp/archipelago-new"
    ssh $SSH_OPTS "archipelago@192.168.1.228" "cd '$TARGET_DIR' && tar cf - web/dist/neode-ui 2>/dev/null" | ssh $SSH_OPTS "$SEC_TARGET" "mkdir -p /tmp/web-deploy && cd /tmp/web-deploy && tar xf -"
    ssh $SSH_OPTS "$SEC_TARGET" '
        sudo systemctl stop archipelago
        sudo cp /tmp/archipelago-new /usr/local/bin/archipelago
        sudo chmod +x /usr/local/bin/archipelago
        rm -f /tmp/archipelago-new
        sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +
        sudo cp -r /tmp/web-deploy/web/dist/neode-ui/* /opt/archipelago/web-ui/ 2>/dev/null || true
        sudo chown -R 1000:1000 /opt/archipelago/web-ui
    '

    # Deploy AIUI — prefer a sibling dist on the build host, but fall back to
    # streaming from .228's /opt/archipelago/web-ui/aiui (where the ISO build
    # deposited it). Without the fallback, secondaries lose AIUI whenever the
    # deploy runs from a machine that doesn't have an ../AIUI checkout.
    AIUI_DIST="$PROJECT_DIR/../AIUI/packages/app/dist"
    if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
        echo "   Deploying AIUI to .$SEC_LABEL (from local sibling dist)..."
        ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*"
        cd "$AIUI_DIST" && tar --no-xattrs -cf - . | ssh $SSH_OPTS "$SEC_TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
        cd "$PROJECT_DIR"
        ssh $SSH_OPTS "$SEC_TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui"
    elif ssh $SSH_OPTS archipelago@192.168.1.228 "[ -f /opt/archipelago/web-ui/aiui/index.html ]" 2>/dev/null; then
        echo "   Deploying AIUI to .$SEC_LABEL (streaming from .228)..."
        ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*"
        ssh $SSH_OPTS archipelago@192.168.1.228 "sudo tar --no-xattrs -cf - -C /opt/archipelago/web-ui/aiui ." \
          | ssh $SSH_OPTS "$SEC_TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
        ssh $SSH_OPTS "$SEC_TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui"
    else
        echo "   ⚠️  AIUI not available locally or on .228 — skipping AIUI deploy to .$SEC_LABEL"
    fi

    # Sync nginx config + snippets
    NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf"
    SNIPPETS_DIR="$PROJECT_DIR/image-recipe/configs/snippets"
    if [ -f "$NGINX_CFG" ]; then
        echo "   Syncing nginx config to .$SEC_LABEL..."
        scp $SSH_OPTS "$NGINX_CFG" "$SEC_TARGET:/tmp/nginx-archipelago.conf" 2>/dev/null || true
        ssh $SSH_OPTS "$SEC_TARGET" '
            sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
            sudo rm -f /etc/nginx/conf.d/external-app-proxies.conf
            sudo sed -i "s|proxy_pass http://127.0.0.1:3141/;|proxy_pass http://127.0.0.1:3142/;|g" /etc/nginx/sites-available/archipelago
            rm -f /tmp/nginx-archipelago.conf
        ' 2>/dev/null || true
    fi
    if [ -d "$SNIPPETS_DIR" ]; then
        echo "   Syncing nginx snippets to .$SEC_LABEL..."
        ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /etc/nginx/snippets" 2>/dev/null || true
        for f in "$SNIPPETS_DIR"/*.conf; do
            [ -f "$f" ] && scp $SSH_OPTS "$f" "$SEC_TARGET:/tmp/nginx-snippet-$(basename "$f")" 2>/dev/null || true
        done
        ssh $SSH_OPTS "$SEC_TARGET" '
            for f in /tmp/nginx-snippet-*.conf; do
              [ -f "$f" ] && sudo mv "$f" "/etc/nginx/snippets/$(basename "$f" | sed "s/^nginx-snippet-//")"
            done
        ' 2>/dev/null || true
    fi
    ssh $SSH_OPTS "$SEC_TARGET" 'sudo nginx -t 2>&1 && echo "   nginx config OK" || echo "   nginx config test failed"' 2>/dev/null || true

    # Sync systemd service file
    SERVICE_FILE="$PROJECT_DIR/image-recipe/configs/archipelago.service"
    if [ -f "$SERVICE_FILE" ]; then
        echo "   Syncing systemd service to .$SEC_LABEL..."
        scp $SSH_OPTS "$SERVICE_FILE" "$SEC_TARGET:/tmp/archipelago.service" 2>/dev/null || true
        ssh $SSH_OPTS "$SEC_TARGET" '
            if ! diff -q /tmp/archipelago.service /etc/systemd/system/archipelago.service >/dev/null 2>&1; then
              sudo cp /tmp/archipelago.service /etc/systemd/system/archipelago.service
              sudo systemctl daemon-reload
              echo "   Service file updated"
            else
              echo "   Service file unchanged"
            fi
            rm -f /tmp/archipelago.service
        ' 2>/dev/null || true
    fi
    if [ -n "${TELEMETRY_COLLECTOR_URL:-}" ]; then
        echo "   Syncing telemetry collector config to .$SEC_LABEL..."
        TMP_TELEMETRY_ENV="$(mktemp)"
        printf 'TELEMETRY_COLLECTOR_URL=%s\n' "$TELEMETRY_COLLECTOR_URL" > "$TMP_TELEMETRY_ENV"
        scp $SSH_OPTS "$TMP_TELEMETRY_ENV" "$SEC_TARGET:/tmp/telemetry.env" 2>/dev/null || true
        rm -f "$TMP_TELEMETRY_ENV"
        ssh $SSH_OPTS "$SEC_TARGET" '
            sudo mkdir -p /var/lib/archipelago
            sudo cp /tmp/telemetry.env /var/lib/archipelago/telemetry.env
            sudo chown archipelago:archipelago /var/lib/archipelago/telemetry.env
            sudo chmod 600 /var/lib/archipelago/telemetry.env
            rm -f /tmp/telemetry.env
        ' 2>/dev/null || true
    fi

    # Deploy udev rule for mesh radio
    UDEV_RULE="$PROJECT_DIR/image-recipe/configs/99-mesh-radio.rules"
    if [ -f "$UDEV_RULE" ]; then
        echo "   Syncing udev rule to .$SEC_LABEL..."
        scp $SSH_OPTS "$UDEV_RULE" "$SEC_TARGET:/tmp/99-mesh-radio.rules" 2>/dev/null || true
        ssh $SSH_OPTS "$SEC_TARGET" '
            if ! diff -q /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules >/dev/null 2>&1; then
              sudo cp /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules
              sudo udevadm control --reload-rules
              sudo udevadm trigger --subsystem-match=tty
              echo "   Mesh radio udev rule installed"
            else
              echo "   Mesh radio udev rule unchanged"
            fi
            rm -f /tmp/99-mesh-radio.rules
        ' 2>/dev/null || true
    fi

    # Dev mode + FileBrowser
    ssh $SSH_OPTS "$SEC_TARGET" '
        # Dev mode
        if ! grep -q "ARCHIPELAGO_DEV_MODE=true" /etc/systemd/system/archipelago.service.d/override.conf 2>/dev/null; then
            sudo mkdir -p /etc/systemd/system/archipelago.service.d
            printf "[Service]\nEnvironment=ARCHIPELAGO_DEV_MODE=true\n" | sudo tee /etc/systemd/system/archipelago.service.d/override.conf > /dev/null
            sudo systemctl daemon-reload
        fi
        # FileBrowser fix
        DOCKER=podman; command -v podman >/dev/null 2>&1 || DOCKER=docker
        FB=$($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -x filebrowser || true)
        if [ -n "$FB" ]; then
            RO=$($DOCKER inspect filebrowser 2>/dev/null | grep -oP "\"ReadonlyRootfs\":\s*\K\w+" || echo "false")
            if [ "$RO" = "true" ]; then
                $DOCKER stop filebrowser 2>/dev/null; $DOCKER rm filebrowser 2>/dev/null
                sudo mkdir -p /var/lib/archipelago/filebrowser
                $DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
                  --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
                  --security-opt no-new-privileges:true \
                  -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>/dev/null
            fi
        fi
    ' 2>/dev/null || true

    # Write deploy manifest
    local DEPLOY_TS
    DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    ssh $SSH_OPTS "$SEC_TARGET" "sudo tee /opt/archipelago/deploy-manifest.json > /dev/null" <<-MANIFEST_SEC_EOF
{
  "commit": "$DEPLOY_COMMIT_FULL",
  "commit_short": "$DEPLOY_COMMIT",
  "branch": "$DEPLOY_BRANCH",
  "dirty": $DEPLOY_DIRTY,
  "deployed_at": "$DEPLOY_TS",
  "deployed_from": "$(hostname)",
  "target": "$SEC_TARGET"
}
MANIFEST_SEC_EOF

    ssh $SSH_OPTS "$SEC_TARGET" "sudo systemctl start archipelago && sudo systemctl restart nginx"

    # Run container doctor
    echo "   Running container doctor on .$SEC_LABEL..."
    "$SCRIPT_DIR/container-doctor.sh" "$SEC_TARGET" 2>&1 | sed 's/^/   /' || true

    # Post-deploy health check
    echo "   Checking .$SEC_LABEL health..."
    local HEALTH="fail"
    for i in $(seq 1 12); do
        sleep 5
        HEALTH=$(curl -s --max-time 5 "http://$SEC_IP/health" 2>/dev/null || { echo "WARNING: Health check failed for $SEC_IP" >&2; echo ""; })
        if [ "$HEALTH" = "OK" ]; then
            echo "   ✅ $SEC_IP deployed (health OK after $((i * 5))s)"
            break
        fi
    done
    if [ "$HEALTH" != "OK" ]; then
        echo "   ⚠️  $SEC_IP deployed but health check failed after 60s"
    fi
}

# When --both: deploy to 228 first, then copy to 198 + 253
if [ "$BOTH" = true ]; then
    echo "Deploying to all LAN servers (228, then 198 + 253)..."
    # Release lock so the recursive --live call can acquire it
    rm -rf "$LOCK_DIR" 2>/dev/null; trap - EXIT
    "$0" --live
    echo ""

    # Fetch built binary from .228 (shared by all secondary nodes)
    if ! scp $SSH_OPTS "archipelago@192.168.1.228:$TARGET_DIR/core/target/release/archipelago" /tmp/archipelago-both 2>/dev/null; then
        echo "   ERROR: Failed to copy binary from .228 — is the build available?"
        exit 1
    fi

    # Deploy to each secondary node
    deploy_secondary "archipelago@192.168.1.198" "198"
    deploy_secondary "archipelago@192.168.1.253" "253"

    rm -f /tmp/archipelago-both
    exit 0
fi

# Sync code
section_start
progress "Syncing code"
rsync -avz --delete \
    -e "ssh $SSH_OPTS" \
    --exclude 'node_modules' \
    --exclude 'target' \
    --exclude 'dist' \
    --exclude '.git' \
    --exclude '.codex-target-*' \
    --exclude '.codex-tmp' \
    --exclude 'uploads' \
    --exclude 'image-recipe/build' \
    --exclude 'image-recipe/results' \
    --exclude 'image-recipe/_archived/build' \
    --exclude 'image-recipe/_archived/results' \
    "$PROJECT_DIR/" "$TARGET_HOST:$TARGET_DIR/"
section_end

if [ "$QUICK" = true ]; then
    echo ""
    echo "✅ Quick sync complete!"
    exit 0
fi

# Build on target
echo ""
progress "Building frontend"
section_start
ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/neode-ui && npm install --silent && npm run build" 2>&1 | sed 's/^/   /'
section_end

# Backend (if Rust is installed) — skip with --frontend-only
if [ "$FRONTEND_ONLY" = true ]; then
    echo "   Skipping backend build (--frontend-only)"
elif ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env 2>/dev/null && command -v cargo" >/dev/null 2>&1; then
    progress "Building backend (Rust release)"
    section_start
    ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env && cd $TARGET_DIR/core && cargo build --release 2>&1" | sed 's/^/   /'
    section_end
else
    echo "   ⚠️  Rust not installed on target, skipping backend build"
fi

if [ "$LIVE" = true ]; then

    # Create rollback backup before deploying
    progress "Creating rollback backup"
    ssh $SSH_OPTS "$TARGET_HOST" '
        sudo mkdir -p /opt/archipelago/rollback
        [ -f /usr/local/bin/archipelago ] && sudo cp /usr/local/bin/archipelago /opt/archipelago/rollback/archipelago.bak 2>/dev/null || true
        [ -d /opt/archipelago/web-ui ] && sudo tar cf /opt/archipelago/rollback/web-ui.tar -C /opt/archipelago/web-ui . 2>/dev/null || true
    ' 2>/dev/null || true

    # Deploy backend (check if binary exists) — skip with --frontend-only
    if [ "$FRONTEND_ONLY" = true ]; then
        echo "   Skipping backend deploy (--frontend-only)"
    elif ssh $SSH_OPTS "$TARGET_HOST" "[ -f $TARGET_DIR/core/target/release/archipelago ]" 2>/dev/null; then
        progress "Deploying backend binary"
        ssh $SSH_OPTS "$TARGET_HOST" 'sudo systemctl stop archipelago --no-block 2>/dev/null; sleep 2; sudo kill -9 $(pgrep -x archipelago) 2>/dev/null; sleep 1; true'
        if [ "$RESET_MESH" = true ]; then
            echo "   Wiping mesh cache (peers/messages/sessions) per --reset-mesh"
            ssh $SSH_OPTS "$TARGET_HOST" 'sudo rm -f /var/lib/archipelago/messages.json /var/lib/archipelago/sessions.json /var/lib/archipelago/mesh-outbox.json 2>/dev/null; true'
        fi
        ssh $SSH_OPTS "$TARGET_HOST" "sudo cp $TARGET_DIR/core/target/release/archipelago /usr/local/bin/"
    fi

    # Deploy frontend (preserve aiui/ and claude-login.html — they are NOT part of the neode-ui build)
    progress "Deploying frontend"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name 'aiui' ! -name 'claude-login.html' -exec rm -rf {} +"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo cp -rf $TARGET_DIR/web/dist/neode-ui/* /opt/archipelago/web-ui/"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui"

    # Build and deploy AIUI (non-fatal — never delete existing AIUI on failure)
    progress "Building & deploying AIUI"
    AIUI_DIR="$PROJECT_DIR/../AIUI"
    AIUI_DIST="$AIUI_DIR/packages/app/dist"
    # Auto-build AIUI if dist is missing or older than source
    if [ -d "$AIUI_DIR/packages/app/src" ] && ( [ ! -f "$AIUI_DIST/index.html" ] || [ "$(find "$AIUI_DIR/packages/app/src" -newer "$AIUI_DIST/index.html" -print -quit 2>/dev/null)" != "" ] ); then
        echo "$(timestamp)    Building AIUI (source newer than dist or dist missing)..."
        (cd "$AIUI_DIR" && VITE_BASE_PATH=/aiui/ pnpm build 2>&1 | tail -5) || echo "$(timestamp)    ⚠️  AIUI build failed"
    fi
    # Fallback: if the AIUI sibling checkout is missing, use the pre-built
    # dist shipped in this repo at demo/aiui/. That path is what we ship in
    # the release tarball too, so local-and-fleet-update stay consistent.
    if [ ! -f "$AIUI_DIST/index.html" ] && [ -f "$PROJECT_DIR/demo/aiui/index.html" ]; then
        echo "$(timestamp)    AIUI sibling dist missing — using demo/aiui/ from repo"
        AIUI_DIST="$PROJECT_DIR/demo/aiui"
    fi
    if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
        echo "$(timestamp)    Deploying AIUI..."
        if true; then
            echo "$(timestamp)    Deploying AIUI..."
            ssh $SSH_OPTS "$TARGET_HOST" "sudo mkdir -p /opt/archipelago/web-ui/aiui"
            ssh $SSH_OPTS "$TARGET_HOST" "sudo rm -rf /opt/archipelago/web-ui/aiui/*"
            cd "$AIUI_DIST" && tar --no-xattrs -cf - . | ssh $SSH_OPTS "$TARGET_HOST" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
            cd "$PROJECT_DIR"
            ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui && sudo chmod 755 /opt/archipelago/web-ui/aiui && sudo find /opt/archipelago/web-ui/aiui -type d -exec chmod 755 {} \;"
        else
            echo "$(timestamp)    ⚠️  AIUI build failed — keeping existing AIUI on server"
        fi
    else
        echo "$(timestamp)    ⚠️  AIUI not found at $AIUI_DIR, skipping"
    fi
    
    # Sync nginx config from image-recipe (single source of truth)
    progress "Syncing nginx configuration"
    NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf"
    SNIPPETS_DIR="$PROJECT_DIR/image-recipe/configs/snippets"
    if [ -f "$NGINX_CFG" ]; then
      scp $SSH_OPTS "$NGINX_CFG" "$TARGET_HOST:/tmp/nginx-archipelago.conf" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" '
        sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
        # Make sites-enabled a symlink to sites-available so future
        # config updates actually take effect. Older deploys left
        # sites-enabled as a regular file that fell out of sync.
        if [ -f /etc/nginx/sites-enabled/archipelago ] && [ ! -L /etc/nginx/sites-enabled/archipelago ]; then
            sudo rm -f /etc/nginx/sites-enabled/archipelago
            sudo ln -s /etc/nginx/sites-available/archipelago /etc/nginx/sites-enabled/archipelago
        elif [ ! -e /etc/nginx/sites-enabled/archipelago ]; then
            sudo ln -s /etc/nginx/sites-available/archipelago /etc/nginx/sites-enabled/archipelago
        fi
        rm -f /tmp/nginx-archipelago.conf
      ' 2>/dev/null || true
    fi

    # Sync nginx snippet files (HTTPS app proxies, PWA headers — included by main config)
    if [ -d "$SNIPPETS_DIR" ]; then
      ssh $SSH_OPTS "$TARGET_HOST" "sudo mkdir -p /etc/nginx/snippets" 2>/dev/null || true
      for f in "$SNIPPETS_DIR"/*.conf; do
        [ -f "$f" ] && scp $SSH_OPTS "$f" "$TARGET_HOST:/tmp/nginx-snippet-$(basename "$f")" 2>/dev/null || true
      done
      ssh $SSH_OPTS "$TARGET_HOST" '
        for f in /tmp/nginx-snippet-*.conf; do
          [ -f "$f" ] && sudo mv "$f" "/etc/nginx/snippets/$(basename "$f" | sed "s/^nginx-snippet-//")"
        done
      ' 2>/dev/null || true
    fi

    # Remove old port-based external app proxies config
    ssh $SSH_OPTS "$TARGET_HOST" 'sudo rm -f /etc/nginx/conf.d/external-app-proxies.conf' 2>/dev/null || true

    # Fix nginx Claude API proxy port (template uses 3141, proxy runs on 3142)
    ssh $SSH_OPTS "$TARGET_HOST" 'sudo sed -i "s|proxy_pass http://127.0.0.1:3141/;|proxy_pass http://127.0.0.1:3142/;|g" /etc/nginx/sites-available/archipelago' 2>/dev/null || true

    # Validate nginx config after all changes
    ssh $SSH_OPTS "$TARGET_HOST" 'sudo nginx -t 2>&1 && echo "   nginx config OK" || echo "   ⚠️  nginx config test failed"' 2>/dev/null || true

    # Sync systemd service file (single source of truth: image-recipe/configs/)
    progress "Syncing systemd service"
    SERVICE_FILE="$PROJECT_DIR/image-recipe/configs/archipelago.service"
    if [ -f "$SERVICE_FILE" ]; then
      scp $SSH_OPTS "$SERVICE_FILE" "$TARGET_HOST:/tmp/archipelago.service" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" '
        if ! diff -q /tmp/archipelago.service /etc/systemd/system/archipelago.service >/dev/null 2>&1; then
          sudo cp /tmp/archipelago.service /etc/systemd/system/archipelago.service
          sudo systemctl daemon-reload
          echo "   Service file updated"
        else
          echo "   Service file unchanged"
        fi
        rm -f /tmp/archipelago.service
      ' 2>/dev/null || true
    fi

    # Sync kiosk display helpers and units for HDMI/TV nodes. Existing nodes may
    # not have a git checkout, so live deploy must carry these outside OTA too.
    KIOSK_LAUNCHER="$PROJECT_DIR/image-recipe/configs/archipelago-kiosk-launcher.sh"
    if [ -f "$KIOSK_LAUNCHER" ]; then
      scp $SSH_OPTS "$KIOSK_LAUNCHER" "$TARGET_HOST:/tmp/archipelago-kiosk-launcher" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" '
        sudo install -m 755 /tmp/archipelago-kiosk-launcher /usr/local/bin/archipelago-kiosk-launcher
        rm -f /tmp/archipelago-kiosk-launcher
        echo "   Kiosk launcher updated"
      ' 2>/dev/null || true
    fi
    for unit in archipelago-kiosk.service archipelago-kiosk-watchdog.service; do
      KIOSK_UNIT="$PROJECT_DIR/image-recipe/configs/$unit"
      [ -f "$KIOSK_UNIT" ] || continue
      scp $SSH_OPTS "$KIOSK_UNIT" "$TARGET_HOST:/tmp/$unit" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" "
        if ! diff -q '/tmp/$unit' '/etc/systemd/system/$unit' >/dev/null 2>&1; then
          sudo install -m 644 '/tmp/$unit' '/etc/systemd/system/$unit'
          sudo systemctl daemon-reload
          echo '   $unit updated'
        else
          echo '   $unit unchanged'
        fi
        rm -f '/tmp/$unit'
      " 2>/dev/null || true
    done
    if [ -n "${TELEMETRY_COLLECTOR_URL:-}" ]; then
      progress "Syncing telemetry collector config"
      TMP_TELEMETRY_ENV="$(mktemp)"
      printf 'TELEMETRY_COLLECTOR_URL=%s\n' "$TELEMETRY_COLLECTOR_URL" > "$TMP_TELEMETRY_ENV"
      scp $SSH_OPTS "$TMP_TELEMETRY_ENV" "$TARGET_HOST:/tmp/telemetry.env" 2>/dev/null || true
      rm -f "$TMP_TELEMETRY_ENV"
      ssh $SSH_OPTS "$TARGET_HOST" '
        sudo mkdir -p /var/lib/archipelago
        sudo cp /tmp/telemetry.env /var/lib/archipelago/telemetry.env
        sudo chown archipelago:archipelago /var/lib/archipelago/telemetry.env
        sudo chmod 600 /var/lib/archipelago/telemetry.env
        rm -f /tmp/telemetry.env
      ' 2>/dev/null || true
    fi

    # Deploy udev rule for mesh radio stable naming (/dev/mesh-radio)
    UDEV_RULE="$PROJECT_DIR/image-recipe/configs/99-mesh-radio.rules"
    if [ -f "$UDEV_RULE" ]; then
      scp $SSH_OPTS "$UDEV_RULE" "$TARGET_HOST:/tmp/99-mesh-radio.rules" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" '
        if ! diff -q /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules >/dev/null 2>&1; then
          sudo cp /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --subsystem-match=tty
          echo "   Mesh radio udev rule installed"
        else
          echo "   Mesh radio udev rule unchanged"
        fi
        rm -f /tmp/99-mesh-radio.rules
      ' 2>/dev/null || true
    fi

    # Deploy Claude API proxy (auto-install if missing)
    progress "Setting up Claude API proxy"
    ssh $SSH_OPTS "$TARGET_HOST" '
        echo "   Updating Claude API proxy on port 3142..."
        # Check for API key in existing service or setup-aiui-server.sh
        EXISTING_KEY=$(grep -oP "ANTHROPIC_API_KEY=\K.*" /etc/systemd/system/claude-api-proxy.service 2>/dev/null || true)
        if [ -z "$EXISTING_KEY" ]; then
          echo "   ⚠️  No ANTHROPIC_API_KEY found — run setup-aiui-server.sh first to configure"
        else
          # Proxy script
          sudo tee /opt/archipelago/claude-api-proxy.py > /dev/null << '\''PYEOF'\''
#!/usr/bin/env python3
import http.server, json, ssl, sys, os, urllib.request, urllib.error
API_KEY = os.environ.get("ANTHROPIC_API_KEY", "")
PORT = 3142
class Handler(http.server.BaseHTTPRequestHandler):
    def do_POST(self):
        if self.path == "/health":
            self.send_response(200); self.send_header("Content-Type","application/json"); self.end_headers()
            self.wfile.write(b"{\"status\":\"ok\"}"); return
        cl = int(self.headers.get("Content-Length", 0))
        body = self.rfile.read(cl)
        try: data = json.loads(body)
        except: data = {}
        if "max_tokens" not in data: data["max_tokens"] = 8096
        for f in ["webSearch","web_search"]: data.pop(f, None)
        # Normalize model IDs — map short/dotted names to full API model IDs
        MODEL_MAP = {
            "claude-haiku-4.5": "claude-haiku-4-5-20251001",
            "claude-haiku-4-5": "claude-haiku-4-5-20251001",
            "claude-sonnet-4": "claude-sonnet-4-20250514",
            "claude-sonnet-4.5": "claude-sonnet-4-5-20250514",
            "claude-sonnet-4-5": "claude-sonnet-4-5-20250514",
            "claude-opus-4": "claude-opus-4-20250514",
        }
        m = data.get("model", "")
        if m in MODEL_MAP: data["model"] = MODEL_MAP[m]
        body = json.dumps(data).encode()
        if not API_KEY:
            err = json.dumps({"type":"error","error":{"type":"auth_error","message":"AIUI not configured. Set your Anthropic API key in Settings > AIUI to enable AI chat."}}).encode()
            self.send_response(401); self.send_header("Content-Type","application/json"); self.send_header("Content-Length",str(len(err))); self.end_headers(); self.wfile.write(err); return
        headers = {"Content-Type":"application/json","x-api-key":API_KEY,"anthropic-version":"2023-06-01","anthropic-dangerous-direct-browser-access":"true"}
        for h in ["anthropic-version","anthropic-beta"]:
            if self.headers.get(h): headers[h] = self.headers[h]
        req = urllib.request.Request("https://api.anthropic.com"+self.path, data=body, headers=headers, method="POST")
        try:
            ctx = ssl.create_default_context()
            resp = urllib.request.urlopen(req, context=ctx, timeout=300)
            self.send_response(resp.status)
            is_stream = "text/event-stream" in (resp.headers.get("Content-Type","") or "")
            for k,v in resp.headers.items():
                if k.lower() not in ("transfer-encoding","connection"): self.send_header(k,v)
            if is_stream: self.send_header("Transfer-Encoding","chunked")
            self.end_headers()
            if is_stream:
                while True:
                    chunk = resp.read(4096)
                    if not chunk: break
                    self.wfile.write(b"%x\r\n" % len(chunk)); self.wfile.write(chunk); self.wfile.write(b"\r\n"); self.wfile.flush()
                self.wfile.write(b"0\r\n\r\n"); self.wfile.flush()
            else: self.wfile.write(resp.read())
        except urllib.error.HTTPError as e:
            self.send_response(e.code); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(e.read())
        except Exception as e:
            self.send_response(502); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(json.dumps({"error":str(e)}).encode())
    def do_GET(self):
        if self.path == "/health":
            self.send_response(200); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(b"{\"status\":\"ok\"}")
        else: self.send_response(404); self.end_headers()
    def log_message(self, fmt, *args): pass
if not API_KEY: print("WARNING: ANTHROPIC_API_KEY not set — AIUI will return setup instructions")
server = http.server.HTTPServer(("127.0.0.1", PORT), Handler)
print(f"Claude API proxy on port {PORT}")
server.serve_forever()
PYEOF
          sudo systemctl daemon-reload
          sudo systemctl enable claude-api-proxy
          sudo systemctl restart claude-api-proxy
          sleep 1
          echo "   Claude API proxy: $(systemctl is-active claude-api-proxy)"
        fi
    ' 2>/dev/null || true

    # Dev mode for Tailscale HTTP access (cookies need Secure flag disabled over plain HTTP)
    progress "Configuring dev mode"
    ssh $SSH_OPTS "$TARGET_HOST" '
      if [ -f /etc/systemd/system/archipelago.service.d/override.conf ] && grep -q "ARCHIPELAGO_DEV_MODE=true" /etc/systemd/system/archipelago.service.d/override.conf 2>/dev/null; then
        echo "   Dev mode already enabled"
      else
        echo "   Enabling dev mode (for Tailscale HTTP cookie support)..."
        sudo mkdir -p /etc/systemd/system/archipelago.service.d
        printf "[Service]\nEnvironment=ARCHIPELAGO_DEV_MODE=true\n" | sudo tee /etc/systemd/system/archipelago.service.d/override.conf > /dev/null
        sudo systemctl daemon-reload
        echo "   Dev mode enabled"
      fi
    ' 2>/dev/null || true

    # Create data directories for DWN, content sharing, federation, identities
    progress "Creating data directories"
    ssh $SSH_OPTS "$TARGET_HOST" '
      # Rootless podman: allow binding to ports >= 80 (default is 1024)
      if ! grep -q "unprivileged_port_start=80" /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null; then
        echo "net.ipv4.ip_unprivileged_port_start=80" | sudo tee /etc/sysctl.d/99-rootless-podman.conf > /dev/null
        sudo sysctl -p /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null
        echo "   Rootless port binding enabled (>=80)"
      fi
      # Rootless podman: enable lingering for container persistence
      if [ "$(loginctl show-user archipelago 2>/dev/null | grep Linger)" != "Linger=yes" ]; then
        sudo loginctl enable-linger archipelago
        echo "   Linger enabled for archipelago user"
      fi
      # Rootless podman: enable podman socket
      systemctl --user enable podman.socket 2>/dev/null || true
      systemctl --user start podman.socket 2>/dev/null || true

      sudo mkdir -p /var/lib/archipelago/dwn/messages
      sudo mkdir -p /var/lib/archipelago/dwn/protocols
      sudo mkdir -p /var/lib/archipelago/content/files
      sudo mkdir -p /var/lib/archipelago/federation
      sudo mkdir -p /var/lib/archipelago/identity
      sudo mkdir -p /var/lib/archipelago/identities
      sudo mkdir -p /var/lib/archipelago/tor-config
      sudo chown -R archipelago:archipelago /var/lib/archipelago/dwn /var/lib/archipelago/content /var/lib/archipelago/federation /var/lib/archipelago/identity /var/lib/archipelago/identities /var/lib/archipelago/tor-config 2>/dev/null || true
      # Fix secrets directory ownership (must be readable by archipelago user, not root)
      sudo chown -R archipelago:archipelago /var/lib/archipelago/secrets 2>/dev/null || true
      sudo chmod 700 /var/lib/archipelago/secrets 2>/dev/null || true
      # Fix any root-owned files in data dir - dead mans switch, sessions, server-name
      sudo find /var/lib/archipelago -maxdepth 1 -name "*.json" -user root -exec chown archipelago:archipelago {} \; 2>/dev/null || true
      sudo chown archipelago:archipelago /var/lib/archipelago/server-name 2>/dev/null || true
      echo "   Data directories OK"

      # Rootless podman UID mapping: fix data dir ownership so container processes
      # can write. Rootless podman maps container UIDs via subuid (container UID 0 →
      # host UID 1000, container UID N → host UID 100000+N).
      echo "   Fixing rootless podman UID mapping..."
      # Containers running as root (UID 0 inside → host UID 100000 via subuid)
      for dir in lnd electrumx btcpay nbxplorer immich jellyfin vaultwarden \
                 home-assistant fedimint fedimint-gateway photoprism ollama filebrowser; do
        [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
      done
      # Bitcoin Knots: container UID 101 → host UID 100101
      [ -d /var/lib/archipelago/bitcoin ] && sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
      # Postgres containers: container UID 70 → host UID 100070
      for dir in postgres-btcpay immich-db; do
        [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null
      done
      # MariaDB: container UID 999 → host UID 100999
      [ -d /var/lib/archipelago/mempool ] && sudo chown -R 100999:100999 /var/lib/archipelago/mempool 2>/dev/null
      # Grafana: container UID 472 → host UID 100472
      [ -d /var/lib/archipelago/grafana ] && sudo chown -R 100472:100472 /var/lib/archipelago/grafana 2>/dev/null
      echo "   UID mapping done"
    ' 2>/dev/null || true

    # Deploy nostr-provider.js for NIP-07 iframe signing (window.nostr support)
    progress "Deploying nostr-provider.js"
    scp $SSH_OPTS "$PROJECT_DIR/neode-ui/public/nostr-provider.js" "$TARGET_HOST:/tmp/nostr-provider.js" 2>/dev/null && \
    ssh $SSH_OPTS "$TARGET_HOST" 'sudo cp /tmp/nostr-provider.js /opt/archipelago/web-ui/nostr-provider.js && echo "   nostr-provider.js deployed"' 2>/dev/null || echo "   (nostr-provider.js not found, skipping)"

    # Deploy tor-helper: script + systemd path unit for privileged Tor management
    progress "Deploying tor-helper"
    scp $SSH_OPTS \
      "$PROJECT_DIR/scripts/tor-helper.sh" \
      "$PROJECT_DIR/image-recipe/configs/archipelago-tor-helper.path" \
      "$PROJECT_DIR/image-recipe/configs/archipelago-tor-helper.service" \
      "$TARGET_HOST:/tmp/" 2>/dev/null && \
    ssh $SSH_OPTS "$TARGET_HOST" '
      sudo mkdir -p /opt/archipelago/scripts
      sudo cp /tmp/tor-helper.sh /opt/archipelago/scripts/tor-helper.sh
      sudo chmod 755 /opt/archipelago/scripts/tor-helper.sh
      sudo chown root:root /opt/archipelago/scripts/tor-helper.sh
      sudo cp /tmp/archipelago-tor-helper.path /etc/systemd/system/
      sudo cp /tmp/archipelago-tor-helper.service /etc/systemd/system/
      sudo systemctl daemon-reload
      sudo systemctl enable archipelago-tor-helper.path
      sudo systemctl start archipelago-tor-helper.path
      echo "   tor-helper deployed with systemd path unit"
    ' 2>/dev/null || echo "   (tor-helper deploy skipped)"

    # Sync nginx config (second pass — includes HTTPS snippets)
    scp $SSH_OPTS "$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf" "$TARGET_HOST:/tmp/nginx-archipelago.conf" 2>/dev/null && \
    ssh $SSH_OPTS "$TARGET_HOST" '
      sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
      # Also sync HTTPS snippets if they exist
      sudo mkdir -p /etc/nginx/snippets
      echo "   Nginx config synced"
    ' 2>/dev/null || echo "   (nginx config sync skipped)"
    # Sync HTTPS app proxies snippet if it exists
    if [ -f "$PROJECT_DIR/image-recipe/configs/snippets/archipelago-https-app-proxies.conf" ]; then
      scp $SSH_OPTS "$PROJECT_DIR/image-recipe/configs/snippets/archipelago-https-app-proxies.conf" "$TARGET_HOST:/tmp/https-app-proxies.conf" 2>/dev/null && \
      ssh $SSH_OPTS "$TARGET_HOST" 'sudo cp /tmp/https-app-proxies.conf /etc/nginx/snippets/archipelago-https-app-proxies.conf' 2>/dev/null || true
    fi

    # Fix FileBrowser — recreate if read-only root, create if missing
    progress "Checking FileBrowser"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      FB_EXISTS=$($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -x filebrowser || true)
      if [ -n "$FB_EXISTS" ]; then
        RO=$($DOCKER inspect filebrowser 2>/dev/null | grep -oP "\"ReadonlyRootfs\":\s*\K\w+" || echo "false")
        if [ "$RO" = "true" ]; then
          echo "   FileBrowser has read-only root — recreating..."
          $DOCKER stop filebrowser 2>/dev/null
          $DOCKER rm filebrowser 2>/dev/null
          sudo mkdir -p /var/lib/archipelago/filebrowser
          $DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
            --security-opt no-new-privileges:true \
            -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>&1 | tail -1
          echo "   FileBrowser recreated"
        else
          echo "   FileBrowser OK"
        fi
      else
        echo "   Creating FileBrowser..."
        sudo mkdir -p /var/lib/archipelago/filebrowser
        $DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
          --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
          --security-opt no-new-privileges:true \
          -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>&1 | tail -1
        echo "   FileBrowser created"
      fi
    ' 2>/dev/null || true

    # Write deploy manifest — stamps the server with exactly what was deployed
    progress "Writing deploy manifest"
    DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    ssh $SSH_OPTS "$TARGET_HOST" "sudo tee /opt/archipelago/deploy-manifest.json > /dev/null" << MANIFEST_EOF
{
  "commit": "$DEPLOY_COMMIT_FULL",
  "commit_short": "$DEPLOY_COMMIT",
  "branch": "$DEPLOY_BRANCH",
  "dirty": $DEPLOY_DIRTY,
  "deployed_at": "$DEPLOY_TS",
  "deployed_from": "$(hostname)",
  "target": "$TARGET_HOST"
}
MANIFEST_EOF

    # Write build-info.txt — this is what the UI sidebar reads for the
    # displayed version (overrides the binary's CARGO_PKG_VERSION). Keeping
    # it synced with Cargo.toml on every deploy prevents the 1.3.x drift
    # we saw on .198/.253 where stale build-info survived across upgrades.
    DEPLOY_PKG_VERSION=$(grep '^version' "$PROJECT_DIR/core/archipelago/Cargo.toml" | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
    ssh $SSH_OPTS "$TARGET_HOST" "sudo tee /opt/archipelago/build-info.txt > /dev/null" << BUILDINFO_EOF
version=$DEPLOY_PKG_VERSION
build=$DEPLOY_TS
commit=$DEPLOY_COMMIT
date=$DEPLOY_TS
type=deployed
BUILDINFO_EOF

    # Ensure NTP and swap are configured (prevents OOM kills and clock drift)
    progress "Ensuring NTP + swap"
    ssh $SSH_OPTS "$TARGET_HOST" '
        # NTP via chrony
        if ! dpkg -l chrony >/dev/null 2>&1; then
            sudo rm -f /usr/sbin/policy-rc.d
            sudo apt-get update -qq && sudo apt-get install -y chrony 2>/dev/null
        fi
        sudo systemctl enable chrony 2>/dev/null
        sudo systemctl start chrony 2>/dev/null
        sudo timedatectl set-ntp true 2>/dev/null
        # Swap
        if [ ! -f /swapfile ]; then
            TOTAL_KB=$(grep MemTotal /proc/meminfo | awk "{print \$2}")
            SZ=$((TOTAL_KB / 1024 / 1024))
            [ "$SZ" -gt 8 ] && SZ=8
            [ "$SZ" -lt 2 ] && SZ=2
            sudo fallocate -l ${SZ}G /swapfile && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile
            grep -q "/swapfile" /etc/fstab || echo "/swapfile none swap sw 0 0" | sudo tee -a /etc/fstab
            echo "   Created ${SZ}G swap"
        fi
        sudo swapon /swapfile 2>/dev/null || true
    ' 2>&1 | tail -5 | sed 's/^/   /' || true

    # Ensure backend binds to localhost only (security: no direct LAN access to port 5678)
    progress "Securing backend bind address"
    ssh $SSH_OPTS "$TARGET_HOST" '
      if grep -q "ARCHIPELAGO_BIND=0.0.0.0" /etc/systemd/system/archipelago.service 2>/dev/null; then
        sudo sed -i "s/ARCHIPELAGO_BIND=0.0.0.0:5678/ARCHIPELAGO_BIND=127.0.0.1:5678/" /etc/systemd/system/archipelago.service
        sudo systemctl daemon-reload
        echo "   Fixed: backend now binds to 127.0.0.1 only"
      fi
    ' 2>/dev/null || true

    # Restart services
    progress "Restarting services"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo systemctl start archipelago && sudo systemctl restart nginx"

    # Set up HTTPS for PWA installability (browsers require secure context)
    progress "Setting up HTTPS"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo bash $TARGET_DIR/scripts/setup-https-dev.sh" 2>&1 | sed 's/^/   /' || true
    
    if [ "$FRONTEND_ONLY" = true ]; then
        echo "   Skipping container rebuilds (--frontend-only)"
    fi

    # App containers are now installed exclusively via the Marketplace UI.
    # The deploy script only handles code sync, backend build, and frontend build.
    if false; then  # Legacy app installation removed — kept for reference in git history
    progress "Rebuilding LND UI"
    if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/lnd-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t lnd-ui:local . || docker build --no-cache -t lnd-ui:local .)" 2>&1 | tail -12 | sed 's/^/   /'; then
        echo "   Recreating LND UI container (port 18083)..."
        ssh $SSH_OPTS "$TARGET_HOST" '
          DOCKER=podman
          command -v podman >/dev/null 2>&1 || DOCKER=docker
          for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i lnd-ui); do
            [ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
          done
          $DOCKER run -d --name archy-lnd-ui -p 18083:80 --memory=256m --restart unless-stopped lnd-ui:local
        ' 2>&1 | sed 's/^/   /' || true
    fi

    # Rebuild and recreate ElectrumX UI container (port 50002)
    progress "Rebuilding ElectrumX UI"
    if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/electrs-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t electrs-ui:local . || docker build --no-cache -t electrs-ui:local .)" 2>&1 | tail -12 | sed 's/^/   /'; then
        echo "   Recreating ElectrumX UI container (port 50002, host network)..."
        ssh $SSH_OPTS "$TARGET_HOST" '
          DOCKER=podman
          command -v podman >/dev/null 2>&1 || DOCKER=docker
          for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i electrs-ui); do
            [ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
          done
          $DOCKER run -d --name archy-electrs-ui --network host --memory=256m --restart unless-stopped electrs-ui:local
        ' 2>&1 | sed 's/^/   /' || true
    fi

    # Rebuild and recreate Bitcoin UI container (host network, port 8334 in nginx.conf)
    # Host network required: bitcoin-ui proxies Bitcoin RPC at 127.0.0.1:8332
    progress "Rebuilding Bitcoin UI"
    # Inject real RPC credentials into bitcoin-ui nginx config before building
    ssh $SSH_OPTS "$TARGET_HOST" '
      SECRETS_DIR="/var/lib/archipelago/secrets"
      RPC_PASS=$(sudo cat "$SECRETS_DIR/bitcoin-rpc-password" 2>/dev/null)
      if [ -n "$RPC_PASS" ]; then
        AUTH_B64=$(echo -n "archipelago:${RPC_PASS}" | base64)
        sed -i "s|__BITCOIN_RPC_AUTH__|${AUTH_B64}|g" '"$TARGET_DIR"'/docker/bitcoin-ui/nginx.conf
      fi
    ' 2>/dev/null || true
    if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/bitcoin-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t bitcoin-ui:local . || docker build --no-cache -t bitcoin-ui:local .)" 2>&1 | tail -12 | sed 's/^/   /'; then
        echo "   Recreating Bitcoin UI container (port 8334, host network)..."
        ssh $SSH_OPTS "$TARGET_HOST" '
          DOCKER=podman
          command -v podman >/dev/null 2>&1 || DOCKER=docker
          for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i bitcoin-ui); do
            [ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
          done
          $DOCKER run -d --name archy-bitcoin-ui --network host --memory=256m --restart unless-stopped bitcoin-ui:local
        ' 2>&1 | sed 's/^/   /' || true
    fi

    # Bitcoin Knots: required for Mempool, ElectrumX, BTCPay, Fedimint
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"

    # Read Bitcoin RPC credentials from secrets file (rpcauth — stable across restarts)
    progress "Reading Bitcoin RPC credentials"
    BITCOIN_RPC_PASS=$(ssh $SSH_OPTS "$TARGET_HOST" '
      SECRETS_DIR="/var/lib/archipelago/secrets"
      sudo mkdir -p "$SECRETS_DIR" && sudo chmod 700 "$SECRETS_DIR"
      if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
          openssl rand -hex 16 | sudo tee "$SECRETS_DIR/bitcoin-rpc-password" > /dev/null
          sudo chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
      fi
      sudo cat "$SECRETS_DIR/bitcoin-rpc-password"
    ' 2>/dev/null)
    BITCOIN_RPC_USER="archipelago"
    if [ -z "$BITCOIN_RPC_PASS" ]; then
        echo "   WARNING: Could not read Bitcoin RPC password from server"
        return 1
    fi

    # Read per-installation database passwords from server secrets
    DB_PASSWORDS=$(ssh $SSH_OPTS "$TARGET_HOST" '
      SECRETS_DIR="/var/lib/archipelago/secrets"
      for svc in mempool btcpay immich penpot mysql-root; do
          if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
              openssl rand -base64 24 | sudo tee "$SECRETS_DIR/${svc}-db-password" > /dev/null
              sudo chmod 600 "$SECRETS_DIR/${svc}-db-password"
          fi
      done
      echo "MEMPOOL_DB_PASS=$(sudo cat "$SECRETS_DIR/mempool-db-password")"
      echo "BTCPAY_DB_PASS=$(sudo cat "$SECRETS_DIR/btcpay-db-password")"
      echo "IMMICH_DB_PASS=$(sudo cat "$SECRETS_DIR/immich-db-password")"
      echo "PENPOT_DB_PASS=$(sudo cat "$SECRETS_DIR/penpot-db-password")"
      echo "MYSQL_ROOT_PASS=$(sudo cat "$SECRETS_DIR/mysql-root-db-password")"
      # Fedimint gateway password and hash
      if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
          FEDI_PASS=$(openssl rand -base64 16)
          echo "$FEDI_PASS" | sudo tee "$SECRETS_DIR/fedimint-gateway-password" > /dev/null
          sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-password"
          if command -v htpasswd >/dev/null 2>&1; then
              htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ":\n" | sudo tee "$SECRETS_DIR/fedimint-gateway-hash" > /dev/null
              sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
          fi
      fi
      if [ -f "$SECRETS_DIR/fedimint-gateway-hash" ]; then
          echo "FEDI_HASH=$(sudo cat "$SECRETS_DIR/fedimint-gateway-hash")"
      fi
    ' 2>/dev/null)
    # Safe variable parsing — never eval untrusted SSH output
    while IFS='=' read -r key value; do
        # Skip empty lines
        [ -z "$key" ] && continue
        # Only allow expected variable names
        case "$key" in
            MEMPOOL_DB_PASS)  MEMPOOL_DB_PASS="$value" ;;
            BTCPAY_DB_PASS)   BTCPAY_DB_PASS="$value" ;;
            IMMICH_DB_PASS)   IMMICH_DB_PASS="$value" ;;
            PENPOT_DB_PASS)   PENPOT_DB_PASS="$value" ;;
            MYSQL_ROOT_PASS)  MYSQL_ROOT_PASS="$value" ;;
            FEDI_HASH)        FEDI_HASH="$value" ;;
            *)  echo "   WARNING: Ignoring unexpected variable from server: $key" ;;
        esac
    done <<< "$DB_PASSWORDS"
    # Fallback if hash not available
    if [ -z "${FEDI_HASH:-}" ]; then
        FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
    fi

    progress "Ensuring Bitcoin Knots"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      \$DOCKER network create archy-net 2>/dev/null || true
      NET_OPT='--network archy-net'
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|archy-bitcoin-knots'; then
        echo '   Creating Bitcoin Knots (mainnet, archipelago RPC)...'
        sudo mkdir -p /var/lib/archipelago/bitcoin
        # Demo mode: prune=550 saves ~194GB disk, but disables txindex (incompatible with electrumx)
        if [ "$DEMO" = "true" ]; then
          BTC_EXTRA_ARGS="-prune=550"
          BTC_DBCACHE=512
        else
          BTC_EXTRA_ARGS="-txindex=1"
          BTC_DBCACHE=4096
        fi
        \$DOCKER run -d --name bitcoin-knots --restart unless-stopped \$NET_OPT \
          --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -p 8332:8332 -p 8333:8333 \
          -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
          ${BITCOIN_KNOTS_IMAGE} \
          -server=1 \$BTC_EXTRA_ARGS \
          -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
          -dbcache=\$BTC_DBCACHE
        echo '   Bitcoin Knots started (sync may take hours)'
      else
        \$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Fix Mempool: clean duplicates, ensure full stack - mysql, backend (8999), frontend (4080)
    progress "Fixing Mempool stack"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      NET_OPT='--network archy-net'
      # Clean any duplicate/old mempool containers (user may have two versions)
      # EXCLUDE electrumx/mempool-electrs - indexing takes days, do not recreate on every deploy
      for c in mempool mempool-api mempool-web archy-mempool-api archy-mempool-web; do
        \$DOCKER stop \$c 2>/dev/null
        \$DOCKER rm -f \$c 2>/dev/null
      done
      # Create mysql-mempool if missing
      if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'mysql-mempool|archy-mempool-db'; then
        echo '   Creating mysql-mempool...'
        sudo mkdir -p /var/lib/archipelago/mysql-mempool
        \$DOCKER run -d --name archy-mempool-db --restart unless-stopped \$NET_OPT \
          --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
          -e MYSQL_DATABASE=mempool \
          -e MYSQL_USER=mempool \
          -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
          -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
          "$MARIADB_IMAGE"
        sleep 3
      fi
      MYSQL_CNT=\$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
      MYSQL_CNT=\${MYSQL_CNT:-archy-mempool-db}
      # Ensure DB is on archy-net so mempool-api can resolve it
      \$DOCKER network connect archy-net \$MYSQL_CNT 2>/dev/null || true
      # Stop and remove old mempool-electrs if present (replaced by electrumx)
      if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q mempool-electrs; then
        echo '   Removing old mempool-electrs (replaced by ElectrumX)...'
        \$DOCKER stop mempool-electrs 2>/dev/null
        \$DOCKER rm -f mempool-electrs 2>/dev/null
      fi
      # Create electrumx ONLY if missing - do NOT recreate (indexing takes days)
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
        if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
          echo '   Starting existing electrumx (preserving index)...'
          \$DOCKER start electrumx 2>/dev/null || true
        else
          echo '   Creating electrumx (indexer - may take days to sync, do not recreate)...'
          sudo mkdir -p /var/lib/archipelago/electrumx
          \$DOCKER run -d --name electrumx --restart unless-stopped \$NET_OPT \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
            --security-opt no-new-privileges:true \
            -p 50001:50001 \
            -v /var/lib/archipelago/electrumx:/data \
            -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
            -e COIN=Bitcoin \
            -e DB_DIRECTORY=/data \
            -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
            "$ELECTRUMX_IMAGE"
        fi
      fi
      # Create/recreate mempool-api (backend on 8999) - required for mempool to work
      for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mempool-api|archy-mempool-api'); do
        echo '   Recreating mempool-api (backend)...'
        \$DOCKER stop \"\$c\" 2>/dev/null
        \$DOCKER rm -f \"\$c\" 2>/dev/null
      done
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
        echo '   Creating mempool-api (backend)...'
        sudo mkdir -p /var/lib/archipelago/mempool
        \$DOCKER run -d --name mempool-api --restart unless-stopped \$NET_OPT \
        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
        --security-opt no-new-privileges:true \
        -p 8999:8999 \
        -v /var/lib/archipelago/mempool:/data \
        -e MEMPOOL_BACKEND=electrum \
        -e ELECTRUM_HOST=electrumx \
        -e ELECTRUM_PORT=50001 \
        -e ELECTRUM_TLS_ENABLED=false \
        -e CORE_RPC_HOST=\$TARGET_IP \
        -e CORE_RPC_PORT=8332 \
        -e CORE_RPC_USERNAME=archipelago \
        -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
        -e DATABASE_ENABLED=true \
        -e DATABASE_HOST=\$MYSQL_CNT \
        -e DATABASE_DATABASE=mempool \
        -e DATABASE_USERNAME=mempool \
        -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
        "$MEMPOOL_BACKEND_IMAGE"
      fi
      # Recreate mempool frontend - handle both 'mempool' and 'mempool-web' (frontend was on wrong port 8999)
      for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^mempool\$|mempool-web|archy-mempool-web'); do
        echo '   Recreating mempool frontend on 4080...'
        \$DOCKER stop \"\$c\" 2>/dev/null
        \$DOCKER rm -f \"\$c\" 2>/dev/null
        break
      done
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-mempool-web; then
        echo '   Creating mempool frontend on 4080...'
        \$DOCKER run -d --name archy-mempool-web --restart unless-stopped \$NET_OPT \
          --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
          --security-opt no-new-privileges:true \
          -p 4080:8080 \
          -e FRONTEND_HTTP_PORT=8080 \
          -e BACKEND_MAINNET_HTTP_HOST=mempool-api \
          "$MEMPOOL_WEB_IMAGE"
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Fix BTCPay Server: requires PostgreSQL + NBXplorer (BTCPay needs NBXplorer for block indexing)
    progress "Fixing BTCPay stack"
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      \$DOCKER network create archy-net 2>/dev/null || true
      NET_OPT='--network archy-net'
      # Ensure bitcoin-knots is on archy-net for NBXplorer/BTCPay to reach it
      \$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
      # Create PostgreSQL for BTCPay if missing
      if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
        echo '   Creating archy-btcpay-db (PostgreSQL)...'
        sudo mkdir -p /var/lib/archipelago/postgres-btcpay
        \$DOCKER run -d --name archy-btcpay-db --restart unless-stopped \$NET_OPT \
          --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
          -e POSTGRES_DB=btcpay \
          -e POSTGRES_USER=btcpay \
          -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
          "$BTCPAY_POSTGRES_IMAGE"
        sleep 3
      fi
      # Create NBXplorer database in PostgreSQL (NBXplorer needs its own DB)
      \$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \
        \$DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true
      # Create NBXplorer (required by BTCPay - indexes blocks for payment tracking)
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
        if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
          \$DOCKER start archy-nbxplorer 2>/dev/null || true
        else
          echo '   Creating archy-nbxplorer...'
          sudo mkdir -p /var/lib/archipelago/nbxplorer
          \$DOCKER run -d --name archy-nbxplorer --restart unless-stopped \$NET_OPT \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
            --security-opt no-new-privileges:true \
            -p 32838:32838 \
            -v /var/lib/archipelago/nbxplorer:/data \
            -e NBXPLORER_DATADIR=/data \
            -e NBXPLORER_NETWORK=mainnet \
            -e NBXPLORER_CHAINS=btc \
            -e NBXPLORER_BIND=0.0.0.0:32838 \
            -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
            -e NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER \
            -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
            -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
            "$NBXPLORER_IMAGE"
          sleep 5
        fi
      fi
      # Recreate btcpay-server with PostgreSQL, NBXplorer URL, and Bitcoin RPC
      for c in btcpay-server archy-btcpay; do
        if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx \"\$c\"; then
          echo '   Recreating btcpay-server with NBXplorer...'
          \$DOCKER stop \"\$c\" 2>/dev/null
          \$DOCKER rm -f \"\$c\" 2>/dev/null
        fi
      done
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
        echo '   Creating btcpay-server on 23000...'
        sudo mkdir -p /var/lib/archipelago/btcpay
        \$DOCKER run -d --name btcpay-server --restart unless-stopped \$NET_OPT \
          --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -p 23000:49392 \
          -v /var/lib/archipelago/btcpay:/datadir \
          -e ASPNETCORE_URLS=http://0.0.0.0:49392 \
          -e BTCPAY_PROTOCOL=http \
          -e BTCPAY_HOST=\$TARGET_IP:23000 \
          -e BTCPAY_CHAINS=btc \
          -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
          -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
          -e BTCPAY_BTCRPCUSER=archipelago \
          -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
          -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
          "$BTCPAY_IMAGE"
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Ensure Immich stack (postgres + redis + server) - creates if missing
    progress "Ensuring Immich stack"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      # Remove old single-container 'immich' if present (wrong port mapping, conflicts with immich_server)
      if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
        echo '   Removing old immich container (use immich_server)...'
        \$DOCKER stop immich 2>/dev/null
        \$DOCKER rm -f immich 2>/dev/null
        \$DOCKER start immich_server 2>/dev/null || true
      fi
      if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
        echo '   Creating Immich stack...'
        sudo mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
        \$DOCKER network create immich-net 2>/dev/null || true
        if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
          \$DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
            -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
            -e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
            "$IMMICH_POSTGRES_IMAGE" 2>/dev/null || true
          sleep 5
        fi
        if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
          \$DOCKER run -d --name immich_redis --restart unless-stopped --network immich-net \
            "$VALKEY_IMAGE" 2>/dev/null || true
          sleep 2
        fi
        if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
          \$DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
            -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
            -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
            -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
            -e UPLOAD_LOCATION=/usr/src/app/upload \
            "$IMMICH_SERVER_IMAGE" 2>/dev/null || true
        fi
        echo '   Immich stack created (may take 1-2 min to become ready)'
      else
        echo '   Immich already running'
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Tor: global hidden services - each service gets its own .onion address
    progress "Setting up Tor"
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      sudo mkdir -p /var/lib/archipelago/tor

      # Ensure services.json exists with default services
      SERVICES_JSON=/var/lib/archipelago/tor/services.json
      if [ ! -f "\$SERVICES_JSON" ]; then
        sudo python3 -c '
import json
services = [
    {"name": "archipelago", "local_port": 80, "enabled": True},
    {"name": "bitcoin", "local_port": 8333, "enabled": True},
    {"name": "electrumx", "local_port": 50001, "enabled": True},
    {"name": "lnd", "local_port": 9735, "enabled": True},
    {"name": "btcpay", "local_port": 23000, "enabled": True},
    {"name": "mempool", "local_port": 4080, "enabled": True},
    {"name": "fedimint", "local_port": 8175, "enabled": True}
]
with open("/var/lib/archipelago/tor/services.json", "w") as f:
    json.dump({"services": services}, f, indent=2)
print("services.json created")
'
      fi

      # Generate torrc from services.json — use /var/lib/tor/ for hidden services
      sudo python3 -c '
import json, os

# Protocol services get direct port mapping; web apps map port 80 to their local port
PROTOCOL_SERVICES = {"bitcoin", "bitcoin-knots", "electrs", "electrumx", "lnd"}

lines = ["# Auto-generated by Archipelago deploy", "SocksPort 0.0.0.0:9050", "# ControlPort disabled", ""]

# Try reading services config (check both paths for compatibility)
cfg = None
for path in ["/var/lib/archipelago/tor-config/services.json", "/var/lib/archipelago/tor/services.json"]:
    try:
        with open(path) as f:
            cfg = json.load(f)
        break
    except Exception:
        pass

if cfg:
    for svc in cfg.get("services", []):
        if not svc.get("enabled", True):
            continue
        n = svc["name"]
        p = svc["local_port"]
        lines.append("HiddenServiceDir /var/lib/tor/hidden_service_%s" % n)
        if n in PROTOCOL_SERVICES:
            # Protocol: direct port mapping
            lines.append("HiddenServicePort %d 127.0.0.1:%d" % (p, p))
            if n == "lnd":
                lines.append("HiddenServicePort 9735 127.0.0.1:9735")
                lines.append("HiddenServicePort 10009 127.0.0.1:10009")
        else:
            # Web app: map port 80 on .onion to local app port (access via app.onion without port)
            lines.append("HiddenServicePort 80 127.0.0.1:%d" % p)
        lines.append("")
else:
    # Fallback: default services
    for n, mappings in [("archipelago",[(80,80)]),("bitcoin",[(8333,8333)]),("electrs",[(50001,50001)]),("lnd",[(8080,8080),(9735,9735),(10009,10009)]),("btcpay",[(80,23000)]),("mempool",[(80,4080)]),("fedimint",[(80,8175)])]:
        lines.append("HiddenServiceDir /var/lib/tor/hidden_service_%s" % n)
        for remote_p, local_p in mappings:
            lines.append("HiddenServicePort %d 127.0.0.1:%d" % (remote_p, local_p))
        lines.append("")

with open("/etc/tor/torrc", "w") as f:
    f.write("\n".join(lines) + "\n")
enabled = sum(1 for s in (cfg or {}).get("services", []) if s.get("enabled", True))
print("torrc generated with %d services" % (enabled or 7))
'

      # Remove any old Tor container (system Tor is preferred)
      for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'archy-tor|^tor\$'); do
        \$DOCKER stop \"\$c\" 2>/dev/null
        \$DOCKER rm -f \"\$c\" 2>/dev/null
      done

      # Use system Tor (preferred — no AppArmor issues with default paths)
      if command -v tor >/dev/null 2>&1; then
        sudo systemctl enable tor 2>/dev/null
        sudo systemctl enable tor@default 2>/dev/null
        sudo systemctl restart tor 2>/dev/null
        sudo systemctl restart tor@default 2>/dev/null
        echo '   Using system Tor daemon'
      else
        echo '   Installing system Tor...'
        sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null || true
        if command -v tor >/dev/null 2>&1; then
          sudo systemctl enable tor 2>/dev/null
          sudo systemctl enable tor@default 2>/dev/null
          sudo systemctl restart tor 2>/dev/null
          sudo systemctl restart tor@default 2>/dev/null
          echo '   System Tor installed and started'
        else
          echo '   WARNING: Could not install Tor'
        fi
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Tor diagnostic: check if hostname files exist (may take 30-60s after Tor starts)
    echo "   Checking Tor hostname files..."
    ssh $SSH_OPTS "$TARGET_HOST" "
      # Check all hidden_service_* dirs for hostname files (check both paths)
      for dir in /var/lib/tor/hidden_service_*/ /var/lib/archipelago/tor/hidden_service_*/; do
        [ -d \"\$dir\" ] || continue
        svc=\$(basename \"\$dir\" | sed 's/hidden_service_//')
        f=\"\${dir}hostname\"
        if [ -f \"\$f\" ]; then
          echo \"   ✓ \$svc: \$(cat \$f)\"
        else
          echo \"   ✗ \$svc: hostname not yet generated (Tor may need 30-60s)\"
        fi
      done
    " 2>&1 | sed 's/^/   /' || true

    # Recreate Fedimint with FM_API_URL for Guardian UI (fixes "Api URL must be configured")
    section_start
    progress "Fixing Fedimint"
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    TIMEOUT_CMD=""
    command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90"
    command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90"
    ($TIMEOUT_CMD ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^fedimint\$'); do
        echo '   Recreating fedimint with FM_API_URL...'
        \$DOCKER stop \"\$c\" 2>/dev/null
        \$DOCKER rm -f \"\$c\" 2>/dev/null
        \$DOCKER run -d --name fedimint --restart unless-stopped \
          --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -p 8173:8173 -p 8174:8174 -p 8175:8175 \
          -v /var/lib/archipelago/fedimint:/data \
          -e FM_DATA_DIR=/data \
          -e FM_BITCOIND_USERNAME=archipelago \
          -e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \
          -e FM_BITCOIN_NETWORK=bitcoin \
          -e FM_BIND_P2P=0.0.0.0:8173 \
          -e FM_BIND_API=0.0.0.0:8174 \
          -e FM_BIND_UI=0.0.0.0:8175 \
          -e FM_P2P_URL=fedimint://$TARGET_IP:8173 \
          -e FM_API_URL=ws://$TARGET_IP:8174 \
          -e FM_BITCOIND_URL=http://$TARGET_IP:8332 \
          "$FEDIMINT_IMAGE"
        break
      done

      # Ensure Fedimint Gateway companion container
      # Auto-detect LND: if running with credentials, use lnd mode; otherwise use ldk (built-in)
      \$DOCKER rm -f fedimint-gateway 2>/dev/null || true
      echo '   Creating fedimint-gateway...'
      sudo mkdir -p /var/lib/archipelago/fedimint-gateway
      LND_CERT=/var/lib/archipelago/lnd/tls.cert
      LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
      GW_COMMON=\"-p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data "$FEDIMINT_GATEWAY_IMAGE" gatewayd --data-dir /data --listen 0.0.0.0:8176 --bcrypt-password-hash '$FEDI_HASH' --network bitcoin --bitcoind-url http://$TARGET_IP:8332 --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS\"
      if \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then
        echo '   LND detected — using lnd mode'
        \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
          --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -p 8176:8176 \
          -v /var/lib/archipelago/fedimint-gateway:/data \
          -v /var/lib/archipelago/lnd/tls.cert:/lnd/tls.cert:ro \
          -v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \
          "$FEDIMINT_GATEWAY_IMAGE" \
          gatewayd --data-dir /data --listen 0.0.0.0:8176 \
          --bcrypt-password-hash '$FEDI_HASH' \
          --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
          --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
          lnd --lnd-rpc-host $TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon
      else
        echo '   No LND found — using ldk (built-in Lightning)'
        \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
          --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
          --security-opt no-new-privileges:true \
          -p 8176:8176 -p 9737:9737 \
          -v /var/lib/archipelago/fedimint-gateway:/data \
          "$FEDIMINT_GATEWAY_IMAGE" \
          gatewayd --data-dir /data --listen 0.0.0.0:8176 \
          --bcrypt-password-hash '$FEDI_HASH' \
          --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
          --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
          ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway
      fi
    " 2>&1 | sed 's/^/   /') || echo "   (Fedimint fix timed out or skipped - run manually if needed)"
    section_end

    # LND: Lightning Network Daemon (requires bitcoin-knots on archy-net)
    progress "Ensuring LND"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx lnd; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx lnd; then
          $DOCKER start lnd 2>/dev/null || true
          echo "   LND started (existing)"
        else
          echo "   Creating LND..."
          sudo mkdir -p /var/lib/archipelago/lnd
          if [ ! -f /var/lib/archipelago/lnd/lnd.conf ]; then
            cat > /tmp/lnd.conf <<LNDCONF
[Application Options]
listen=0.0.0.0:9735
rpclisten=0.0.0.0:10009
restlisten=0.0.0.0:8080
debuglevel=info
noseedbackup=true
tor.active=false

[Bitcoin]
bitcoin.mainnet=true
bitcoin.node=bitcoind

[Bitcoind]
bitcoind.rpchost=bitcoin-knots:8332
bitcoind.rpcuser=archipelago
bitcoind.rpcpass=$BITCOIN_RPC_PASS
bitcoind.rpcpolling=true
bitcoind.estimatemode=ECONOMICAL

[autopilot]
autopilot.active=false
LNDCONF
            sudo cp /tmp/lnd.conf /var/lib/archipelago/lnd/lnd.conf
          else
            # Always ensure LND config has correct RPC credentials from secrets
            LND_CONF=/var/lib/archipelago/lnd/lnd.conf
            CURRENT_PASS=$(sudo grep "bitcoind.rpcpass=" "$LND_CONF" 2>/dev/null | cut -d= -f2)
            NEEDS_FIX=0
            grep -q "rpccookie" "$LND_CONF" 2>/dev/null && NEEDS_FIX=1
            grep -q "rpchost=127.0.0.1" "$LND_CONF" 2>/dev/null && NEEDS_FIX=1
            RPC_PASS_EXPECTED=$(sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null)
            [ "$CURRENT_PASS" != "$RPC_PASS_EXPECTED" ] && NEEDS_FIX=1
            if [ "$NEEDS_FIX" = "1" ]; then
              echo "   Syncing LND config with current RPC credentials..."
              sudo sed -i "/bitcoind.rpccookie/d" "$LND_CONF"
              sudo sed -i "s|bitcoind.rpchost=127.0.0.1:8332|bitcoind.rpchost=bitcoin-knots:8332|" "$LND_CONF"
              sudo sed -i "s|bitcoind.rpcpass=.*|bitcoind.rpcpass=$RPC_PASS_EXPECTED|" "$LND_CONF"
              if ! sudo grep -q "bitcoind.rpcuser=" "$LND_CONF" 2>/dev/null; then
                sudo sed -i "/bitcoind.rpchost=/a bitcoind.rpcuser=archipelago" "$LND_CONF"
              fi
              if ! sudo grep -q "bitcoind.rpcpass=" "$LND_CONF" 2>/dev/null; then
                sudo sed -i "/bitcoind.rpcuser=/a bitcoind.rpcpass=$RPC_PASS_EXPECTED" "$LND_CONF"
              fi
              sudo chown 100000:100000 "$LND_CONF"
              RESTART_LND=1
              echo "   LND config updated"
            fi
          fi
          $DOCKER run -d --name lnd --restart unless-stopped --network archy-net \
            --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
            --security-opt no-new-privileges:true \
            -p 9735:9735 -p 10009:10009 -p 18080:8080 \
            -v /var/lib/archipelago/lnd:/root/.lnd \
            "$LND_IMAGE"
          echo "   LND created"
        fi
      else
        echo "   LND already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Home Assistant
    progress "Ensuring Home Assistant"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx homeassistant; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx homeassistant; then
          $DOCKER start homeassistant 2>/dev/null || true
        else
          echo "   Creating Home Assistant..."
          sudo mkdir -p /var/lib/archipelago/home-assistant
          $DOCKER run -d --name homeassistant --restart unless-stopped \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
            --security-opt no-new-privileges:true \
            -p 8123:8123 -v /var/lib/archipelago/home-assistant:/config \
            -e TZ=UTC \
            "$HOMEASSISTANT_IMAGE"
        fi
      else
        echo "   Home Assistant already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Grafana
    progress "Ensuring Grafana"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx grafana; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx grafana; then
          $DOCKER start grafana 2>/dev/null || true
        else
          echo "   Creating Grafana..."
          sudo mkdir -p /var/lib/archipelago/grafana
          sudo chown 472:472 /var/lib/archipelago/grafana 2>/dev/null || true
          $DOCKER run -d --name grafana --restart unless-stopped \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
            --security-opt no-new-privileges:true \
            -p 3000:3000 -v /var/lib/archipelago/grafana:/var/lib/grafana \
            -e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \
            "$GRAFANA_IMAGE"
        fi
      else
        echo "   Grafana already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Jellyfin
    progress "Ensuring Jellyfin"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx jellyfin; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx jellyfin; then
          $DOCKER start jellyfin 2>/dev/null || true
        else
          echo "   Creating Jellyfin..."
          sudo mkdir -p /var/lib/archipelago/jellyfin/config /var/lib/archipelago/jellyfin/cache
          $DOCKER run -d --name jellyfin --restart unless-stopped \
            --cap-drop ALL --security-opt no-new-privileges:true \
            -p 8096:8096 \
            -v /var/lib/archipelago/jellyfin/config:/config \
            -v /var/lib/archipelago/jellyfin/cache:/cache \
            "$JELLYFIN_IMAGE"
        fi
      else
        echo "   Jellyfin already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Vaultwarden
    progress "Ensuring Vaultwarden"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx vaultwarden; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx vaultwarden; then
          $DOCKER start vaultwarden 2>/dev/null || true
        else
          echo "   Creating Vaultwarden..."
          sudo mkdir -p /var/lib/archipelago/vaultwarden
          $DOCKER run -d --name vaultwarden --restart unless-stopped \
            --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
            --security-opt no-new-privileges:true \
            -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
            "$VAULTWARDEN_IMAGE"
        fi
      else
        echo "   Vaultwarden already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # SearXNG (privacy search engine — used by AIUI web search)
    progress "Ensuring SearXNG"
    ssh $SSH_OPTS "$TARGET_HOST" '
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx searxng; then
        if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx searxng; then
          $DOCKER start searxng 2>/dev/null || true
        else
          echo "   Creating SearXNG..."
          $DOCKER run -d --name searxng --restart unless-stopped \
            --cap-drop ALL --security-opt no-new-privileges:true \
            -p 8888:8080 \
            ${SEARXNG_IMAGE}
        fi
      else
        echo "   SearXNG already running"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Ollama — optional, install from marketplace if needed
    # (removed from auto-deploy: large image, not needed for core functionality)

    fi  # end legacy app installation (dead code, kept for git history)

    # Ensure UFW allows forwarded traffic (required for podman container port access from LAN)
    progress "Fixing UFW forward policy"
    ssh $SSH_OPTS "$TARGET_HOST" '
      if grep -q "DEFAULT_FORWARD_POLICY=\"DROP\"" /etc/default/ufw 2>/dev/null; then
        sudo sed -i "s/DEFAULT_FORWARD_POLICY=\"DROP\"/DEFAULT_FORWARD_POLICY=\"ACCEPT\"/" /etc/default/ufw
        sudo ufw reload 2>/dev/null
        echo "   Fixed UFW forward policy (was DROP, now ACCEPT)"
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Fix IndeedHub for iframe + NIP-07: remove X-Frame-Options, inject nostr-provider.js,
    # resolve container IPs for nginx proxy (DNS resolver 127.0.0.11 is unreliable in podman)
    progress "Fixing IndeedHub for NIP-07"
    ssh $SSH_OPTS "$TARGET_HOST" '
      podman_quick() { timeout 20 podman "$@"; }
      podman_exec_quick() { timeout 20 podman exec "$@"; }

      if podman_quick ps --format "{{.Names}}" 2>/dev/null | grep -q "^indeedhub$"; then
        CHANGED=false
        NETWORK=$(podman_quick inspect indeedhub --format "{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}}{{end}}" 2>/dev/null || true)

        # Remove X-Frame-Options so iframe works
        if podman_exec_quick indeedhub grep -q "X-Frame-Options" /etc/nginx/conf.d/default.conf 2>/dev/null; then
          podman_exec_quick indeedhub sed -i "/X-Frame-Options/d" /etc/nginx/conf.d/default.conf || true
          CHANGED=true
          echo "   Removed X-Frame-Options from IndeedHub"
        fi

        # Fix Host header for NIP-98 auth — $host strips port, $http_host preserves it
        podman_exec_quick indeedhub sh -c "sed -i '"'"'s/proxy_set_header Host \$host;/proxy_set_header Host \$http_host;/g'"'"' /etc/nginx/conf.d/default.conf" 2>/dev/null && CHANGED=true && echo "   Fixed Host header for NIP-98 auth" || true

        # Inject nostr-provider.js for NIP-07 signing
        if ! podman_exec_quick indeedhub test -f /usr/share/nginx/html/nostr-provider.js 2>/dev/null; then
          podman_quick cp /opt/archipelago/web-ui/nostr-provider.js indeedhub:/usr/share/nginx/html/nostr-provider.js 2>/dev/null || true
          echo "   Copied nostr-provider.js into IndeedHub"
        fi

        # Add nostr-provider.js + sub_filter to nginx config
        if ! podman_exec_quick indeedhub grep -q "nostr-provider" /etc/nginx/conf.d/default.conf 2>/dev/null; then
          podman_exec_quick indeedhub cat /etc/nginx/conf.d/default.conf > /tmp/ih-nginx.conf 2>/dev/null || true
          # Add nostr-provider location block before sw.js block
          sed -i "/location = \/sw.js {/i\\    location = /nostr-provider.js {\n        add_header Cache-Control \"no-cache, no-store, must-revalidate\";\n        expires off;\n    }\n" /tmp/ih-nginx.conf
          # Add sub_filter for nostr-provider injection
          sed -i "/try_files.*index.html/a\\        sub_filter_once on;\n        sub_filter '"'"'</head>'"'"' '"'"'<script src=\"/nostr-provider.js\"></script></head>'"'"';" /tmp/ih-nginx.conf
          podman_quick cp /tmp/ih-nginx.conf indeedhub:/etc/nginx/conf.d/default.conf 2>/dev/null || true
          rm -f /tmp/ih-nginx.conf
          CHANGED=true
          echo "   Injected nostr-provider.js into IndeedHub nginx"
        fi

        # Replace DNS-based upstream resolution with hardcoded container IPs
        # (podman DNS resolver 127.0.0.11 is unreliable, causing 502 errors)
        API_IP=$(podman_quick inspect indeedhub-build_api_1 --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)
        MINIO_IP=$(podman_quick inspect indeedhub-minio --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)
        RELAY_IP=$(podman_quick inspect indeedhub-relay --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)

        if [ -n "$API_IP" ] && [ -n "$MINIO_IP" ] && [ -n "$RELAY_IP" ]; then
          podman_exec_quick indeedhub cat /etc/nginx/conf.d/default.conf > /tmp/ih-nginx.conf 2>/dev/null || true
          # Remove DNS resolver lines and replace upstream variables with hardcoded IPs
          sed -i "s|resolver 127.0.0.11 valid=30s ipv6=off;||g" /tmp/ih-nginx.conf
          sed -i "s|set \$api_upstream http://api:4000;|set \$api_upstream http://$API_IP:4000;|g" /tmp/ih-nginx.conf
          sed -i "s|set \$minio_upstream http://minio:9000;|set \$minio_upstream http://$MINIO_IP:9000;|g" /tmp/ih-nginx.conf
          sed -i "s|set \$relay_upstream http://relay:8080;|set \$relay_upstream http://$RELAY_IP:8080;|g" /tmp/ih-nginx.conf
          sed -i "s|proxy_set_header Host \$host;|proxy_set_header Host \$http_host;|g" /tmp/ih-nginx.conf
          podman_quick cp /tmp/ih-nginx.conf indeedhub:/etc/nginx/conf.d/default.conf 2>/dev/null || true
          rm -f /tmp/ih-nginx.conf
          CHANGED=true
          echo "   Patched IndeedHub nginx with container IPs (API=$API_IP MINIO=$MINIO_IP RELAY=$RELAY_IP)"
        fi

        if [ "$CHANGED" = true ]; then
          podman_exec_quick indeedhub nginx -s reload 2>/dev/null || true
        fi
      fi
    ' 2>&1 | sed 's/^/   /' || true

    # Run container doctor — auto-fix common container health issues
    progress "Running container doctor"
    "$SCRIPT_DIR/container-doctor.sh" "$TARGET_HOST" 2>&1 | sed 's/^/   /' || true

    # Post-deploy health check — wait up to 60s for server to come healthy
    echo ""
    progress "Post-deploy health check"
    HEALTH_OK=false
    for i in $(seq 1 12); do
        POST_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || { echo "WARNING: Post-deploy health check failed for $TARGET_IP_ONLY" >&2; echo "000"; })
        if [ "$POST_HEALTH" = "200" ]; then
            echo "   Health: OK (200) after $((i * 5))s"
            HEALTH_OK=true
            break
        fi
        echo "   Health: $POST_HEALTH (waiting... ${i}/12)"
        sleep 5
    done
    if [ "$HEALTH_OK" = false ]; then
        echo "   ⚠️  Server did not become healthy within 60s (last: $POST_HEALTH)"
        echo "   Attempting automatic rollback..."
        ssh $SSH_OPTS "$TARGET_HOST" '
            if [ -f /opt/archipelago/rollback/archipelago.bak ]; then
                sudo systemctl stop archipelago 2>/dev/null
                sudo cp /opt/archipelago/rollback/archipelago.bak /usr/local/bin/archipelago
                if [ -f /opt/archipelago/rollback/web-ui.tar ]; then
                    sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +
                    sudo tar xf /opt/archipelago/rollback/web-ui.tar -C /opt/archipelago/web-ui
                fi
                sudo systemctl start archipelago
                echo "ROLLBACK_DONE"
            else
                echo "NO_ROLLBACK_AVAILABLE"
            fi
        ' 2>/dev/null | while IFS= read -r line; do
            if [ "$line" = "ROLLBACK_DONE" ]; then
                echo "   🔄 Rollback complete — previous version restored"
            elif [ "$line" = "NO_ROLLBACK_AVAILABLE" ]; then
                echo "   ⚠️  No rollback backup available"
            fi
        done
        echo "   Check: sudo journalctl -u archipelago -n 50"
    fi

    DEPLOY_END=$(date +%s)
    DEPLOY_ELAPSED=$((DEPLOY_END - DEPLOY_START))

    # Append to local deploy history log (gitignored)
    DEPLOY_LOG="$PROJECT_DIR/scripts/deploy-history.log"
    echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) | $DEPLOY_BRANCH@$DEPLOY_COMMIT | dirty=$DEPLOY_DIRTY | target=$TARGET_HOST | ${DEPLOY_ELAPSED}s" >> "$DEPLOY_LOG"

    # Auto-tag successful deploys (only on clean commits, skip if already tagged)
    if [ "$DEPLOY_DIRTY" = "0" ]; then
        EXISTING_TAG=$(git tag --points-at "$DEPLOY_COMMIT" 2>/dev/null | grep "^v" | head -1)
        if [ -z "$EXISTING_TAG" ]; then
            LAST_ALPHA=$(git tag -l 'v1.2.0-alpha.*' | sort -V | tail -1 | sed 's/.*alpha\.//')
            NEXT_ALPHA=$(( ${LAST_ALPHA:-0} + 1 ))
            DEPLOY_TAG="v1.2.0-alpha.${NEXT_ALPHA}"
            git tag -a "$DEPLOY_TAG" "$DEPLOY_COMMIT" -m "Auto-tagged by deploy to $TARGET_IP_ONLY" 2>/dev/null && \
                echo "   Tagged: $DEPLOY_TAG" || true
        fi
    fi

    echo ""
    echo "$(timestamp) ✅ Deployed to live system! (${DEPLOY_ELAPSED}s total)"
    echo "   Commit: $DEPLOY_BRANCH @ $DEPLOY_COMMIT (dirty=$DEPLOY_DIRTY)"
    echo "   Backend: $(ssh $SSH_OPTS "$TARGET_HOST" 'sudo systemctl is-active archipelago')"
    echo "   Web UI: http://$TARGET_IP_ONLY"
    echo "   PWA install: https://$TARGET_IP_ONLY (use HTTPS, accept cert once, then Install app)"
else
    echo ""
    echo "✅ Build complete!"
    echo ""
    echo "To test frontend dev server:"
    echo "   ssh $TARGET_HOST"
    echo "   cd ~/archy/neode-ui && npm run dev -- --host 0.0.0.0"
    echo "   Then open: http://$(echo "$TARGET_HOST" | cut -d@ -f2):5173"
    echo ""
    echo "To deploy to live system:"
    echo "   ./scripts/deploy-to-target.sh --live"
fi