archy/scripts/deploy-to-target.sh

#!/bin/bash
#
# Deploy Archipelago code to the HP ProDesk target
#
# Usage:
#   ./scripts/deploy-to-target.sh                # Sync and rebuild
#   ./scripts/deploy-to-target.sh --quick        # Sync only, no rebuild
#   ./scripts/deploy-to-target.sh --live         # Deploy to live system (default: 192.168.1.228)
#   ./scripts/deploy-to-target.sh --both         # Deploy to 228, then copy to 198
#   ./scripts/deploy-to-target.sh --frontend-only  # Frontend-only deploy (skip Rust build + container rebuilds)
#

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

# Load deploy config (password etc.) - deploy-config.sh is gitignored
[ -f "$SCRIPT_DIR/deploy-config.sh" ] && . "$SCRIPT_DIR/deploy-config.sh"

# Configuration
TARGET_HOST="${ARCHIPELAGO_TARGET:-archipelago@192.168.1.228}"
TARGET_DIR="/home/archipelago/archy"
SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
SSH_OPTS="-o StrictHostKeyChecking=no -i $SSH_KEY"

DEPLOY_START=$(date +%s)
timestamp() { echo "[$(date +%H:%M:%S)]"; }

echo "╔════════════════════════════════════════════════════════════════╗"
echo "║  Deploying to Archipelago Target                              ║"
echo "╚════════════════════════════════════════════════════════════════╝"
echo ""
echo "$(timestamp) Target: $TARGET_HOST"
echo ""

# Parse arguments
QUICK=false
LIVE=false
BOTH=false
FRONTEND_ONLY=false
for arg in "$@"; do
    case $arg in
        --quick) QUICK=true ;;
        --live) LIVE=true ;;
        --both) BOTH=true ;;
        --frontend-only) FRONTEND_ONLY=true; LIVE=true ;;
    esac
done

# Section timing helper
section_start() { SECTION_START=$(date +%s); }
section_end() {
    local elapsed=$(($(date +%s) - SECTION_START))
    echo "   (${elapsed}s)"
}

# SSH connectivity pre-flight check
echo "$(timestamp) Checking SSH connectivity..."
if ! ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1; then
    echo "   ERROR: Cannot connect to $TARGET_HOST"
    echo "   Check that the server is on and reachable."
    exit 1
fi
echo "   Connected."

# Pre-deploy health check (informational — warns but does not block)
TARGET_IP_ONLY="$(echo "$TARGET_HOST" | cut -d@ -f2)"
PRE_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || echo "000")
if [ "$PRE_HEALTH" = "200" ]; then
    echo "   Server health: OK (200)"
else
    echo "   ⚠️  Server health: $PRE_HEALTH (may be down or unhealthy — deploying anyway)"
fi
echo ""

# When --both: deploy to 228 first, then copy to 198
if [ "$BOTH" = true ]; then
    echo "Deploying to both servers (228, then 198)..."
    "$0" --live
    echo ""
    echo "📤 Copying to 192.168.1.198 (no rsync/cargo on that node)..."
    scp $SSH_OPTS archipelago@192.168.1.228:$TARGET_DIR/core/target/release/archipelago /tmp/archipelago-both 2>/dev/null || true
    scp $SSH_OPTS /tmp/archipelago-both archipelago@192.168.1.198:/tmp/archipelago-new
    ssh $SSH_OPTS archipelago@192.168.1.228 "cd $TARGET_DIR && tar cf - web/dist/neode-ui 2>/dev/null" | ssh $SSH_OPTS archipelago@192.168.1.198 "mkdir -p /tmp/web-deploy && cd /tmp/web-deploy && tar xf -"
    ssh $SSH_OPTS archipelago@192.168.1.198 '
        sudo systemctl stop archipelago
        sudo cp /tmp/archipelago-new /usr/local/bin/archipelago
        sudo chmod +x /usr/local/bin/archipelago
        rm -f /tmp/archipelago-new
        sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +
        sudo cp -r /tmp/web-deploy/web/dist/neode-ui/* /opt/archipelago/web-ui/ 2>/dev/null || true
        sudo chown -R 1000:1000 /opt/archipelago/web-ui
        sudo systemctl start archipelago
        sudo systemctl restart nginx
        echo "   ✅ 192.168.1.198 deployed"
    '
    rm -f /tmp/archipelago-both
    exit 0
fi

# Sync code
section_start
echo "$(timestamp) 📦 Syncing code..."
rsync -avz --delete \
    -e "ssh $SSH_OPTS" \
    --exclude 'node_modules' \
    --exclude 'target' \
    --exclude 'dist' \
    --exclude '.git' \
    --exclude 'image-recipe/build' \
    --exclude 'image-recipe/results' \
    "$PROJECT_DIR/" "$TARGET_HOST:$TARGET_DIR/"
section_end

if [ "$QUICK" = true ]; then
    echo ""
    echo "✅ Quick sync complete!"
    exit 0
fi

# Build on target
echo ""
echo "$(timestamp) 🔨 Building on target..."

# Frontend
section_start
echo "$(timestamp)    Building frontend (vue-tsc + vite)..."
ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/neode-ui && npm install --silent && npm run build" 2>&1 | sed 's/^/   /'
section_end

# Backend (if Rust is installed) — skip with --frontend-only
if [ "$FRONTEND_ONLY" = true ]; then
    echo "$(timestamp)    Skipping backend build (--frontend-only)"
elif ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env 2>/dev/null && command -v cargo" >/dev/null 2>&1; then
    section_start
    echo "$(timestamp)    Building backend (Rust release — this takes 1-2 min)..."
    ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env && cd $TARGET_DIR/core && cargo build --release 2>&1" | sed 's/^/   /'
    section_end
else
    echo "   ⚠️  Rust not installed on target, skipping backend build"
fi

if [ "$LIVE" = true ]; then
    echo ""
    echo "$(timestamp) 🚀 Deploying to live system..."

    # Deploy backend (check if binary exists) — skip with --frontend-only
    if [ "$FRONTEND_ONLY" = true ]; then
        echo "$(timestamp)    Skipping backend deploy (--frontend-only)"
    elif ssh $SSH_OPTS "$TARGET_HOST" "[ -f $TARGET_DIR/core/target/release/archipelago ]" 2>/dev/null; then
        echo "$(timestamp)    Deploying backend binary..."
        ssh $SSH_OPTS "$TARGET_HOST" "sudo systemctl stop archipelago"
        ssh $SSH_OPTS "$TARGET_HOST" "sudo cp $TARGET_DIR/core/target/release/archipelago /usr/local/bin/"
    fi

    # Deploy frontend (preserve aiui/ and claude-login.html — they are NOT part of the neode-ui build)
    echo "$(timestamp)    Deploying frontend..."
    ssh $SSH_OPTS "$TARGET_HOST" "sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name 'aiui' ! -name 'claude-login.html' -exec rm -rf {} +"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo cp -rf $TARGET_DIR/web/dist/neode-ui/* /opt/archipelago/web-ui/"
    ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui"

    # Build and deploy AIUI (non-fatal — never delete existing AIUI on failure)
    AIUI_DIR="$PROJECT_DIR/../AIUI"
    AIUI_DIST="$AIUI_DIR/packages/app/dist"
    if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
        # Use pre-built AIUI dist (build with: cd ../AIUI && VITE_BASE_PATH=/aiui/ pnpm build)
        echo "$(timestamp)    Using pre-built AIUI dist..."
        if true; then
            echo "$(timestamp)    Deploying AIUI..."
            ssh $SSH_OPTS "$TARGET_HOST" "sudo mkdir -p /opt/archipelago/web-ui/aiui"
            ssh $SSH_OPTS "$TARGET_HOST" "sudo rm -rf /opt/archipelago/web-ui/aiui/*"
            cd "$AIUI_DIST" && tar cf - . | ssh $SSH_OPTS "$TARGET_HOST" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
            cd "$PROJECT_DIR"
            ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui && sudo chmod 755 /opt/archipelago/web-ui/aiui && sudo find /opt/archipelago/web-ui/aiui -type d -exec chmod 755 {} \;"
        else
            echo "$(timestamp)    ⚠️  AIUI build failed — keeping existing AIUI on server"
        fi
    else
        echo "$(timestamp)    ⚠️  AIUI not found at $AIUI_DIR, skipping"
    fi
    
    # Sync nginx config from image-recipe (single source of truth)
    NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf"
    if [ -f "$NGINX_CFG" ]; then
      echo "$(timestamp)    Syncing nginx config..."
      scp $SSH_OPTS "$NGINX_CFG" "$TARGET_HOST:/tmp/nginx-archipelago.conf" 2>/dev/null || true
      ssh $SSH_OPTS "$TARGET_HOST" '
        sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
        sudo nginx -t 2>&1 && echo "   nginx config OK" || echo "   ⚠️  nginx config test failed, keeping old config"
        rm -f /tmp/nginx-archipelago.conf
      ' 2>/dev/null || true
    fi

    # Restart services
    echo "$(timestamp)    Restarting services..."
    ssh $SSH_OPTS "$TARGET_HOST" "sudo systemctl start archipelago && sudo systemctl restart nginx"

    # Set up HTTPS for PWA installability (browsers require secure context)
    echo "$(timestamp)    Setting up HTTPS for PWA install..."
    ssh $SSH_OPTS "$TARGET_HOST" "sudo bash $TARGET_DIR/scripts/setup-https-dev.sh" 2>&1 | sed 's/^/   /' || true
    
    if [ "$FRONTEND_ONLY" = true ]; then
        echo "$(timestamp)    Skipping container rebuilds (--frontend-only)"
    else
    # Rebuild and recreate LND UI container (port 8081 so Launch from UI and http://host:8081 both work)
    echo "$(timestamp)    Rebuilding LND UI..."
    if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/lnd-ui && (command -v podman >/dev/null 2>&1 && sudo podman build --no-cache -t lnd-ui:latest . || sudo docker build --no-cache -t lnd-ui:latest .)" 2>&1 | tail -12 | sed 's/^/   /'; then
        echo "   Recreating LND UI container (port 8081)..."
        ssh $SSH_OPTS "$TARGET_HOST" '
          DOCKER=podman
          command -v podman >/dev/null 2>&1 || DOCKER=docker
          for c in $(sudo $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i lnd-ui); do
            [ -n "$c" ] && sudo $DOCKER stop "$c" 2>/dev/null; sudo $DOCKER rm -f "$c" 2>/dev/null
          done
          sudo $DOCKER run -d --name archy-lnd-ui -p 8081:80 --restart unless-stopped lnd-ui:latest
        ' 2>&1 | sed 's/^/   /' || true
    fi

    # Rebuild and recreate Electrs UI container (port 50002)
    echo "$(timestamp)    Rebuilding Electrs UI..."
    if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/electrs-ui && (command -v podman >/dev/null 2>&1 && sudo podman build --no-cache -t electrs-ui:latest . || sudo docker build --no-cache -t electrs-ui:latest .)" 2>&1 | tail -12 | sed 's/^/   /'; then
        echo "   Recreating Electrs UI container (port 50002, host network)..."
        ssh $SSH_OPTS "$TARGET_HOST" '
          DOCKER=podman
          command -v podman >/dev/null 2>&1 || DOCKER=docker
          for c in $(sudo $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i electrs-ui); do
            [ -n "$c" ] && sudo $DOCKER stop "$c" 2>/dev/null; sudo $DOCKER rm -f "$c" 2>/dev/null
          done
          sudo $DOCKER run -d --name archy-electrs-ui --network host --restart unless-stopped electrs-ui:latest
        ' 2>&1 | sed 's/^/   /' || true
    fi

    # Bitcoin Knots: required for Mempool, Electrs, BTCPay, Fedimint
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    echo "$(timestamp)    Ensuring Bitcoin Knots..."
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      sudo \$DOCKER network create archy-net 2>/dev/null || true
      NET_OPT='--network archy-net'
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|archy-bitcoin-knots'; then
        echo '   Creating Bitcoin Knots (mainnet, archipelago RPC)...'
        sudo mkdir -p /var/lib/archipelago/bitcoin
        sudo \$DOCKER run -d --name bitcoin-knots --restart unless-stopped \$NET_OPT \
          -p 8332:8332 -p 8333:8333 \
          -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
          docker.io/bitcoinknots/bitcoin:latest \
          -server=1 -txindex=1 \
          -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
          -rpcuser=archipelago -rpcpassword=archipelago123 \
          -dbcache=4096
        echo '   Bitcoin Knots started (sync may take hours)'
      else
        sudo \$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Fix Mempool: clean duplicates, ensure full stack - mysql, backend (8999), frontend (4080)
    echo "$(timestamp)    Fixing Mempool stack..."
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      NET_OPT='--network archy-net'
      # Clean any duplicate/old mempool containers (user may have two versions)
      # EXCLUDE mempool-electrs - indexing takes days, do not recreate on every deploy
      for c in mempool mempool-api mempool-web archy-mempool-api archy-mempool-web; do
        sudo \$DOCKER stop \$c 2>/dev/null
        sudo \$DOCKER rm -f \$c 2>/dev/null
      done
      # Create mysql-mempool if missing
      if ! sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'mysql-mempool|archy-mempool-db'; then
        echo '   Creating mysql-mempool...'
        sudo mkdir -p /var/lib/archipelago/mysql-mempool
        sudo \$DOCKER run -d --name archy-mempool-db --restart unless-stopped \$NET_OPT \
          -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
          -e MYSQL_DATABASE=mempool \
          -e MYSQL_USER=mempool \
          -e MYSQL_PASSWORD=mempoolpass \
          -e MYSQL_ROOT_PASSWORD=rootpass \
          docker.io/mariadb:10.11
        sleep 3
      fi
      MYSQL_CNT=\$(sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
      MYSQL_CNT=\${MYSQL_CNT:-archy-mempool-db}
      # Ensure DB is on archy-net so mempool-api can resolve it
      sudo \$DOCKER network connect archy-net \$MYSQL_CNT 2>/dev/null || true
      # Create mempool-electrs ONLY if missing - do NOT recreate (indexing takes days, data is 800GB+)
      # One-time migration: if on bridge (wrong network), recreate with archy-net so it can reach bitcoin-knots
      if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q mempool-electrs; then
        MEMPOOL_ELECTRS_NET=\$(sudo \$DOCKER inspect mempool-electrs --format '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}}{{end}}' 2>/dev/null || true)
        if [ \"\$MEMPOOL_ELECTRS_NET\" = \"bridge\" ] || [ \"\$MEMPOOL_ELECTRS_NET\" = \"\" ]; then
          echo '   Migrating mempool-electrs to archy-net (preserving 800GB+ index)...'
          sudo \$DOCKER stop mempool-electrs 2>/dev/null
          sudo \$DOCKER rm mempool-electrs 2>/dev/null
        fi
      fi
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-electrs; then
        if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q mempool-electrs; then
          echo '   Starting existing mempool-electrs (preserving index)...'
          sudo \$DOCKER start mempool-electrs 2>/dev/null || true
        else
          echo '   Creating mempool-electrs (indexer - may take days to sync, do not recreate)...'
          sudo mkdir -p /var/lib/archipelago/mempool-electrs
          # Use archy-net + bitcoin-knots for reliable Bitcoin connectivity (not host IP from bridge)
          sudo \$DOCKER run -d --name mempool-electrs --restart unless-stopped \$NET_OPT \
            -p 50001:50001 \
            -v /var/lib/archipelago/mempool-electrs:/data \
            docker.io/mempool/electrs:latest \
            --daemon-rpc-addr bitcoin-knots:8332 \
            --cookie archipelago:archipelago123 \
            --jsonrpc-import \
            --electrum-rpc-addr 0.0.0.0:50001 \
            --db-dir /data \
            --lightmode
        fi
      fi
      # Create/recreate mempool-api (backend on 8999) - required for mempool to work
      for c in \$(sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mempool-api|archy-mempool-api'); do
        echo '   Recreating mempool-api (backend)...'
        sudo \$DOCKER stop \"\$c\" 2>/dev/null
        sudo \$DOCKER rm -f \"\$c\" 2>/dev/null
      done
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
        echo '   Creating mempool-api (backend)...'
        sudo mkdir -p /var/lib/archipelago/mempool
        sudo \$DOCKER run -d --name mempool-api --restart unless-stopped \$NET_OPT \
        -p 8999:8999 \
        -v /var/lib/archipelago/mempool:/data \
        -e MEMPOOL_BACKEND=electrum \
        -e ELECTRUM_HOST=mempool-electrs \
        -e ELECTRUM_PORT=50001 \
        -e ELECTRUM_TLS_ENABLED=false \
        -e CORE_RPC_HOST=\$TARGET_IP \
        -e CORE_RPC_PORT=8332 \
        -e CORE_RPC_USERNAME=archipelago \
        -e CORE_RPC_PASSWORD=archipelago123 \
        -e DATABASE_ENABLED=true \
        -e DATABASE_HOST=\$MYSQL_CNT \
        -e DATABASE_DATABASE=mempool \
        -e DATABASE_USERNAME=mempool \
        -e DATABASE_PASSWORD=mempoolpass \
        docker.io/mempool/backend:v2.5.0
      fi
      # Recreate mempool frontend - handle both 'mempool' and 'mempool-web' (frontend was on wrong port 8999)
      for c in \$(sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^mempool\$|mempool-web|archy-mempool-web'); do
        echo '   Recreating mempool frontend on 4080...'
        sudo \$DOCKER stop \"\$c\" 2>/dev/null
        sudo \$DOCKER rm -f \"\$c\" 2>/dev/null
        break
      done
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-mempool-web; then
        echo '   Creating mempool frontend on 4080...'
        sudo \$DOCKER run -d --name archy-mempool-web --restart unless-stopped \$NET_OPT \
          -p 4080:8080 \
          -e FRONTEND_HTTP_PORT=8080 \
          -e BACKEND_MAINNET_HTTP_HOST=mempool-api \
          docker.io/mempool/frontend:v2.5.0
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Fix BTCPay Server: requires PostgreSQL + NBXplorer (BTCPay needs NBXplorer for block indexing)
    echo "$(timestamp)    Fixing BTCPay Server stack..."
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      sudo \$DOCKER network create archy-net 2>/dev/null || true
      NET_OPT='--network archy-net'
      # Ensure bitcoin-knots is on archy-net for NBXplorer/BTCPay to reach it
      sudo \$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
      # Create PostgreSQL for BTCPay if missing
      if ! sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
        echo '   Creating archy-btcpay-db (PostgreSQL)...'
        sudo mkdir -p /var/lib/archipelago/postgres-btcpay
        sudo \$DOCKER run -d --name archy-btcpay-db --restart unless-stopped \$NET_OPT \
          -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
          -e POSTGRES_DB=btcpay \
          -e POSTGRES_USER=btcpay \
          -e POSTGRES_PASSWORD=btcpaypass \
          docker.io/postgres:15-alpine
        sleep 3
      fi
      # Create NBXplorer database in PostgreSQL (NBXplorer needs its own DB)
      sudo \$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \
        sudo \$DOCKER exec -e PGPASSWORD=btcpaypass archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true
      # Create NBXplorer (required by BTCPay - indexes blocks for payment tracking)
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
        if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
          sudo \$DOCKER start archy-nbxplorer 2>/dev/null || true
        else
          echo '   Creating archy-nbxplorer...'
          sudo mkdir -p /var/lib/archipelago/nbxplorer
          sudo \$DOCKER run -d --name archy-nbxplorer --restart unless-stopped \$NET_OPT \
            -p 32838:32838 \
            -v /var/lib/archipelago/nbxplorer:/data \
            -e NBXPLORER_DATADIR=/data \
            -e NBXPLORER_NETWORK=mainnet \
            -e NBXPLORER_CHAINS=btc \
            -e NBXPLORER_BIND=0.0.0.0:32838 \
            -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
            -e NBXPLORER_BTCRPCUSER=archipelago \
            -e NBXPLORER_BTCRPCPASSWORD=archipelago123 \
            -e NBXPLORER_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
            docker.io/nicolasdorier/nbxplorer:2.6.0
          sleep 5
        fi
      fi
      # Recreate btcpay-server with PostgreSQL, NBXplorer URL, and Bitcoin RPC
      for c in btcpay-server archy-btcpay; do
        if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx \"\$c\"; then
          echo '   Recreating btcpay-server with NBXplorer...'
          sudo \$DOCKER stop \"\$c\" 2>/dev/null
          sudo \$DOCKER rm -f \"\$c\" 2>/dev/null
        fi
      done
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
        echo '   Creating btcpay-server on 23000...'
        sudo mkdir -p /var/lib/archipelago/btcpay
        sudo \$DOCKER run -d --name btcpay-server --restart unless-stopped \$NET_OPT \
          -p 23000:49392 \
          -v /var/lib/archipelago/btcpay:/datadir \
          -e ASPNETCORE_URLS=http://0.0.0.0:49392 \
          -e BTCPAY_PROTOCOL=http \
          -e BTCPAY_HOST=\$TARGET_IP:23000 \
          -e BTCPAY_CHAINS=btc \
          -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
          -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
          -e BTCPAY_BTCRPCUSER=archipelago \
          -e BTCPAY_BTCRPCPASSWORD=archipelago123 \
          -e BTCPAY_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
          docker.io/btcpayserver/btcpayserver:1.13.5
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Ensure Immich stack (postgres + redis + server) - creates if missing
    echo "$(timestamp)    Ensuring Immich stack..."
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      # Remove old single-container 'immich' if present (wrong port mapping, conflicts with immich_server)
      if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
        echo '   Removing old immich container (use immich_server)...'
        sudo \$DOCKER stop immich 2>/dev/null
        sudo \$DOCKER rm -f immich 2>/dev/null
        sudo \$DOCKER start immich_server 2>/dev/null || true
      fi
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
        echo '   Creating Immich stack...'
        sudo mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
        sudo \$DOCKER network create immich-net 2>/dev/null || true
        if ! sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
          sudo \$DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
            -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
            -e POSTGRES_PASSWORD=immichpass -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
            ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>/dev/null || true
          sleep 5
        fi
        if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
          sudo \$DOCKER run -d --name immich_redis --restart unless-stopped --network immich-net \
            docker.io/valkey/valkey:7-alpine 2>/dev/null || true
          sleep 2
        fi
        if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
          sudo \$DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
            -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
            -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=immichpass \
            -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
            -e UPLOAD_LOCATION=/usr/src/app/upload \
            ghcr.io/immich-app/immich-server:release 2>/dev/null || true
        fi
        echo '   Immich stack created (may take 1-2 min to become ready)'
      else
        echo '   Immich already running'
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Tor: global hidden services - each service gets its own .onion address
    echo "$(timestamp)    Setting up Tor..."
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      TARGET_IP='$TARGET_IP'
      sudo mkdir -p /var/lib/archipelago/tor

      # Ensure services.json exists with default services
      SERVICES_JSON=/var/lib/archipelago/tor/services.json
      if [ ! -f "\$SERVICES_JSON" ]; then
        echo '{"services":[
          {"name":"archipelago","local_port":80,"enabled":true},
          {"name":"lnd","local_port":8081,"enabled":true},
          {"name":"btcpay","local_port":23000,"enabled":true},
          {"name":"mempool","local_port":4080,"enabled":true},
          {"name":"fedimint","local_port":8175,"enabled":true}
        ]}' | sudo tee "\$SERVICES_JSON" > /dev/null
      fi

      # Generate torrc dynamically from services.json
      TORRC=/var/lib/archipelago/tor/torrc
      echo 'SocksPort 9050' | sudo tee "\$TORRC" > /dev/null
      echo 'ControlPort 0' | sudo tee -a "\$TORRC" > /dev/null
      echo 'DataDirectory /var/lib/archipelago/tor' | sudo tee -a "\$TORRC" > /dev/null

      # Read services from JSON and generate HiddenService lines
      # Use python3 (available on Debian 12) to parse JSON and emit torrc lines
      python3 << 'PYEOF' | sudo tee -a "\$TORRC" > /dev/null
import json
try:
    with open("/var/lib/archipelago/tor/services.json") as f:
        cfg = json.load(f)
    for svc in cfg.get("services", []):
        if svc.get("enabled", True):
            n = svc["name"]
            p = svc["local_port"]
            print("HiddenServiceDir /var/lib/archipelago/tor/hidden_service_%s/" % n)
            print("HiddenServicePort 80 127.0.0.1:%d" % p)
except Exception:
    # Fallback defaults
    for n, p in [("archipelago",80),("lnd",8081),("btcpay",23000),("mempool",4080),("fedimint",8175)]:
        print("HiddenServiceDir /var/lib/archipelago/tor/hidden_service_%s/" % n)
        print("HiddenServicePort 80 127.0.0.1:%d" % p)
PYEOF
      for c in \$(sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'archy-tor|^tor\$'); do
        sudo \$DOCKER stop \"\$c\" 2>/dev/null
        sudo \$DOCKER rm -f \"\$c\" 2>/dev/null
      done
      if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-tor; then
        echo '   Creating Tor container (host network for hidden services)...'
        if sudo \$DOCKER run -d --name archy-tor --restart unless-stopped --network host \
          -v /var/lib/archipelago/tor:/var/lib/archipelago/tor \
          --entrypoint tor \
          docker.io/andrius/alpine-tor:latest \
          -f /var/lib/archipelago/tor/torrc 2>/dev/null; then
          echo '   Tor container started (andrius/alpine-tor)'
        else
          echo '   Tor container image failed, trying system tor...'
          sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null || true
          if command -v tor >/dev/null 2>&1; then
            sudo cp /var/lib/archipelago/tor/torrc /etc/tor/torrc 2>/dev/null || true
            sudo chown -R debian-tor:debian-tor /var/lib/archipelago/tor 2>/dev/null || true
            # Let archipelago user read hostname files (group-readable)
            sudo usermod -aG debian-tor archipelago 2>/dev/null || true
            sudo chmod 750 /var/lib/archipelago/tor 2>/dev/null || true
            sudo find /var/lib/archipelago/tor -name 'hidden_service_*' -type d -exec chmod 750 {} \; 2>/dev/null || true
            sudo find /var/lib/archipelago/tor -name 'hostname' -exec chmod 640 {} \; 2>/dev/null || true
            # Systemd override so Tor can write to custom data dir
            sudo mkdir -p /etc/systemd/system/tor@default.service.d
            echo -e '[Service]\nReadWriteDirectories=-/var/lib/archipelago/tor' | sudo tee /etc/systemd/system/tor@default.service.d/override.conf > /dev/null
            sudo systemctl daemon-reload
            sudo systemctl enable tor 2>/dev/null
            sudo systemctl restart tor 2>/dev/null
            echo '   Using system Tor daemon'
          fi
        fi
      fi
    " 2>&1 | sed 's/^/   /' || true

    # Tor diagnostic: check if hostname files exist (may take 30-60s after Tor starts)
    echo "   Checking Tor hostname files..."
    ssh $SSH_OPTS "$TARGET_HOST" "
      # Check all hidden_service_* dirs for hostname files
      for dir in /var/lib/archipelago/tor/hidden_service_*/; do
        [ -d \"\$dir\" ] || continue
        svc=\$(basename \"\$dir\" | sed 's/hidden_service_//')
        f=\"\${dir}hostname\"
        if [ -f \"\$f\" ]; then
          echo \"   ✓ \$svc: \$(cat \$f)\"
        else
          echo \"   ✗ \$svc: hostname not yet generated (Tor may need 30-60s)\"
        fi
      done
    " 2>&1 | sed 's/^/   /' || true

    # Recreate Fedimint with FM_API_URL for Guardian UI (fixes "Api URL must be configured")
    section_start
    echo "$(timestamp)    Fixing Fedimint API URL..."
    TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
    TIMEOUT_CMD=""
    command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90"
    command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90"
    ($TIMEOUT_CMD ssh $SSH_OPTS "$TARGET_HOST" "
      DOCKER=podman
      command -v podman >/dev/null 2>&1 || DOCKER=docker
      for c in \$(sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^fedimint\$'); do
        echo '   Recreating fedimint with FM_API_URL...'
        sudo \$DOCKER stop \"\$c\" 2>/dev/null
        sudo \$DOCKER rm -f \"\$c\" 2>/dev/null
        sudo \$DOCKER run -d --name fedimint --restart unless-stopped \
          -p 8173:8173 -p 8174:8174 -p 8175:8175 \
          -v /var/lib/archipelago/fedimint:/data \
          -e FM_DATA_DIR=/data \
          -e FM_BITCOIND_USERNAME=archipelago \
          -e FM_BITCOIND_PASSWORD=archipelago123 \
          -e FM_BITCOIN_NETWORK=bitcoin \
          -e FM_BIND_P2P=0.0.0.0:8173 \
          -e FM_BIND_API=0.0.0.0:8174 \
          -e FM_BIND_UI=0.0.0.0:8175 \
          -e FM_P2P_URL=fedimint://$TARGET_IP:8173 \
          -e FM_API_URL=ws://$TARGET_IP:8174 \
          -e FM_BITCOIND_URL=http://$TARGET_IP:8332 \
          docker.io/fedimint/fedimintd:v0.10.0
        break
      done

      # Ensure Fedimint Gateway companion container
      # Auto-detect LND: if running with credentials, use lnd mode; otherwise use ldk (built-in)
      sudo \$DOCKER rm -f fedimint-gateway 2>/dev/null || true
      echo '   Creating fedimint-gateway...'
      sudo mkdir -p /var/lib/archipelago/fedimint-gateway
      LND_CERT=/var/lib/archipelago/lnd/tls.cert
      LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
      GW_COMMON=\"-p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data docker.io/fedimint/gatewayd:v0.10.0 gatewayd --data-dir /data --listen 0.0.0.0:8176 --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' --network bitcoin --bitcoind-url http://$TARGET_IP:8332 --bitcoind-username archipelago --bitcoind-password archipelago123\"
      if sudo \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then
        echo '   LND detected — using lnd mode'
        sudo \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
          -p 8176:8176 \
          -v /var/lib/archipelago/fedimint-gateway:/data \
          -v /var/lib/archipelago/lnd/tls.cert:/lnd/tls.cert:ro \
          -v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \
          docker.io/fedimint/gatewayd:v0.10.0 \
          gatewayd --data-dir /data --listen 0.0.0.0:8176 \
          --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
          --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
          --bitcoind-username archipelago --bitcoind-password archipelago123 \
          lnd --lnd-rpc-host $TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon
      else
        echo '   No LND found — using ldk (built-in Lightning)'
        sudo \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
          -p 8176:8176 -p 9737:9737 \
          -v /var/lib/archipelago/fedimint-gateway:/data \
          docker.io/fedimint/gatewayd:v0.10.0 \
          gatewayd --data-dir /data --listen 0.0.0.0:8176 \
          --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
          --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
          --bitcoind-username archipelago --bitcoind-password archipelago123 \
          ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway
      fi
    " 2>&1 | sed 's/^/   /') || echo "   (Fedimint fix timed out or skipped - run manually if needed)"
    section_end

    fi  # end FRONTEND_ONLY guard

    # Post-deploy health check — wait up to 60s for server to come healthy
    echo ""
    echo "$(timestamp) 🩺 Post-deploy health check..."
    HEALTH_OK=false
    for i in $(seq 1 12); do
        POST_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || echo "000")
        if [ "$POST_HEALTH" = "200" ]; then
            echo "   Health: OK (200) after $((i * 5))s"
            HEALTH_OK=true
            break
        fi
        echo "   Health: $POST_HEALTH (waiting... ${i}/12)"
        sleep 5
    done
    if [ "$HEALTH_OK" = false ]; then
        echo "   ⚠️  Server did not become healthy within 60s (last: $POST_HEALTH)"
        echo "   Rollback: ssh $TARGET_HOST and check 'sudo journalctl -u archipelago -n 50'"
    fi

    DEPLOY_END=$(date +%s)
    DEPLOY_ELAPSED=$((DEPLOY_END - DEPLOY_START))
    echo ""
    echo "$(timestamp) ✅ Deployed to live system! (${DEPLOY_ELAPSED}s total)"
    echo "   Backend: $(ssh $SSH_OPTS "$TARGET_HOST" 'sudo systemctl is-active archipelago')"
    echo "   Web UI: http://$TARGET_IP_ONLY"
    echo "   PWA install: https://$TARGET_IP_ONLY (use HTTPS, accept cert once, then Install app)"
else
    echo ""
    echo "✅ Build complete!"
    echo ""
    echo "To test frontend dev server:"
    echo "   ssh $TARGET_HOST"
    echo "   cd ~/archy/neode-ui && npm run dev -- --host 0.0.0.0"
    echo "   Then open: http://$(echo $TARGET_HOST | cut -d@ -f2):5173"
    echo ""
    echo "To deploy to live system:"
    echo "   ./scripts/deploy-to-target.sh --live"
fi