#!/bin/bash # # Full deploy for Tailscale (or any remote) nodes — split-mode SSH for stability # # Each step is a separate short SSH session to handle unstable Tailscale connections. # Auto-detects build capability: builds locally if cargo/npm present, otherwise copies # pre-built artifacts from the primary build server (.228). # # Usage: # ./scripts/deploy-tailscale.sh archipelago@100.82.97.63 # Single node # ./scripts/deploy-tailscale.sh archipelago@100.122.84.60 # Arch 2 (can build) # ./scripts/deploy-tailscale.sh archipelago@100.124.105.113 # Arch 3 (copy-only) # ./scripts/deploy-tailscale.sh --all # All 3 Tailscale nodes # set -eo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" TARGET_DIR="/home/archipelago/archy" # Load deploy config defaults (IP addresses etc.) [ -f "$SCRIPT_DIR/deploy-config-defaults.sh" ] && . "$SCRIPT_DIR/deploy-config-defaults.sh" # Load deploy config (gitignored — overrides defaults) [ -f "$SCRIPT_DIR/deploy-config.sh" ] && . "$SCRIPT_DIR/deploy-config.sh" # Source pinned image versions (single source of truth) [ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh" # Source shared utility library [ -f "$SCRIPT_DIR/lib/common.sh" ] && . "$SCRIPT_DIR/lib/common.sh" SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}" SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o ConnectTimeout=10 -i $SSH_KEY" BUILD_SOURCE_LAN="archipelago@${DEFAULT_PRIMARY:-192.168.1.228}" BUILD_SOURCE_TS="archipelago@$(tailscale status 2>/dev/null | grep 'archipelago-0' | awk '{print $1}')" # Try LAN first, fall back to Tailscale if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$BUILD_SOURCE_LAN" "echo ok" >/dev/null 2>&1; then BUILD_SOURCE="$BUILD_SOURCE_LAN" elif [ "$BUILD_SOURCE_TS" != "archipelago@" ] && ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$BUILD_SOURCE_TS" "echo ok" >/dev/null 2>&1; then BUILD_SOURCE="$BUILD_SOURCE_TS" echo "Build source: using Tailscale IP (LAN unreachable)" else BUILD_SOURCE="$BUILD_SOURCE_LAN" echo "WARNING: Build source may be unreachable" fi BUILD_DIR="/home/archipelago/archy" # Node registry TAILSCALE_NODES=( "archipelago@${TAILSCALE_ARCH1:-100.82.97.63}" "archipelago@${TAILSCALE_ARCH2:-100.122.84.60}" "archipelago@${TAILSCALE_ARCH3:-100.124.105.113}" ) TAILSCALE_NAMES=("Arch 1" "Arch 2" "Arch 3") # Git state DEPLOY_COMMIT=$(git -C "$PROJECT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") DEPLOY_COMMIT_FULL=$(git -C "$PROJECT_DIR" rev-parse HEAD 2>/dev/null || echo "unknown") DEPLOY_BRANCH=$(git -C "$PROJECT_DIR" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") DEPLOY_DIRTY=false [ -n "$(git -C "$PROJECT_DIR" status --porcelain 2>/dev/null | grep -v '^??' | grep -v '\.claude/memory/')" ] && DEPLOY_DIRTY=true DEPLOY_START=$(date +%s) ts() { echo "[$(date +%H:%M:%S)]"; } step_num=0 step() { step_num=$((step_num + 1)); echo ""; echo "$(ts) ━━━ Step $step_num: $1"; } # Temp directory for intermediate files (cleaned up on exit) TMPDIR="/tmp/archipelago-deploy-$$" mkdir -p "$TMPDIR" trap 'rm -rf "$TMPDIR"' EXIT # ── Deploy a single node ───────────────────────────────────────────────── deploy_node() { local TARGET="$1" local NODE_NAME="${2:-$TARGET}" local TARGET_IP="$(echo "$TARGET" | cut -d@ -f2)" step_num=0 echo "" echo "╔════════════════════════════════════════════════════════════════╗" echo "║ Deploying to $NODE_NAME ($TARGET_IP)" echo "╚════════════════════════════════════════════════════════════════╝" echo "$(ts) Branch: $DEPLOY_BRANCH @ $DEPLOY_COMMIT (dirty=$DEPLOY_DIRTY)" # ── Step 1: SSH connectivity ───────────────────────────────────── step "Checking SSH connectivity" if ! ssh $SSH_OPTS "$TARGET" "echo ok" >/dev/null 2>&1; then echo " ERROR: Cannot connect to $TARGET" return 1 fi echo " Connected." # ── Step 2: Prerequisites ──────────────────────────────────────── step "Checking prerequisites" ssh $SSH_OPTS "$TARGET" ' NEED="" command -v rsync >/dev/null 2>&1 || NEED="$NEED rsync" command -v python3 >/dev/null 2>&1 || NEED="$NEED python3" if [ -n "$NEED" ]; then echo " Installing:$NEED" sudo apt-get update -qq && sudo apt-get install -y -qq $NEED 2>&1 | tail -3 else echo " All prerequisites present" fi ' 2>&1 # ── Step 3: Detect build capability ────────────────────────────── step "Detecting build capability" CAN_BUILD=false HAS_CARGO=$(ssh $SSH_OPTS "$TARGET" "source ~/.cargo/env 2>/dev/null; command -v cargo >/dev/null 2>&1 && echo yes || echo no" 2>/dev/null) HAS_NPM=$(ssh $SSH_OPTS "$TARGET" "command -v npm >/dev/null 2>&1 && echo yes || echo no" 2>/dev/null) if [ "$HAS_CARGO" = "yes" ] && [ "$HAS_NPM" = "yes" ]; then CAN_BUILD=true echo " Build capable (cargo + npm present)" else echo " Copy-only (cargo=$HAS_CARGO, npm=$HAS_NPM) — will copy from $BUILD_SOURCE" fi # ── Step 4: Rootful→rootless migration (one-time) ──────────────── step "Checking for rootful containers (migration)" ssh $SSH_OPTS "$TARGET" ' MIGRATION_FLAG="/var/lib/archipelago/.rootless-migrated" if [ -f "$MIGRATION_FLAG" ]; then ROOTLESS=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l) echo " Already migrated ($ROOTLESS rootless containers)" else # Check if rootful podman has any containers (sudo = rootful context) ROOTFUL=$(sudo podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l) ROOTLESS=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l) echo " Rootful: $ROOTFUL, Rootless: $ROOTLESS" if [ "$ROOTFUL" -gt 0 ] && [ "$ROOTFUL" != "$ROOTLESS" ]; then echo " MIGRATING: Stopping $ROOTFUL rootful containers..." sudo podman stop --all --timeout 30 2>/dev/null || true sudo podman rm --all --force 2>/dev/null || true echo " Rootful containers removed (data preserved in /var/lib/archipelago/)" else echo " No rootful containers to migrate" fi sudo touch "$MIGRATION_FLAG" fi ' 2>&1 # ── Step 5: Sync code ──────────────────────────────────────────── step "Syncing code" rsync -az --delete \ --exclude='.git' --exclude='node_modules' --exclude='target/debug' \ --exclude='target/release/deps' --exclude='target/release/build' \ --exclude='target/release/.fingerprint' --exclude='target/release/incremental' \ --exclude='web/dist' --exclude='.DS_Store' --exclude='image-recipe/build' \ --exclude='image-recipe/results' \ -e "ssh $SSH_OPTS" \ "$PROJECT_DIR/" "$TARGET:$TARGET_DIR/" || { echo " rsync failed"; return 1; } echo " Synced." # ── Step 6: Build or copy artifacts ────────────────────────────── if [ "$CAN_BUILD" = true ]; then step "Building frontend on target" ssh $SSH_OPTS "$TARGET" "cd $TARGET_DIR/neode-ui && npm install --silent 2>&1 && npm run build 2>&1" | tail -10 step "Building backend on target" ssh $SSH_OPTS "$TARGET" "source ~/.cargo/env 2>/dev/null && cd $TARGET_DIR/core && cargo build --release 2>&1" | tail -15 BINARY_OK=$(ssh $SSH_OPTS "$TARGET" "[ -f $TARGET_DIR/core/target/release/archipelago ] && echo ok || echo fail" 2>/dev/null) if [ "$BINARY_OK" != "ok" ]; then echo " Backend build failed!"; return 1; fi echo " Build complete." else step "Copying pre-built artifacts from $BUILD_SOURCE" # Verify build source has artifacts BUILD_OK=$(ssh $SSH_OPTS "$BUILD_SOURCE" "[ -f $BUILD_DIR/core/target/release/archipelago ] && echo ok || echo fail" 2>/dev/null) if [ "$BUILD_OK" != "ok" ]; then echo " ERROR: No binary on $BUILD_SOURCE — deploy to .228 first" return 1 fi # Copy binary via local /tmp (SSH pipes unreliable with complex options) echo " Copying binary..." scp $SSH_OPTS "$BUILD_SOURCE:$BUILD_DIR/core/target/release/archipelago" /tmp/archipelago-deploy 2>/dev/null scp $SSH_OPTS /tmp/archipelago-deploy "$TARGET:/tmp/archipelago-new" 2>/dev/null rm -f /tmp/archipelago-deploy # Copy frontend via tar through local echo " Copying frontend..." ssh $SSH_OPTS "$BUILD_SOURCE" "cd $BUILD_DIR && tar cf - web/dist/neode-ui 2>/dev/null" > /tmp/frontend-deploy.tar cat /tmp/frontend-deploy.tar | ssh $SSH_OPTS "$TARGET" "mkdir -p /tmp/web-deploy && cd /tmp/web-deploy && tar xf -" rm -f /tmp/frontend-deploy.tar # Transfer custom UI images (individual tarballs — never combined) echo " Transferring custom UI images..." for ui_img in bitcoin-ui lnd-ui electrs-ui; do HAS_IMG=$(ssh $SSH_OPTS "$BUILD_SOURCE" "podman images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q '${ui_img}:' && echo yes || echo no" 2>/dev/null) if [ "$HAS_IMG" = "yes" ]; then echo " $ui_img..." if ssh $SSH_OPTS "$BUILD_SOURCE" "podman save 'localhost/${ui_img}:local' 2>/dev/null" > "/tmp/${ui_img}.tar" 2>/dev/null && [ -s "/tmp/${ui_img}.tar" ]; then ssh $SSH_OPTS "$TARGET" "podman load" < "/tmp/${ui_img}.tar" 2>&1 | tail -1 else echo " $ui_img: not available on build server, skipping" fi rm -f "/tmp/${ui_img}.tar" else echo " $ui_img: not found on build server, skipping" fi done # Install Node.js if missing (needed for some container builds) if [ "$HAS_NPM" != "yes" ]; then echo " Installing Node.js on target..." ssh $SSH_OPTS "$TARGET" ' curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - 2>&1 | tail -3 sudo apt-get install -y -qq nodejs 2>&1 | tail -3 ' 2>&1 || true fi echo " Artifacts copied." fi # ── Step 7: Rollback backup ────────────────────────────────────── step "Creating rollback backup" ssh $SSH_OPTS "$TARGET" ' sudo mkdir -p /opt/archipelago/rollback [ -f /usr/local/bin/archipelago ] && sudo cp /usr/local/bin/archipelago /opt/archipelago/rollback/archipelago.bak 2>/dev/null || true [ -d /opt/archipelago/web-ui ] && sudo tar cf /opt/archipelago/rollback/web-ui.tar -C /opt/archipelago/web-ui . 2>/dev/null || true echo " Rollback backup created" ' 2>&1 # ── Step 8: Deploy binary ──────────────────────────────────────── step "Deploying binary" ssh $SSH_OPTS "$TARGET" 'sudo systemctl stop archipelago --no-block 2>/dev/null; sleep 2; sudo kill -9 $(pgrep -x archipelago) 2>/dev/null; sleep 1; true' 2>/dev/null if [ "$CAN_BUILD" = true ]; then ssh $SSH_OPTS "$TARGET" "sudo cp $TARGET_DIR/core/target/release/archipelago /usr/local/bin/" else ssh $SSH_OPTS "$TARGET" "sudo cp /tmp/archipelago-new /usr/local/bin/archipelago && sudo chmod +x /usr/local/bin/archipelago && rm -f /tmp/archipelago-new" fi echo " Binary deployed." # ── Step 9: Deploy frontend ────────────────────────────────────── step "Deploying frontend" ssh $SSH_OPTS "$TARGET" 'sudo mkdir -p /opt/archipelago/web-ui && sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +' 2>/dev/null if [ "$CAN_BUILD" = true ]; then ssh $SSH_OPTS "$TARGET" "sudo cp -rf $TARGET_DIR/web/dist/neode-ui/* /opt/archipelago/web-ui/" else ssh $SSH_OPTS "$TARGET" "sudo cp -rf /tmp/web-deploy/web/dist/neode-ui/* /opt/archipelago/web-ui/ 2>/dev/null && rm -rf /tmp/web-deploy" fi ssh $SSH_OPTS "$TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui" echo " Frontend deployed." # ── Step 10: Deploy AIUI ───────────────────────────────────────── step "Deploying AIUI" AIUI_DIST="$PROJECT_DIR/../AIUI/packages/app/dist" if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then ssh $SSH_OPTS "$TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*" (cd "$AIUI_DIST" && tar --no-xattrs -cf - .) | ssh $SSH_OPTS "$TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/ 2>/dev/null" ssh $SSH_OPTS "$TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui" echo " AIUI deployed." else echo " AIUI not found, skipping." fi # ── Step 11: Sync nginx config ─────────────────────────────────── step "Syncing nginx config" NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf" SNIPPETS_DIR="$PROJECT_DIR/image-recipe/configs/snippets" if [ -f "$NGINX_CFG" ]; then scp $SSH_OPTS "$NGINX_CFG" "$TARGET:/tmp/nginx-archipelago.conf" 2>/dev/null || true ssh $SSH_OPTS "$TARGET" ' sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago sudo mkdir -p /etc/nginx/snippets sudo rm -f /etc/nginx/conf.d/external-app-proxies.conf rm -f /tmp/nginx-archipelago.conf ' 2>/dev/null fi if [ -d "$SNIPPETS_DIR" ]; then for f in "$SNIPPETS_DIR"/*.conf; do [ -f "$f" ] && scp $SSH_OPTS "$f" "$TARGET:/tmp/nginx-snippet-$(basename "$f")" 2>/dev/null || true done ssh $SSH_OPTS "$TARGET" ' for f in /tmp/nginx-snippet-*.conf; do [ -f "$f" ] && sudo mv "$f" "/etc/nginx/snippets/$(basename "$f" | sed "s/^nginx-snippet-//")" done ' 2>/dev/null || true fi ssh $SSH_OPTS "$TARGET" 'sudo nginx -t 2>&1 && echo " nginx config OK" || echo " nginx config FAILED"' 2>/dev/null || true # ── Step 12: Sync systemd service ──────────────────────────────── step "Syncing systemd service" SERVICE_FILE="$PROJECT_DIR/image-recipe/configs/archipelago.service" if [ -f "$SERVICE_FILE" ]; then scp $SSH_OPTS "$SERVICE_FILE" "$TARGET:/tmp/archipelago.service" 2>/dev/null || true ssh $SSH_OPTS "$TARGET" ' if ! diff -q /tmp/archipelago.service /etc/systemd/system/archipelago.service >/dev/null 2>&1; then sudo cp /tmp/archipelago.service /etc/systemd/system/archipelago.service sudo systemctl daemon-reload echo " Service file updated" else echo " Service file unchanged" fi rm -f /tmp/archipelago.service ' 2>/dev/null || true fi # ── Step 13: Rootless podman prereqs ───────────────────────────── step "Setting up rootless podman prerequisites" ssh $SSH_OPTS "$TARGET" ' # Allow binding to ports >= 80 if ! grep -q "unprivileged_port_start=80" /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null; then echo "net.ipv4.ip_unprivileged_port_start=80" | sudo tee /etc/sysctl.d/99-rootless-podman.conf > /dev/null sudo sysctl -p /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null echo " Rootless port binding enabled (>=80)" fi # Linger for container persistence if [ "$(loginctl show-user archipelago 2>/dev/null | grep Linger)" != "Linger=yes" ]; then sudo loginctl enable-linger archipelago echo " Linger enabled" fi # Podman socket systemctl --user enable podman.socket 2>/dev/null || true systemctl --user start podman.socket 2>/dev/null || true # Ensure subuid/subgid grep -q "^archipelago:" /etc/subuid 2>/dev/null || { echo "archipelago:100000:65536" | sudo tee -a /etc/subuid > /dev/null echo "archipelago:100000:65536" | sudo tee -a /etc/subgid > /dev/null echo " subuid/subgid configured" } # Ensure /etc/hosts is readable (rootless podman needs it) sudo chmod 644 /etc/hosts 2>/dev/null echo " Rootless prerequisites OK" ' 2>&1 # ── Step 14: Data dirs + UID mapping ───────────────────────────── step "Creating data directories + UID mapping" ssh $SSH_OPTS "$TARGET" ' sudo mkdir -p /var/lib/archipelago/dwn/messages /var/lib/archipelago/dwn/protocols sudo mkdir -p /var/lib/archipelago/content/files /var/lib/archipelago/federation sudo mkdir -p /var/lib/archipelago/identities /var/lib/archipelago/tor-config sudo mkdir -p /var/lib/archipelago/searxng /var/lib/archipelago/vaultwarden sudo mkdir -p /var/lib/archipelago/photoprism /var/lib/archipelago/filebrowser sudo mkdir -p /var/lib/archipelago/nextcloud sudo chown -R archipelago:archipelago /var/lib/archipelago/dwn /var/lib/archipelago/content \ /var/lib/archipelago/federation /var/lib/archipelago/identities /var/lib/archipelago/tor-config 2>/dev/null || true echo " Fixing rootless podman UID mapping..." # Containers running as root (UID 0 → host UID 100000) for dir in lnd electrumx btcpay nbxplorer jellyfin vaultwarden \ home-assistant fedimint fedimint-gateway photoprism ollama filebrowser \ nextcloud uptime-kuma onlyoffice nginx-proxy-manager portainer nostr-rs-relay searxng; do [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null done # Bitcoin Knots: container UID 101 → host UID 100101 [ -d /var/lib/archipelago/bitcoin ] && sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null # Postgres: container UID 70 → host UID 100070 for dir in postgres-btcpay; do [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null done # MariaDB: container UID 999 → host UID 100999 for dir in mempool mysql-mempool; do [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100999:100999 "/var/lib/archipelago/$dir" 2>/dev/null done # Grafana: container UID 472 → host UID 100472 [ -d /var/lib/archipelago/grafana ] && sudo chown -R 100472:100472 /var/lib/archipelago/grafana 2>/dev/null echo " UID mapping done" ' 2>&1 # ── Step 15: Dev mode ──────────────────────────────────────────── step "Configuring dev mode (HTTP cookie support)" ssh $SSH_OPTS "$TARGET" ' if [ -f /etc/systemd/system/archipelago.service.d/override.conf ] && grep -q "ARCHIPELAGO_DEV_MODE=true" /etc/systemd/system/archipelago.service.d/override.conf 2>/dev/null; then echo " Dev mode already enabled" else sudo mkdir -p /etc/systemd/system/archipelago.service.d printf "[Service]\nEnvironment=ARCHIPELAGO_DEV_MODE=true\n" | sudo tee /etc/systemd/system/archipelago.service.d/override.conf > /dev/null sudo systemctl daemon-reload echo " Dev mode enabled" fi ' 2>&1 # ── Step 16: Deploy nostr-provider.js ──────────────────────────── step "Deploying nostr-provider.js" if [ -f "$PROJECT_DIR/neode-ui/public/nostr-provider.js" ]; then scp $SSH_OPTS "$PROJECT_DIR/neode-ui/public/nostr-provider.js" "$TARGET:/tmp/nostr-provider.js" 2>/dev/null && \ ssh $SSH_OPTS "$TARGET" 'sudo cp /tmp/nostr-provider.js /opt/archipelago/web-ui/nostr-provider.js && rm -f /tmp/nostr-provider.js && echo " deployed"' 2>/dev/null else echo " nostr-provider.js not found, skipping" fi # ── Step 17: Deploy udev rule ──────────────────────────────────── UDEV_RULE="$PROJECT_DIR/image-recipe/configs/99-mesh-radio.rules" if [ -f "$UDEV_RULE" ]; then step "Deploying mesh radio udev rule" scp $SSH_OPTS "$UDEV_RULE" "$TARGET:/tmp/99-mesh-radio.rules" 2>/dev/null || true ssh $SSH_OPTS "$TARGET" ' if ! diff -q /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules >/dev/null 2>&1; then sudo cp /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules sudo udevadm control --reload-rules 2>/dev/null echo " Installed" else echo " Unchanged" fi rm -f /tmp/99-mesh-radio.rules ' 2>/dev/null || true fi # ── Step 18: NTP + swap ────────────────────────────────────────── step "Ensuring NTP + swap" ssh $SSH_OPTS "$TARGET" ' if ! dpkg -l chrony >/dev/null 2>&1; then sudo rm -f /usr/sbin/policy-rc.d sudo apt-get update -qq && sudo apt-get install -y chrony 2>/dev/null fi sudo systemctl enable chrony 2>/dev/null sudo systemctl start chrony 2>/dev/null sudo timedatectl set-ntp true 2>/dev/null if [ ! -f /swapfile ]; then TOTAL_KB=$(grep MemTotal /proc/meminfo | awk "{print \$2}") SZ=$((TOTAL_KB / 1024 / 1024)) [ "$SZ" -gt 8 ] && SZ=8; [ "$SZ" -lt 2 ] && SZ=2 sudo fallocate -l ${SZ}G /swapfile && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile grep -q "/swapfile" /etc/fstab || echo "/swapfile none swap sw 0 0" | sudo tee -a /etc/fstab echo " Created ${SZ}G swap" fi sudo swapon /swapfile 2>/dev/null || true echo " NTP + swap OK" ' 2>&1 | tail -5 # ── Step 19: Restart services ──────────────────────────────────── step "Restarting services" ssh $SSH_OPTS "$TARGET" "sudo systemctl start archipelago && sudo systemctl restart nginx && echo ' Services restarted'" 2>&1 # ── Step 20: Setup HTTPS ───────────────────────────────────────── step "Setting up HTTPS" ssh $SSH_OPTS "$TARGET" "sudo bash $TARGET_DIR/scripts/setup-https-dev.sh" 2>&1 | tail -5 | sed 's/^/ /' || true # ── Step 21: Read secrets ──────────────────────────────────────── step "Reading secrets from server" BITCOIN_RPC_PASS=$(ssh $SSH_OPTS "$TARGET" ' SECRETS_DIR="/var/lib/archipelago/secrets" sudo mkdir -p "$SECRETS_DIR" && sudo chmod 700 "$SECRETS_DIR" if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then openssl rand -base64 24 | sudo tee "$SECRETS_DIR/bitcoin-rpc-password" > /dev/null sudo chmod 600 "$SECRETS_DIR/bitcoin-rpc-password" fi sudo cat "$SECRETS_DIR/bitcoin-rpc-password" ' 2>/dev/null) BITCOIN_RPC_USER="archipelago" # Read DB passwords from secrets (safe parsing — no eval) ssh $SSH_OPTS "$TARGET" ' SECRETS_DIR="/var/lib/archipelago/secrets" for svc in mempool btcpay mysql-root; do if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then openssl rand -base64 24 | sudo tee "$SECRETS_DIR/${svc}-db-password" > /dev/null sudo chmod 600 "$SECRETS_DIR/${svc}-db-password" fi done # Fedimint gateway if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then FEDI_PASS=$(openssl rand -base64 16) echo "$FEDI_PASS" | sudo tee "$SECRETS_DIR/fedimint-gateway-password" > /dev/null sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-password" if command -v htpasswd >/dev/null 2>&1; then htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ":\n" | sudo tee "$SECRETS_DIR/fedimint-gateway-hash" > /dev/null sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-hash" fi fi ' 2>/dev/null # Read each password individually (avoids eval on SSH output) MEMPOOL_DB_PASS=$(ssh $SSH_OPTS "$TARGET" 'sudo cat /var/lib/archipelago/secrets/mempool-db-password 2>/dev/null' 2>/dev/null) BTCPAY_DB_PASS=$(ssh $SSH_OPTS "$TARGET" 'sudo cat /var/lib/archipelago/secrets/btcpay-db-password 2>/dev/null' 2>/dev/null) MYSQL_ROOT_PASS=$(ssh $SSH_OPTS "$TARGET" 'sudo cat /var/lib/archipelago/secrets/mysql-root-db-password 2>/dev/null' 2>/dev/null) FEDI_HASH=$(ssh $SSH_OPTS "$TARGET" 'sudo cat /var/lib/archipelago/secrets/fedimint-gateway-hash 2>/dev/null' 2>/dev/null) [ -z "${FEDI_HASH:-}" ] && FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' if [ -z "$BITCOIN_RPC_PASS" ]; then echo " WARNING: Could not read Bitcoin RPC password — skipping container setup" else echo " Secrets loaded." # ── Step 22: Create containers ─────────────────────────────── step "Creating containers (this may take a while on first run)" # All container creation in a single SSH session to reduce connection overhead. # Uses the same container logic as deploy-to-target.sh --live. ssh $SSH_OPTS "$TARGET" " DOCKER=podman command -v podman >/dev/null 2>&1 || DOCKER=docker TARGET_IP='$TARGET_IP' # Create archy-net bridge \$DOCKER network create archy-net 2>/dev/null || true NET_OPT='--network archy-net' echo ' === Bitcoin Knots ===' # Clean old bitcoin.conf that conflicts with container CLI args (double rpcbind) if [ -f /var/lib/archipelago/bitcoin/bitcoin.conf ]; then if grep -q 'rpcbind' /var/lib/archipelago/bitcoin/bitcoin.conf 2>/dev/null; then echo ' Cleaning old bitcoin.conf (conflicting rpcbind)...' printf 'printtoconsole=1\n' | sudo tee /var/lib/archipelago/bitcoin/bitcoin.conf > /dev/null sudo chown 100101:100101 /var/lib/archipelago/bitcoin/bitcoin.conf 2>/dev/null fi fi if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|archy-bitcoin-knots'; then echo ' Creating Bitcoin Knots...' sudo mkdir -p /var/lib/archipelago/bitcoin DISK_GB=\$(df --output=size -BG / 2>/dev/null | tail -1 | tr -dc '0-9') if [ \"\${DISK_GB:-0}\" -lt 1000 ]; then BTC_EXTRA_ARGS='-prune=550' BTC_DBCACHE=512 echo ' Small disk — pruning enabled' else BTC_EXTRA_ARGS='-txindex=1' BTC_DBCACHE=4096 fi \$DOCKER run -d --name bitcoin-knots --restart unless-stopped \$NET_OPT \ --health-cmd 'bitcoin-cli getnetworkinfo' --health-interval=60s --health-timeout=10s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8332:8332 -p 8333:8333 \ -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \ $BITCOIN_KNOTS_IMAGE \ -server=1 \$BTC_EXTRA_ARGS \ -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \ -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS \ -dbcache=\$BTC_DBCACHE else \$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true echo ' Bitcoin Knots already running' fi echo ' === Mempool Stack ===' if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'mysql-mempool|archy-mempool-db'; then echo ' Creating mysql-mempool...' sudo mkdir -p /var/lib/archipelago/mysql-mempool \$DOCKER run -d --name archy-mempool-db --restart unless-stopped \$NET_OPT \ --health-cmd 'mariadbd-safe --help > /dev/null 2>&1 || mariadb -uroot -e SELECT\ 1' --health-interval=30s --health-timeout=5s --health-retries=3 \ -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \ -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool \ -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \ $MARIADB_IMAGE sleep 3 fi MYSQL_CNT=\$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1) MYSQL_CNT=\${MYSQL_CNT:-archy-mempool-db} \$DOCKER start \$MYSQL_CNT 2>/dev/null || true \$DOCKER network connect archy-net \$MYSQL_CNT 2>/dev/null || true # Sync MariaDB user password with secrets (data dir may have stale password) sleep 3 \$DOCKER exec \$MYSQL_CNT mariadb -uroot -p"$MYSQL_ROOT_PASS" -e "ALTER USER 'mempool'@'%' IDENTIFIED BY '$MEMPOOL_DB_PASS';" 2>/dev/null \ && echo " MariaDB mempool password synced" \ || echo " MariaDB password sync skipped - may need data reinit" if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then \$DOCKER start electrumx 2>/dev/null || true else echo ' Creating electrumx...' sudo mkdir -p /var/lib/archipelago/electrumx \$DOCKER run -d --name electrumx --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8000/' --health-interval=30s --health-timeout=5s --health-retries=3 \ -p 50001:50001 -v /var/lib/archipelago/electrumx:/data \ -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \ -e COIN=Bitcoin -e DB_DIRECTORY=/data \ -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \ $ELECTRUMX_IMAGE fi fi if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then echo ' Creating mempool-api...' sudo mkdir -p /var/lib/archipelago/mempool \$DOCKER run -d --name mempool-api --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8999/api/v1/backend-info' --health-interval=30s --health-timeout=5s --health-retries=3 \ -p 8999:8999 -v /var/lib/archipelago/mempool:/data \ -e MEMPOOL_BACKEND=electrum -e ELECTRUM_HOST=electrumx -e ELECTRUM_PORT=50001 \ -e ELECTRUM_TLS_ENABLED=false -e CORE_RPC_HOST=\$TARGET_IP -e CORE_RPC_PORT=8332 \ -e CORE_RPC_USERNAME=archipelago -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \ -e DATABASE_ENABLED=true -e DATABASE_HOST=\$MYSQL_CNT -e DATABASE_DATABASE=mempool \ -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \ $MEMPOOL_BACKEND_IMAGE fi if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-mempool-web; then echo ' Creating mempool frontend...' \$DOCKER run -d --name archy-mempool-web --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8080/' --health-interval=30s --health-timeout=5s --health-retries=3 \ -p 4080:8080 -e FRONTEND_HTTP_PORT=8080 -e BACKEND_MAINNET_HTTP_HOST=mempool-api \ $MEMPOOL_WEB_IMAGE fi echo ' === BTCPay Stack ===' # Recreate btcpay-db if postgres version mismatch (15→16 incompatible) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then echo ' Recreating archy-btcpay-db (was stopped/broken)...' \$DOCKER rm -f archy-btcpay-db 2>/dev/null \$DOCKER rm -f postgres-btcpay 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then echo ' Creating archy-btcpay-db...' sudo mkdir -p /var/lib/archipelago/postgres-btcpay \$DOCKER run -d --name archy-btcpay-db --restart unless-stopped \$NET_OPT \ --health-cmd 'pg_isready -U postgres' --health-interval=30s --health-timeout=5s --health-retries=3 \ -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \ -e POSTGRES_DB=btcpay -e POSTGRES_USER=btcpay -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \ $BTCPAY_POSTGRES_IMAGE sleep 3 fi \$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \ \$DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then \$DOCKER start archy-nbxplorer 2>/dev/null || true else echo ' Creating archy-nbxplorer...' sudo mkdir -p /var/lib/archipelago/nbxplorer \$DOCKER run -d --name archy-nbxplorer --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:32838/' --health-interval=30s --health-timeout=5s --health-retries=3 \ -p 32838:32838 -v /var/lib/archipelago/nbxplorer:/data \ -e NBXPLORER_DATADIR=/data -e NBXPLORER_NETWORK=mainnet -e NBXPLORER_CHAINS=btc \ -e NBXPLORER_BIND=0.0.0.0:32838 -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \ -e NBXPLORER_BTCRPCUSER=archipelago -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \ -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \ $NBXPLORER_IMAGE sleep 5 fi fi if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then echo ' Creating btcpay-server...' sudo mkdir -p /var/lib/archipelago/btcpay \$DOCKER run -d --name btcpay-server --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:49392/' --health-interval=30s --health-timeout=10s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 23000:49392 -v /var/lib/archipelago/btcpay:/datadir \ -e ASPNETCORE_URLS=http://0.0.0.0:49392 -e BTCPAY_PROTOCOL=http \ -e BTCPAY_HOST=\$TARGET_IP:23000 -e BTCPAY_CHAINS=btc \ -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \ -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \ -e BTCPAY_BTCRPCUSER=archipelago -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \ -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \ $BTCPAY_IMAGE fi echo ' === LND ===' # Always sync LND config with current RPC credentials before starting sudo mkdir -p /var/lib/archipelago/lnd RPC_PASS=\$(sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null) if [ -f /var/lib/archipelago/lnd/lnd.conf ]; then CURRENT_LND_PASS=\$(sudo grep "bitcoind.rpcpass=" /var/lib/archipelago/lnd/lnd.conf 2>/dev/null | cut -d= -f2) if [ "\$CURRENT_LND_PASS" != "\$RPC_PASS" ] && [ -n "\$RPC_PASS" ]; then echo " Syncing LND rpcpass with current secrets..." sudo sed -i "s|bitcoind.rpcpass=.*|bitcoind.rpcpass=\$RPC_PASS|" /var/lib/archipelago/lnd/lnd.conf sudo chown 100000:100000 /var/lib/archipelago/lnd/lnd.conf 2>/dev/null fi fi if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx lnd; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx lnd; then \$DOCKER start lnd 2>/dev/null || true else echo ' Creating LND...' cat > /tmp/lnd.conf </dev/null rm -f /tmp/lnd.conf \$DOCKER run -d --name lnd --restart unless-stopped --network archy-net \ --health-cmd 'curl -sf --insecure https://localhost:8080/v1/getinfo' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 9735:9735 -p 10009:10009 -p 8080:8080 \ -v /var/lib/archipelago/lnd:/root/.lnd \ $LND_IMAGE fi fi echo ' === Fedimint ===' # Recreate fedimint if it exists but is broken (wrong env vars) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then echo ' Recreating fedimint (was stopped/broken)...' \$DOCKER rm -f fedimint 2>/dev/null else echo ' Fedimint already running' fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then echo ' Creating Fedimint...' sudo mkdir -p /var/lib/archipelago/fedimint \$DOCKER run -d --name fedimint --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8175/' --health-interval=60s --health-timeout=10s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8173:8173 -p 8174:8174 -p 8175:8175 \ -v /var/lib/archipelago/fedimint:/data \ -e FM_DATA_DIR=/data -e FM_BITCOIND_USERNAME=archipelago -e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \ -e FM_BITCOIN_NETWORK=bitcoin -e FM_BIND_P2P=0.0.0.0:8173 \ -e FM_BIND_API=0.0.0.0:8174 -e FM_BIND_UI=0.0.0.0:8175 \ -e FM_P2P_URL=fedimint://\$TARGET_IP:8173 -e FM_API_URL=ws://\$TARGET_IP:8174 \ -e FM_BITCOIND_URL=http://\$TARGET_IP:8332 \ -e FM_REL_NOTES_ACK=0_4_xyz \ $FEDIMINT_IMAGE fi # Recreate fedimint-gateway if broken if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then echo ' Recreating fedimint-gateway (was stopped/broken)...' \$DOCKER rm -f fedimint-gateway 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then echo ' Creating fedimint-gateway...' sudo mkdir -p /var/lib/archipelago/fedimint-gateway FEDI_PASS=\$(sudo cat /var/lib/archipelago/secrets/fedimint-gateway-password 2>/dev/null || echo 'archipelago') LND_CERT=/var/lib/archipelago/lnd/tls.cert LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon if \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8176/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data \ -v /var/lib/archipelago/lnd/tls.cert:/lnd/tls.cert:ro \ -v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \ $FEDIMINT_GATEWAY_IMAGE \ gatewayd --data-dir /data --listen 0.0.0.0:8176 \ --password \"\$FEDI_PASS\" \ --network bitcoin --bitcoind-url http://\$TARGET_IP:8332 \ --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \ lnd --lnd-rpc-host \$TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon else \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \$NET_OPT \ --health-cmd 'curl -sf http://localhost:8176/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8176:8176 -p 9737:9737 -v /var/lib/archipelago/fedimint-gateway:/data \ $FEDIMINT_GATEWAY_IMAGE \ gatewayd --data-dir /data --listen 0.0.0.0:8176 \ --password \"\$FEDI_PASS\" \ --network bitcoin --bitcoind-url http://\$TARGET_IP:8332 \ --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \ ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway fi fi echo ' === Simple apps ===' # Home Assistant if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx homeassistant; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx homeassistant; then \$DOCKER start homeassistant 2>/dev/null || true else sudo mkdir -p /var/lib/archipelago/home-assistant \$DOCKER run -d --name homeassistant --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:8123/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8123:8123 -v /var/lib/archipelago/home-assistant:/config -e TZ=UTC \ $HOMEASSISTANT_IMAGE fi fi # Grafana if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx grafana; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx grafana; then \$DOCKER start grafana 2>/dev/null || true else sudo mkdir -p /var/lib/archipelago/grafana sudo chown -R 1000:1000 /var/lib/archipelago/grafana # If old rootful grafana data exists (wrong perms), move aside for fresh start if [ -f /var/lib/archipelago/grafana/grafana.db ]; then sudo mv /var/lib/archipelago/grafana /var/lib/archipelago/grafana-old 2>/dev/null sudo mkdir -p /var/lib/archipelago/grafana sudo chown -R 1000:1000 /var/lib/archipelago/grafana fi \$DOCKER run -d --name grafana --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:3000/api/health' --health-interval=30s --health-timeout=5s --health-retries=3 \ --user 0:0 \ -p 3000:3000 -v /var/lib/archipelago/grafana:/var/lib/grafana \ -e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \ $GRAFANA_IMAGE fi fi # Jellyfin if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx jellyfin; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx jellyfin; then \$DOCKER start jellyfin 2>/dev/null || true else sudo mkdir -p /var/lib/archipelago/jellyfin/config /var/lib/archipelago/jellyfin/cache \$DOCKER run -d --name jellyfin --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:8096/health' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --security-opt no-new-privileges:true \ -p 8096:8096 \ -v /var/lib/archipelago/jellyfin/config:/config \ -v /var/lib/archipelago/jellyfin/cache:/cache \ $JELLYFIN_IMAGE fi fi # Vaultwarden — recreate if broken (permissions/DB) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then \$DOCKER rm -f vaultwarden 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then sudo mkdir -p /var/lib/archipelago/vaultwarden sudo chown -R 100000:100000 /var/lib/archipelago/vaultwarden 2>/dev/null \$DOCKER run -d --name vaultwarden --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \ --security-opt no-new-privileges:true \ -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \ $VAULTWARDEN_IMAGE fi # SearXNG — recreate if broken (permission denied on settings.yml) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then \$DOCKER rm -f searxng 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then sudo mkdir -p /var/lib/archipelago/searxng \$DOCKER run -d --name searxng --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:8080/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -v /var/lib/archipelago/searxng:/etc/searxng \ -p 8888:8080 $SEARXNG_IMAGE fi # FileBrowser — recreate if broken (permission denied on :80) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then \$DOCKER rm -f filebrowser 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then sudo mkdir -p /var/lib/archipelago/filebrowser \$DOCKER run -d --name filebrowser --restart=unless-stopped \ --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --user 0:0 \ -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv \ $FILEBROWSER_IMAGE fi echo ' === Additional apps ===' # Nextcloud — recreate if wrong image version (28→30 not supported, need 29) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then echo ' Recreating nextcloud (was stopped/broken)...' \$DOCKER rm -f nextcloud 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then sudo mkdir -p /var/lib/archipelago/nextcloud \$DOCKER run -d --name nextcloud --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 8085:80 -v /var/lib/archipelago/nextcloud:/var/www/html \ $NEXTCLOUD_IMAGE fi # PhotoPrism — recreate if broken (permissions) if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then \$DOCKER rm -f photoprism 2>/dev/null fi fi if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then sudo mkdir -p /var/lib/archipelago/photoprism sudo chown -R 100000:100000 /var/lib/archipelago/photoprism 2>/dev/null \$DOCKER run -d --name photoprism --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:2342/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \ --security-opt no-new-privileges:true \ -p 2342:2342 -v /var/lib/archipelago/photoprism:/photoprism/storage \ -e PHOTOPRISM_ADMIN_PASSWORD=archipelago -e PHOTOPRISM_DEFAULT_LOCALE=en \ $PHOTOPRISM_IMAGE fi # OnlyOffice if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx onlyoffice; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx onlyoffice; then \$DOCKER start onlyoffice 2>/dev/null || true else \$DOCKER run -d --name onlyoffice --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 9980:80 $ONLYOFFICE_IMAGE fi fi # Nginx Proxy Manager if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx nginx-proxy-manager; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nginx-proxy-manager; then \$DOCKER start nginx-proxy-manager 2>/dev/null || true else sudo mkdir -p /var/lib/archipelago/nginx-proxy-manager/data /var/lib/archipelago/nginx-proxy-manager/letsencrypt \$DOCKER run -d --name nginx-proxy-manager --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:81/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \ --security-opt no-new-privileges:true \ -p 81:81 -p 8084:80 -p 8443:443 \ -v /var/lib/archipelago/nginx-proxy-manager/data:/data \ -v /var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt \ $NPM_IMAGE fi fi # Portainer if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx portainer; then if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx portainer; then \$DOCKER start portainer 2>/dev/null || true else sudo mkdir -p /var/lib/archipelago/portainer \$DOCKER run -d --name portainer --restart unless-stopped \ --health-cmd 'curl -sf http://localhost:9000/' --health-interval=30s --health-timeout=5s --health-retries=3 \ --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \ --security-opt no-new-privileges:true \ -p 9000:9000 -v /var/lib/archipelago/portainer:/data \ -v /run/user/1000/podman/podman.sock:/var/run/docker.sock \ $PORTAINER_IMAGE fi fi echo ' === Custom UI containers ===' # Build custom UI containers if source exists for ui in bitcoin-ui lnd-ui electrs-ui; do CONTAINER_NAME=\"archy-\$ui\" if \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q \"\$CONTAINER_NAME\"; then continue fi case \$ui in bitcoin-ui) PORT_ARG=''; NET_ARG='--network host' ;; lnd-ui) PORT_ARG='-p 18083:80'; NET_ARG='' ;; electrs-ui) PORT_ARG=''; NET_ARG='--network host' ;; esac if [ -d \"$TARGET_DIR/docker/\$ui\" ]; then echo \" Building \$ui...\" if \$DOCKER build --no-cache -t \"\$ui:local\" \"$TARGET_DIR/docker/\$ui\" 2>/dev/null; then \$DOCKER stop \"\$CONTAINER_NAME\" 2>/dev/null; \$DOCKER rm -f \"\$CONTAINER_NAME\" 2>/dev/null \$DOCKER run -d --name \"\$CONTAINER_NAME\" \$PORT_ARG --restart unless-stopped --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \$NET_ARG \"\$ui:local\" echo \" \$ui created\" fi elif \$DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q \"\$ui\"; then IMG=\$(\$DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep \"\$ui\" | head -1) \$DOCKER run -d --name \"\$CONTAINER_NAME\" \$PORT_ARG --restart unless-stopped --health-cmd 'curl -sf http://localhost:80/' --health-interval=30s --health-timeout=5s --health-retries=3 \$NET_ARG \"\$IMG\" fi done # Patch bitcoin-ui with this node's RPC credentials if \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-bitcoin-ui; then RPC_PASS=\$(sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null) if [ -n \"\$RPC_PASS\" ]; then AUTH_B64=\$(echo -n \"archipelago:\${RPC_PASS}\" | base64) \$DOCKER exec archy-bitcoin-ui cat /etc/nginx/conf.d/default.conf > /tmp/btc-ui-nginx.conf 2>/dev/null if grep -q '__BITCOIN_RPC_AUTH__' /tmp/btc-ui-nginx.conf; then sed -i \"s|__BITCOIN_RPC_AUTH__|\${AUTH_B64}|g\" /tmp/btc-ui-nginx.conf else sed -i \"s|proxy_set_header Authorization \\\"Basic .*\\\";|proxy_set_header Authorization \\\"Basic \${AUTH_B64}\\\";|g\" /tmp/btc-ui-nginx.conf fi \$DOCKER cp /tmp/btc-ui-nginx.conf archy-bitcoin-ui:/etc/nginx/conf.d/default.conf 2>/dev/null \$DOCKER exec archy-bitcoin-ui nginx -s reload 2>/dev/null rm -f /tmp/btc-ui-nginx.conf echo ' Bitcoin UI: RPC credentials patched' fi fi # Container summary echo '' TOTAL=\$(\$DOCKER ps --format '{{.Names}}' 2>/dev/null | wc -l) echo \" Total containers running: \$TOTAL\" " 2>&1 | sed 's/^/ /' # ── Step 23: Tor (robust setup) ────────────────────────────── step "Setting up Tor" ssh $SSH_OPTS "$TARGET" ' sudo mkdir -p /var/lib/archipelago/tor # Install Tor if missing if ! command -v tor >/dev/null 2>&1; then echo " Installing Tor..." sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null fi if ! command -v tor >/dev/null 2>&1; then echo " ERROR: Tor installation failed" exit 0 fi # Write services.json SERVICES_JSON=/var/lib/archipelago/tor/services.json if [ ! -f "$SERVICES_JSON" ]; then sudo python3 -c " import json services = [ {\"name\": \"archipelago\", \"local_port\": 80, \"enabled\": True}, {\"name\": \"bitcoin\", \"local_port\": 8333, \"enabled\": True}, {\"name\": \"electrumx\", \"local_port\": 50001, \"enabled\": True}, {\"name\": \"lnd\", \"local_port\": 9735, \"enabled\": True}, {\"name\": \"btcpay\", \"local_port\": 23000, \"enabled\": True}, {\"name\": \"mempool\", \"local_port\": 4080, \"enabled\": True}, {\"name\": \"fedimint\", \"local_port\": 8175, \"enabled\": True} ] with open(\"/var/lib/archipelago/tor/services.json\", \"w\") as f: json.dump({\"services\": services}, f, indent=2) " fi # Enable + start Tor service (try both unit names) sudo systemctl enable tor 2>/dev/null || true sudo systemctl enable tor@default 2>/dev/null || true # Restart Tor — try tor@default first (Debian pattern), fallback to tor if sudo systemctl restart tor@default 2>/dev/null; then echo " Tor running (tor@default)" elif sudo systemctl restart tor 2>/dev/null; then echo " Tor running (tor)" else echo " WARNING: Tor failed to start — check journalctl -u tor" fi # Verify Tor is actually running if systemctl is-active tor@default >/dev/null 2>&1 || systemctl is-active tor >/dev/null 2>&1; then echo " Tor verified active" else echo " WARNING: Tor not active after restart attempt" fi ' 2>&1 | sed 's/^/ /' fi # ── Step 24: UFW forward policy ────────────────────────────────── step "Fixing UFW forward policy" ssh $SSH_OPTS "$TARGET" ' if grep -q "DEFAULT_FORWARD_POLICY=\"DROP\"" /etc/default/ufw 2>/dev/null; then sudo sed -i "s/DEFAULT_FORWARD_POLICY=\"DROP\"/DEFAULT_FORWARD_POLICY=\"ACCEPT\"/" /etc/default/ufw sudo ufw reload 2>/dev/null echo " Fixed (was DROP, now ACCEPT)" else echo " Already ACCEPT" fi ' 2>&1 # ── Step 25: Fix IndeedHub NIP-07 ──────────────────────────────── step "Fixing IndeedHub for NIP-07" ssh $SSH_OPTS "$TARGET" ' if podman ps --format "{{.Names}}" 2>/dev/null | grep -q "^indeedhub$"; then CHANGED=false if podman exec indeedhub grep -q "X-Frame-Options" /etc/nginx/conf.d/default.conf 2>/dev/null; then podman exec indeedhub sed -i "/X-Frame-Options/d" /etc/nginx/conf.d/default.conf CHANGED=true echo " Removed X-Frame-Options" fi if ! podman exec indeedhub test -f /usr/share/nginx/html/nostr-provider.js 2>/dev/null; then podman cp /opt/archipelago/web-ui/nostr-provider.js indeedhub:/usr/share/nginx/html/nostr-provider.js 2>/dev/null echo " Copied nostr-provider.js" fi API_IP=$(podman inspect indeedhub-build_api_1 --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null) MINIO_IP=$(podman inspect indeedhub-minio --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null) RELAY_IP=$(podman inspect indeedhub-relay --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null) if [ -n "$API_IP" ] && [ -n "$MINIO_IP" ] && [ -n "$RELAY_IP" ]; then podman exec indeedhub cat /etc/nginx/conf.d/default.conf > /tmp/ih-nginx.conf 2>/dev/null sed -i "s|resolver 127.0.0.11 valid=30s ipv6=off;||g" /tmp/ih-nginx.conf sed -i "s|set \$api_upstream http://api:4000;|set \$api_upstream http://$API_IP:4000;|g" /tmp/ih-nginx.conf sed -i "s|set \$minio_upstream http://minio:9000;|set \$minio_upstream http://$MINIO_IP:9000;|g" /tmp/ih-nginx.conf sed -i "s|set \$relay_upstream http://relay:8080;|set \$relay_upstream http://$RELAY_IP:8080;|g" /tmp/ih-nginx.conf podman cp /tmp/ih-nginx.conf indeedhub:/etc/nginx/conf.d/default.conf 2>/dev/null rm -f /tmp/ih-nginx.conf CHANGED=true echo " Patched container IPs" fi [ "$CHANGED" = true ] && podman exec indeedhub nginx -s reload 2>/dev/null else echo " IndeedHub not running, skipping" fi ' 2>&1 # ── Step 26: Container doctor ──────────────────────────────────── step "Running container doctor" "$SCRIPT_DIR/container-doctor.sh" "$TARGET" 2>&1 | tail -10 | sed 's/^/ /' || true # ── Step 26b: Restart stopped containers + verify health ────── step "Verifying all containers running" ssh $SSH_OPTS "$TARGET" ' DOCKER=podman; command -v podman >/dev/null 2>&1 || DOCKER=docker # Fix permissions before restart attempts (rootless UID mapping) for dir in vaultwarden photoprism nextcloud filebrowser searxng; do [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null done # Restart any exited containers (unless user-stopped) USER_STOPPED="/var/lib/archipelago/user-stopped.json" for ctr in $($DOCKER ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null); do if [ -f "$USER_STOPPED" ] && grep -q "\"$ctr\"" "$USER_STOPPED" 2>/dev/null; then continue fi echo " Restarting exited container: $ctr" $DOCKER start "$ctr" 2>/dev/null || echo " WARNING: Failed to start $ctr" done # Summary RUNNING=$($DOCKER ps --format "{{.Names}}" 2>/dev/null | wc -l) EXITED=$($DOCKER ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null | wc -l) echo " Containers: $RUNNING running, $EXITED exited" # Verify Tor is still active if systemctl is-active tor@default >/dev/null 2>&1 || systemctl is-active tor >/dev/null 2>&1; then echo " Tor: active" else echo " Tor: NOT RUNNING — attempting restart..." sudo systemctl restart tor@default 2>/dev/null || sudo systemctl restart tor 2>/dev/null || echo " Tor restart failed" fi ' 2>&1 | sed 's/^/ /' # ── Step 27: Deploy manifest ───────────────────────────────────── step "Writing deploy manifest" DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) ssh $SSH_OPTS "$TARGET" "sudo tee /opt/archipelago/deploy-manifest.json > /dev/null" << MANIFEST_EOF { "commit": "$DEPLOY_COMMIT_FULL", "commit_short": "$DEPLOY_COMMIT", "branch": "$DEPLOY_BRANCH", "dirty": $DEPLOY_DIRTY, "deployed_at": "$DEPLOY_TS", "deployed_from": "$(hostname)", "target": "$TARGET" } MANIFEST_EOF echo " Manifest written." # ── Step 28: Health check ──────────────────────────────────────── step "Post-deploy health check" HEALTH_OK=false for i in $(seq 1 12); do HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP/health" 2>/dev/null || { echo "WARNING: Post-deploy health check failed for $TARGET_IP" >&2; echo "000"; }) if [ "$HEALTH" = "200" ]; then echo " Health: OK (200) after $((i * 5))s" HEALTH_OK=true break fi echo " Health: $HEALTH (waiting... ${i}/12)" sleep 5 done if [ "$HEALTH_OK" = false ]; then echo " WARNING: Server did not become healthy within 60s" echo " Check: ssh $TARGET 'sudo journalctl -u archipelago -n 50'" fi local ELAPSED=$(($(date +%s) - DEPLOY_START)) echo "" echo "$(ts) Deploy complete for $NODE_NAME ($TARGET_IP) in ${ELAPSED}s" echo " Commit: $DEPLOY_BRANCH @ $DEPLOY_COMMIT" echo " Web UI: http://$TARGET_IP" # Append to deploy history echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) | $DEPLOY_BRANCH@$DEPLOY_COMMIT | dirty=$DEPLOY_DIRTY | target=$TARGET | ${ELAPSED}s | tailscale" >> "$PROJECT_DIR/scripts/deploy-history.log" } # ── Main ───────────────────────────────────────────────────────────────── if [ "$1" = "--all" ]; then echo "Deploying to all ${#TAILSCALE_NODES[@]} Tailscale nodes..." FAILED=() for i in "${!TAILSCALE_NODES[@]}"; do deploy_node "${TAILSCALE_NODES[$i]}" "${TAILSCALE_NAMES[$i]}" || FAILED+=("${TAILSCALE_NAMES[$i]}") done echo "" echo "════════════════════════════════════════════════════════════════" if [ ${#FAILED[@]} -eq 0 ]; then echo "All ${#TAILSCALE_NODES[@]} nodes deployed successfully." else echo "FAILED: ${FAILED[*]}" echo "Succeeded: $((${#TAILSCALE_NODES[@]} - ${#FAILED[@]}))/${#TAILSCALE_NODES[@]}" exit 1 fi elif [ -n "$1" ]; then # Map friendly names to targets case "$1" in arch1|Arch1) deploy_node "${TAILSCALE_NODES[0]}" "Arch 1" ;; arch2|Arch2) deploy_node "${TAILSCALE_NODES[1]}" "Arch 2" ;; arch3|Arch3) deploy_node "${TAILSCALE_NODES[2]}" "Arch 3" ;; *) deploy_node "$1" "$1" ;; esac else echo "Usage: $0 " echo "" echo "Examples:" echo " $0 arch2 # Deploy to Arch 2" echo " $0 archipelago@100.82.97.63 # Deploy to specific host" echo " $0 --all # Deploy to all 3 Tailscale nodes" exit 1 fi