archy/scripts/first-boot-containers.sh

#!/bin/bash
#
# First-boot container creation for Archipelago autoinstaller
# Creates core containers so My Apps works out of the box after ISO install
# Runs after archipelago-load-images.service and archipelago-setup-tor.service
#
# Based on scripts/deploy-to-target.sh (--live) container logic - do not diverge.
# No set -e: each section continues even if one fails (idempotent, best-effort).
#
LOG="/var/log/archipelago-first-boot.log"
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker

# Must run as root for podman
[ "$(id -u)" -eq 0 ] || { echo "Must run as root" >&2; exit 1; }

TARGET_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
[ -z "$TARGET_IP" ] && TARGET_IP="127.0.0.1"

log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG"; }

# Wait for a container to be healthy (accepting connections)
wait_for_container() {
  local name="$1" check_cmd="$2" max_wait="${3:-30}"
  local waited=0
  while [ $waited -lt $max_wait ]; do
    if eval "$check_cmd" 2>/dev/null; then
      log "  $name is ready (${waited}s)"
      return 0
    fi
    sleep 2
    waited=$((waited + 2))
  done
  log "  WARNING: $name not ready after ${max_wait}s, continuing anyway"
  return 1
}

# Generate per-installation credentials if not already saved
SECRETS_DIR="/var/lib/archipelago/secrets"
mkdir -p "$SECRETS_DIR" && chmod 700 "$SECRETS_DIR"
if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
    openssl rand -base64 24 > "$SECRETS_DIR/bitcoin-rpc-password"
    chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
fi
BITCOIN_RPC_USER="archipelago"
BITCOIN_RPC_PASS=$(cat "$SECRETS_DIR/bitcoin-rpc-password")

# Generate per-installation database passwords if not already saved
for svc in mempool btcpay immich penpot mysql-root; do
    if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
        openssl rand -base64 24 > "$SECRETS_DIR/${svc}-db-password"
        chmod 600 "$SECRETS_DIR/${svc}-db-password"
    fi
done
MEMPOOL_DB_PASS=$(cat "$SECRETS_DIR/mempool-db-password")
BTCPAY_DB_PASS=$(cat "$SECRETS_DIR/btcpay-db-password")
IMMICH_DB_PASS=$(cat "$SECRETS_DIR/immich-db-password")
PENPOT_DB_PASS=$(cat "$SECRETS_DIR/penpot-db-password")
MYSQL_ROOT_PASS=$(cat "$SECRETS_DIR/mysql-root-db-password")

# Generate Fedimint gateway password and bcrypt hash
if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
    FEDI_PASS=$(openssl rand -base64 16)
    echo "$FEDI_PASS" > "$SECRETS_DIR/fedimint-gateway-password"
    chmod 600 "$SECRETS_DIR/fedimint-gateway-password"
    # Pre-compute bcrypt hash (requires htpasswd from apache2-utils)
    if command -v htpasswd >/dev/null 2>&1; then
        htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ':\n' > "$SECRETS_DIR/fedimint-gateway-hash"
        chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
    fi
fi
FEDI_PASS=$(cat "$SECRETS_DIR/fedimint-gateway-password")
if [ -f "$SECRETS_DIR/fedimint-gateway-hash" ]; then
    FEDI_HASH=$(cat "$SECRETS_DIR/fedimint-gateway-hash")
else
    # Fallback: generate hash now
    if command -v htpasswd >/dev/null 2>&1; then
        FEDI_HASH=$(htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ':\n')
        echo "$FEDI_HASH" > "$SECRETS_DIR/fedimint-gateway-hash"
        chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
    else
        log "WARNING: htpasswd not found, using default Fedimint gateway hash"
        FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
    fi
fi
log "Fedimint gateway password stored in $SECRETS_DIR/fedimint-gateway-password"

log "First-boot container creation starting (host=$TARGET_IP)"

# Create swap file if not present (50% of RAM, min 2GB, max 8GB)
if ! swapon --show | grep -q /swapfile; then
    TOTAL_MEM_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
    SWAP_MB=$((TOTAL_MEM_KB / 2 / 1024))
    [ "$SWAP_MB" -lt 2048 ] && SWAP_MB=2048
    [ "$SWAP_MB" -gt 8192 ] && SWAP_MB=8192
    log "Creating ${SWAP_MB}MB swap file..."
    if dd if=/dev/zero of=/swapfile bs=1M count="$SWAP_MB" status=progress 2>>"$LOG"; then
        chmod 600 /swapfile
        mkswap /swapfile >>"$LOG" 2>&1
        swapon /swapfile
        if ! grep -q '/swapfile' /etc/fstab; then
            echo '/swapfile none swap sw 0 0' >> /etc/fstab
        fi
        log "Swap created: ${SWAP_MB}MB"
    else
        log "WARNING: Failed to create swap file"
    fi
else
    log "Swap already configured"
fi

# Rootless podman prerequisites (run as root, configures for archipelago user)
log "Setting up rootless podman prerequisites..."
# Allow binding to ports >= 80 (rootless default is 1024)
if ! grep -q "unprivileged_port_start=80" /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null; then
    echo "net.ipv4.ip_unprivileged_port_start=80" > /etc/sysctl.d/99-rootless-podman.conf
    sysctl -p /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null
    log "  Rootless port binding enabled (>=80)"
fi
# Linger for container persistence after logout
if [ "$(loginctl show-user archipelago 2>/dev/null | grep Linger)" != "Linger=yes" ]; then
    loginctl enable-linger archipelago 2>/dev/null
    log "  Linger enabled for archipelago user"
fi
# Ensure subuid/subgid mappings
grep -q "^archipelago:" /etc/subuid 2>/dev/null || {
    echo "archipelago:100000:65536" >> /etc/subuid
    echo "archipelago:100000:65536" >> /etc/subgid
    log "  subuid/subgid configured"
}
# Ensure /etc/hosts is readable (rootless podman needs it)
chmod 644 /etc/hosts 2>/dev/null

# Ensure network exists (matches deploy)
$DOCKER network create archy-net 2>/dev/null || true

# Rootless podman UID mapping: fix data dir ownership so container processes
# can write. Rootless podman maps container UIDs via subuid (container UID N
# → host UID 100000+N). Must run BEFORE container creation.
log "Fixing rootless podman UID mapping..."
# Containers running as root (UID 0 → host UID 100000)
for dir in lnd electrumx btcpay nbxplorer immich jellyfin vaultwarden \
           home-assistant fedimint fedimint-gateway photoprism ollama filebrowser \
           nextcloud uptime-kuma onlyoffice nginx-proxy-manager portainer nostr-rs-relay; do
    [ -d "/var/lib/archipelago/$dir" ] && chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
done
# Bitcoin Knots: container UID 101 → host UID 100101
[ -d /var/lib/archipelago/bitcoin ] && chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
# Postgres: container UID 70 → host UID 100070
for dir in postgres-btcpay immich-db penpot-postgres; do
    [ -d "/var/lib/archipelago/$dir" ] && chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null
done
# MariaDB: container UID 999 → host UID 100999
for dir in mempool mysql-mempool; do
    [ -d "/var/lib/archipelago/$dir" ] && chown -R 100999:100999 "/var/lib/archipelago/$dir" 2>/dev/null
done
# Grafana: container UID 472 → host UID 100472
[ -d /var/lib/archipelago/grafana ] && chown -R 100472:100472 /var/lib/archipelago/grafana 2>/dev/null
log "UID mapping done"

# ── Memory limits per container ──────────────────────────────────────────
# Matches core/archipelago/src/api/rpc/package.rs get_memory_limit()
# Prevents a single runaway container from OOMing the whole system.
TOTAL_MEM_MB=$(($(awk '/MemTotal/{print $2}' /proc/meminfo) / 1024))
LOW_MEM=false
[ "$TOTAL_MEM_MB" -lt 12000 ] && LOW_MEM=true && log "Low-memory system (${TOTAL_MEM_MB}MB) — reducing limits"

mem_limit() {
  case "$1" in
    bitcoin-knots)       $LOW_MEM && echo "1g"   || echo "2g";;
    onlyoffice)          $LOW_MEM && echo "1g"   || echo "2g";;
    ollama)              $LOW_MEM && echo "1g"   || echo "4g";;
    lnd)                 echo "512m";;
    electrumx)           echo "1g";;
    nextcloud)           echo "1g";;
    immich_server)       echo "1g";;
    btcpay-server)       echo "1g";;
    homeassistant)       echo "512m";;
    fedimint)            echo "512m";;
    fedimint-gateway)    echo "512m";;
    photoprism)          $LOW_MEM && echo "512m" || echo "1g";;
    mempool-api)         echo "512m";;
    jellyfin)            echo "1g";;
    searxng)             echo "512m";;
    archy-btcpay-db)     echo "512m";;
    archy-nbxplorer)     echo "512m";;
    archy-mempool-db)    echo "512m";;
    archy-mempool-web)   echo "256m";;
    grafana)             echo "256m";;
    vaultwarden)         echo "256m";;
    uptime-kuma)         echo "256m";;
    filebrowser)         echo "256m";;
    portainer)           echo "256m";;
    nginx-proxy-manager) echo "256m";;
    immich_postgres)     echo "256m";;
    immich_redis)        echo "128m";;
    tailscale)           echo "256m";;
    indeedhub|archy-bitcoin-ui|archy-lnd-ui|archy-electrs-ui) echo "128m";;
    *)                   echo "512m";;
  esac
}

# ── Tier 1: Databases & Core Infrastructure ──────────────────────────────
log "=== Tier 1: Databases & Core Infrastructure ==="

# 1. Bitcoin Knots (matches deploy exactly)
# Auto-detect: if disk < 1TB, use pruning to prevent disk-full crashes
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|archy-bitcoin-knots'; then
  log "Creating Bitcoin Knots..."
  mkdir -p /var/lib/archipelago/bitcoin
  DISK_GB=$(df --output=size -BG / 2>/dev/null | tail -1 | tr -dc '0-9')
  if [ "${DISK_GB:-0}" -lt 1000 ]; then
    BTC_EXTRA_ARGS="-prune=550"
    BTC_DBCACHE=512
    log "  Small disk (${DISK_GB}GB) — enabling pruning"
  else
    BTC_EXTRA_ARGS="-txindex=1"
    BTC_DBCACHE=4096
    log "  Large disk (${DISK_GB}GB) — enabling txindex"
  fi
  if $DOCKER run -d --name bitcoin-knots --restart unless-stopped --memory=$(mem_limit bitcoin-knots) --network archy-net \
    --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 8332:8332 -p 8333:8333 \
    -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
    docker.io/bitcoinknots/bitcoin:latest \
    -server=1 $BTC_EXTRA_ARGS \
    -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
    -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS \
    -proxy=127.0.0.1:9050 -listen=1 -bind=0.0.0.0:8333 \
    -dbcache=$BTC_DBCACHE 2>>"$LOG"; then
    log "Bitcoin Knots started"
  else
    log "Bitcoin Knots failed (may already exist)"
  fi
else
  $DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
  log "Bitcoin Knots already running"
fi
# Wait for Bitcoin Knots RPC to be responsive (LND, NBXplorer, mempool depend on it)
wait_for_container "Bitcoin Knots RPC" "$DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS getblockchaininfo" 60

# Ensure wallet exists (Bitcoin Knots no longer auto-creates a default wallet)
if ! $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS listwallets 2>/dev/null | grep -q "archipelago"; then
  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS loadwallet "archipelago" 2>/dev/null || \
  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS createwallet "archipelago" 2>/dev/null
  log "Bitcoin Knots wallet 'archipelago' created/loaded"
fi

# 2. Mempool stack (matches deploy)
if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-db|mysql-mempool'; then
  log "Creating mysql-mempool..."
  mkdir -p /var/lib/archipelago/mysql-mempool
  $DOCKER run -d --name archy-mempool-db --restart unless-stopped --memory=$(mem_limit archy-mempool-db) --network archy-net \
    -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
    -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
    -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
    docker.io/mariadb:10.11 2>>"$LOG" || true
  wait_for_container "Mempool MariaDB" "$DOCKER exec archy-mempool-db mariadb -uroot -p$MYSQL_ROOT_PASS -e 'SELECT 1'" 30
fi
MYSQL_CNT=$($DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
MYSQL_CNT=${MYSQL_CNT:-archy-mempool-db}
$DOCKER network connect archy-net "$MYSQL_CNT" 2>/dev/null || true

if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
  if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
    $DOCKER start electrumx 2>/dev/null || true
  else
    log "Creating electrumx..."
    mkdir -p /var/lib/archipelago/electrumx
    $DOCKER run -d --name electrumx --restart unless-stopped --memory=$(mem_limit electrumx) --network archy-net \
      -p 50001:50001 -v /var/lib/archipelago/electrumx:/data \
      -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
      -e COIN=Bitcoin -e DB_DIRECTORY=/data \
      -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
      docker.io/lukechilds/electrumx:v1.18.0 2>>"$LOG" || true
  fi
fi

if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
  log "Creating mempool-api..."
  mkdir -p /var/lib/archipelago/mempool
  $DOCKER run -d --name mempool-api --restart unless-stopped --memory=$(mem_limit mempool-api) --network archy-net \
    -p 8999:8999 -v /var/lib/archipelago/mempool:/data \
    -e MEMPOOL_BACKEND=electrum -e ELECTRUM_HOST=electrumx -e ELECTRUM_PORT=50001 \
    -e ELECTRUM_TLS_ENABLED=false -e CORE_RPC_HOST="$TARGET_IP" -e CORE_RPC_PORT=8332 \
    -e CORE_RPC_USERNAME=$BITCOIN_RPC_USER -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
    -e DATABASE_ENABLED=true -e DATABASE_HOST="$MYSQL_CNT" -e DATABASE_DATABASE=mempool \
    -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
    docker.io/mempool/backend:v2.5.0 2>>"$LOG" || true
fi

if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-web|mempool-web'; then
  log "Creating mempool frontend..."
  $DOCKER run -d --name archy-mempool-web --restart unless-stopped --memory=$(mem_limit archy-mempool-web) --network archy-net \
    -p 4080:8080 -e FRONTEND_HTTP_PORT=8080 -e BACKEND_MAINNET_HTTP_HOST=mempool-api \
    docker.io/mempool/frontend:v2.5.0 2>>"$LOG" || true
fi

# 2b. ElectrumX UI (status dashboard on port 50002, host network for backend access)
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrs-ui; then
  if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'electrs-ui'; then
    log "Starting ElectrumX UI from pre-built image..."
    $DOCKER run -d --name archy-electrs-ui --network host --restart unless-stopped \
      localhost/electrs-ui:latest 2>>"$LOG" || \
    $DOCKER run -d --name archy-electrs-ui --network host --restart unless-stopped \
      electrs-ui:latest 2>>"$LOG" || true
  elif [ -d /opt/archipelago/docker/electrs-ui ]; then
    log "Building and starting ElectrumX UI from source..."
    $DOCKER build -t electrs-ui:latest /opt/archipelago/docker/electrs-ui 2>>"$LOG" && \
    $DOCKER run -d --name archy-electrs-ui --network host --restart unless-stopped \
      electrs-ui:latest 2>>"$LOG" || true
  else
    log "ElectrumX UI: no image or source found, skipping"
  fi
fi

# 3. BTCPay stack (matches deploy)
if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
  log "Creating PostgreSQL for BTCPay..."
  mkdir -p /var/lib/archipelago/postgres-btcpay
  $DOCKER run -d --name archy-btcpay-db --restart unless-stopped --memory=$(mem_limit archy-btcpay-db) --network archy-net \
    -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
    -e POSTGRES_DB=btcpay -e POSTGRES_USER=btcpay -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
    docker.io/postgres:15-alpine 2>>"$LOG" || true
  wait_for_container "BTCPay PostgreSQL" "$DOCKER exec archy-btcpay-db pg_isready -U postgres" 30
fi
# Create nbxplorer DB only if postgres is running
if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
  $DOCKER exec archy-btcpay-db psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='nbxplorer'" 2>/dev/null | grep -q 1 || \
    $DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c "CREATE DATABASE nbxplorer;" 2>/dev/null || true
fi

if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
  if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
    $DOCKER start archy-nbxplorer 2>/dev/null || true
  else
    log "Creating NBXplorer..."
    mkdir -p /var/lib/archipelago/nbxplorer
    $DOCKER run -d --name archy-nbxplorer --restart unless-stopped --memory=$(mem_limit archy-nbxplorer) --network archy-net \
      -p 32838:32838 -v /var/lib/archipelago/nbxplorer:/data \
      -e NBXPLORER_DATADIR=/data -e NBXPLORER_NETWORK=mainnet -e NBXPLORER_CHAINS=btc \
      -e NBXPLORER_BIND=0.0.0.0:32838 -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
      -e NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
      -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
      docker.io/nicolasdorier/nbxplorer:2.6.0 2>>"$LOG" && sleep 5 || true
  fi
fi

if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
  log "Creating BTCPay Server..."
  mkdir -p /var/lib/archipelago/btcpay
  $DOCKER run -d --name btcpay-server --restart unless-stopped --memory=$(mem_limit btcpay-server) --network archy-net \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 23000:49392 -v /var/lib/archipelago/btcpay:/datadir \
    -e ASPNETCORE_URLS=http://0.0.0.0:49392 -e BTCPAY_PROTOCOL=http \
    -e BTCPAY_HOST="$TARGET_IP:23000" -e BTCPAY_CHAINS=btc \
    -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
    -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
    -e BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
    -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
    docker.io/btcpayserver/btcpayserver:1.13.5 2>>"$LOG" || true
fi

# ── Tier 2: Core Services ─────────────────────────────────────────────────
log "=== Tier 2: Core Services ==="
sleep 5  # Let databases stabilize

# 4. LND
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE '^lnd$'; then
  log "Creating LND..."
  mkdir -p /var/lib/archipelago/lnd
  # Create lnd.conf so LND auto-connects to Bitcoin Knots via archy-net
  if [ ! -f /var/lib/archipelago/lnd/lnd.conf ]; then
    cat > /var/lib/archipelago/lnd/lnd.conf <<LNDCONF
[Application Options]
listen=0.0.0.0:9735
rpclisten=0.0.0.0:10009
restlisten=0.0.0.0:8080
debuglevel=info
noseedbackup=true
tor.active=true
tor.socks=127.0.0.1:9050
tor.streamisolation=true

[Bitcoin]
bitcoin.mainnet=true
bitcoin.node=bitcoind

[Bitcoind]
bitcoind.rpchost=bitcoin-knots:8332
bitcoind.rpcuser=archipelago
bitcoind.rpcpass=$BITCOIN_RPC_PASS
bitcoind.rpcpolling=true
bitcoind.estimatemode=ECONOMICAL

[autopilot]
autopilot.active=false
LNDCONF
    log "LND config created (archy-net → bitcoin-knots:8332, rpcpolling)"
  fi
  $DOCKER run -d --name lnd --restart unless-stopped --memory=$(mem_limit lnd) --network archy-net \
    --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 9735:9735 -p 10009:10009 -p 8080:8080 \
    -v /var/lib/archipelago/lnd:/root/.lnd \
    docker.io/lightninglabs/lnd:v0.18.4-beta 2>>"$LOG" || true
fi

# 5. Fedimint
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint; then
  log "Creating Fedimint..."
  mkdir -p /var/lib/archipelago/fedimint
  $DOCKER run -d --name fedimint --restart unless-stopped --memory=$(mem_limit fedimint) --network archy-net \
    --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 8173:8173 -p 8174:8174 -p 8175:8175 \
    -v /var/lib/archipelago/fedimint:/data \
    -e FM_DATA_DIR=/data -e FM_BITCOIND_USERNAME=$BITCOIN_RPC_USER -e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \
    -e FM_BITCOIN_NETWORK=bitcoin -e FM_BIND_P2P=0.0.0.0:8173 \
    -e FM_BIND_API=0.0.0.0:8174 -e FM_BIND_UI=0.0.0.0:8175 \
    -e FM_P2P_URL=fedimint://"$TARGET_IP":8173 -e FM_API_URL=ws://"$TARGET_IP":8174 \
    -e FM_BITCOIND_URL=http://"$TARGET_IP":8332 \
    docker.io/fedimint/fedimintd:v0.10.0 2>>"$LOG" || true
fi

# 5b. Fedimint Gateway (companion to fedimint)
# Auto-detect LND: if running with credentials, use lnd mode; otherwise use ldk (built-in)
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then
  log "Creating Fedimint Gateway..."
  mkdir -p /var/lib/archipelago/fedimint-gateway
  LND_CERT=/var/lib/archipelago/lnd/tls.cert
  LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
  if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q '^lnd$' && [ -f "$LND_CERT" ] && [ -f "$LND_MACAROON" ]; then
    log "  LND detected — using lnd mode"
    $DOCKER run -d --name fedimint-gateway --restart unless-stopped --memory=$(mem_limit fedimint-gateway) --network archy-net \
      --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
      --security-opt no-new-privileges:true \
      -p 8176:8176 \
      -v /var/lib/archipelago/fedimint-gateway:/data \
      -v "$LND_CERT":/lnd/tls.cert:ro \
      -v "$LND_MACAROON":/lnd/admin.macaroon:ro \
      docker.io/fedimint/gatewayd:v0.10.0 \
      gatewayd --data-dir /data --listen 0.0.0.0:8176 \
      --bcrypt-password-hash "$FEDI_HASH" \
      --network bitcoin --bitcoind-url http://"$TARGET_IP":8332 \
      --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
      lnd --lnd-rpc-host "$TARGET_IP":10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon 2>>"$LOG" || true
  else
    log "  No LND found — using ldk (built-in Lightning)"
    $DOCKER run -d --name fedimint-gateway --restart unless-stopped --memory=$(mem_limit fedimint-gateway) --network archy-net \
      --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
      --security-opt no-new-privileges:true \
      -p 8176:8176 -p 9737:9737 \
      -v /var/lib/archipelago/fedimint-gateway:/data \
      docker.io/fedimint/gatewayd:v0.10.0 \
      gatewayd --data-dir /data --listen 0.0.0.0:8176 \
      --bcrypt-password-hash "$FEDI_HASH" \
      --network bitcoin --bitcoind-url http://"$TARGET_IP":8332 \
      --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
      ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway 2>>"$LOG" || true
  fi
fi

# ── Tier 3: Applications ──────────────────────────────────────────────────
log "=== Tier 3: Applications ==="
sleep 5  # Let core services stabilize

# 6. Home Assistant
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'homeassistant|home-assistant'; then
  log "Creating Home Assistant..."
  mkdir -p /var/lib/archipelago/home-assistant
  $DOCKER run -d --name homeassistant --restart unless-stopped --memory=$(mem_limit homeassistant) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 8123:8123 -v /var/lib/archipelago/home-assistant:/config \
    -e TZ=UTC \
    docker.io/homeassistant/home-assistant:2024.1 2>>"$LOG" || true
fi

# 7. Single-container apps (Grafana, Uptime Kuma, Jellyfin, PhotoPrism, Ollama, Vaultwarden)
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q grafana; then
  log "Creating Grafana..."
  mkdir -p /var/lib/archipelago/grafana
  chown 472:472 /var/lib/archipelago/grafana 2>/dev/null || true
  $DOCKER run -d --name grafana --restart unless-stopped --memory=$(mem_limit grafana) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
    --security-opt no-new-privileges:true \
    --read-only --tmpfs /tmp:rw,noexec,nosuid,size=256m --tmpfs /run:rw,noexec,nosuid,size=64m \
    -p 3000:3000 -v /var/lib/archipelago/grafana:/var/lib/grafana \
    -e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \
    docker.io/grafana/grafana:10.2.0 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q uptime-kuma; then
  log "Creating Uptime Kuma..."
  mkdir -p /var/lib/archipelago/uptime-kuma
  $DOCKER run -d --name uptime-kuma --restart unless-stopped --memory=$(mem_limit uptime-kuma) \
    --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID \
    --security-opt no-new-privileges:true \
    -p 3001:3001 -v /var/lib/archipelago/uptime-kuma:/app/data \
    -e TZ=UTC \
    docker.io/louislam/uptime-kuma:1 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q jellyfin; then
  log "Creating Jellyfin..."
  mkdir -p /var/lib/archipelago/jellyfin/config /var/lib/archipelago/jellyfin/cache
  $DOCKER run -d --name jellyfin --restart unless-stopped --memory=$(mem_limit jellyfin) \
    --cap-drop ALL --security-opt no-new-privileges:true \
    -p 8096:8096 \
    -v /var/lib/archipelago/jellyfin/config:/config \
    -v /var/lib/archipelago/jellyfin/cache:/cache \
    docker.io/jellyfin/jellyfin:10.8.13 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q photoprism; then
  log "Creating PhotoPrism..."
  mkdir -p /var/lib/archipelago/photoprism
  $DOCKER run -d --name photoprism --restart unless-stopped --memory=$(mem_limit photoprism) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
    --security-opt no-new-privileges:true \
    -p 2342:2342 -v /var/lib/archipelago/photoprism:/photoprism/storage \
    -e PHOTOPRISM_ADMIN_PASSWORD=archipelago -e PHOTOPRISM_DEFAULT_LOCALE=en \
    docker.io/photoprism/photoprism:latest 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q ollama; then
  log "Creating Ollama..."
  mkdir -p /var/lib/archipelago/ollama
  $DOCKER run -d --name ollama --restart unless-stopped --memory=$(mem_limit ollama) \
    --cap-drop ALL --security-opt no-new-privileges:true \
    --read-only --tmpfs /tmp:rw,noexec,nosuid,size=256m --tmpfs /run:rw,noexec,nosuid,size=64m \
    -p 11434:11434 -v /var/lib/archipelago/ollama:/root/.ollama \
    docker.io/ollama/ollama:latest 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q vaultwarden; then
  log "Creating Vaultwarden..."
  mkdir -p /var/lib/archipelago/vaultwarden
  $DOCKER run -d --name vaultwarden --restart unless-stopped --memory=$(mem_limit vaultwarden) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
    --security-opt no-new-privileges:true \
    -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
    docker.io/vaultwarden/server:1.30.0-alpine 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nextcloud; then
  log "Creating Nextcloud..."
  mkdir -p /var/lib/archipelago/nextcloud
  $DOCKER run -d --name nextcloud --restart unless-stopped --memory=$(mem_limit nextcloud) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 8085:80 -v /var/lib/archipelago/nextcloud:/var/www/html \
    docker.io/library/nextcloud:28 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q searxng; then
  log "Creating SearXNG..."
  $DOCKER run -d --name searxng --restart unless-stopped --memory=$(mem_limit searxng) \
    --cap-drop ALL --security-opt no-new-privileges:true \
    --read-only --tmpfs /tmp:rw,noexec,nosuid,size=256m --tmpfs /run:rw,noexec,nosuid,size=64m \
    -p 8888:8080 \
    docker.io/searxng/searxng:latest 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q onlyoffice; then
  log "Creating OnlyOffice..."
  $DOCKER run -d --name onlyoffice --restart unless-stopped --memory=$(mem_limit onlyoffice) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 9980:80 \
    docker.io/onlyoffice/documentserver:7.5.1 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q filebrowser; then
  log "Creating File Browser..."
  mkdir -p /var/lib/archipelago/filebrowser /var/lib/archipelago/filebrowser-db
  $DOCKER run -d --name filebrowser --restart unless-stopped --memory=$(mem_limit filebrowser) \
    -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv \
    docker.io/filebrowser/filebrowser:v2.27.0 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nginx-proxy-manager; then
  log "Creating Nginx Proxy Manager..."
  mkdir -p /var/lib/archipelago/nginx-proxy-manager/data /var/lib/archipelago/nginx-proxy-manager/letsencrypt
  $DOCKER run -d --name nginx-proxy-manager --restart unless-stopped --memory=$(mem_limit nginx-proxy-manager) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
    --security-opt no-new-privileges:true \
    -p 81:81 -p 8084:80 -p 8443:443 \
    -v /var/lib/archipelago/nginx-proxy-manager/data:/data \
    -v /var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt \
    docker.io/jc21/nginx-proxy-manager:latest 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q portainer; then
  log "Creating Portainer..."
  mkdir -p /var/lib/archipelago/portainer
  $DOCKER run -d --name portainer --restart unless-stopped --memory=$(mem_limit portainer) \
    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
    --security-opt no-new-privileges:true \
    -p 9000:9000 \
    -v /var/lib/archipelago/portainer:/data \
    -v /var/run/podman/podman.sock:/var/run/docker.sock \
    docker.io/portainer/portainer-ce:2.19.4 2>>"$LOG" || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q tailscale; then
  log "Creating Tailscale..."
  mkdir -p /var/lib/archipelago/tailscale
  # Tailscale needs NET_ADMIN + NET_RAW + TUN device (no --privileged)
  $DOCKER run -d --name tailscale --restart unless-stopped --memory=$(mem_limit tailscale) \
    --network host \
    --cap-drop=ALL \
    --cap-add=NET_ADMIN \
    --cap-add=NET_RAW \
    --device=/dev/net/tun:/dev/net/tun \
    --read-only \
    --tmpfs /tmp \
    -v /var/lib/archipelago/tailscale:/var/lib/tailscale \
    -e TS_STATE_DIR=/var/lib/tailscale \
    docker.io/tailscale/tailscale:stable \
    sh -c 'tailscale web --listen 0.0.0.0:8240 & exec tailscaled' 2>>"$LOG" || true
fi

# Immich stack (postgres + redis + server - ML optional)
# Remove old single-container 'immich' if present (wrong port 2283:3001, conflicts with immich_server)
if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
  log "Removing old immich container (use immich_server stack)..."
  $DOCKER stop immich 2>/dev/null || true
  $DOCKER rm -f immich 2>/dev/null || true
  $DOCKER start immich_server 2>/dev/null || true
fi
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
  log "Creating Immich stack..."
  mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
  $DOCKER network create immich-net 2>/dev/null || true
  if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
    $DOCKER run -d --name immich_postgres --restart unless-stopped --memory=$(mem_limit immich_postgres) --network immich-net \
      -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
      -e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
      ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>>"$LOG" || true
    sleep 3
    for i in 1 2 3 4 5 6 7 8 9 10; do
      $DOCKER exec immich_postgres pg_isready -U postgres 2>/dev/null && break
      sleep 2
    done
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
    $DOCKER run -d --name immich_redis --restart unless-stopped --memory=$(mem_limit immich_redis) --network immich-net \
      docker.io/valkey/valkey:7-alpine 2>>"$LOG" || true
    sleep 2
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
    $DOCKER run -d --name immich_server --restart unless-stopped --memory=$(mem_limit immich_server) --network immich-net \
      -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
      -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
      -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
      -e UPLOAD_LOCATION=/usr/src/app/upload \
      ghcr.io/immich-app/immich-server:release 2>>"$LOG" || true
  fi
fi

# Penpot stack (postgres + valkey + backend + exporter + frontend)
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
  log "Creating Penpot stack..."
  mkdir -p /var/lib/archipelago/penpot-assets /var/lib/archipelago/penpot-postgres
  $DOCKER network create penpot-net 2>/dev/null || true
  if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q penpot-postgres; then
    $DOCKER run -d --name penpot-postgres --restart unless-stopped --memory=$(mem_limit penpot-postgres) --network penpot-net \
      -v /var/lib/archipelago/penpot-postgres:/var/lib/postgresql/data \
      -e POSTGRES_DB=penpot -e POSTGRES_USER=penpot -e POSTGRES_PASSWORD=$PENPOT_DB_PASS \
      docker.io/postgres:15 2>>"$LOG" || true
    sleep 5
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-valkey; then
    $DOCKER run -d --name penpot-valkey --restart unless-stopped --memory=$(mem_limit penpot-valkey) --network penpot-net \
      -e VALKEY_EXTRA_FLAGS="--maxmemory 128mb --maxmemory-policy volatile-lfu" \
      docker.io/valkey/valkey:8.1 2>>"$LOG" || true
    sleep 3
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-backend; then
    $DOCKER run -d --name penpot-backend --restart unless-stopped --memory=$(mem_limit penpot-backend) --network penpot-net \
      -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
      -e PENPOT_PUBLIC_URI="http://${TARGET_IP}:9001" \
      -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
      -e PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot \
      -e PENPOT_DATABASE_USERNAME=penpot -e PENPOT_DATABASE_PASSWORD=$PENPOT_DB_PASS \
      -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
      -e PENPOT_OBJECTS_STORAGE_BACKEND=fs \
      -e PENPOT_OBJECTS_STORAGE_FS_DIRECTORY=/opt/data/assets \
      -e PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies \
      docker.io/penpotapp/backend:latest 2>>"$LOG" || true
    sleep 5
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-exporter; then
    $DOCKER run -d --name penpot-exporter --restart unless-stopped --memory=$(mem_limit penpot-exporter) --network penpot-net \
      -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
      -e PENPOT_PUBLIC_URI=http://penpot-frontend:8080 \
      -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
      docker.io/penpotapp/exporter:latest 2>>"$LOG" || true
    sleep 2
  fi
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
    $DOCKER run -d --name penpot-frontend --restart unless-stopped --memory=$(mem_limit penpot-frontend) --network penpot-net \
      -p 9001:8080 -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
      -e PENPOT_PUBLIC_URI="http://${TARGET_IP}:9001" \
      -e PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies \
      docker.io/penpotapp/frontend:latest 2>>"$LOG" || true
  fi
fi

# 8. Nostr relays (optional - only if images were loaded; deploy does not create these on first boot)
# nostr-rs-relay and strfry are in ISO image bundle; create if image exists
if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'nostr-rs-relay'; then
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nostr-rs-relay; then
    log "Creating nostr-rs-relay..."
    mkdir -p /var/lib/archipelago/nostr-rs-relay
    $DOCKER run -d --name nostr-rs-relay --restart unless-stopped --memory=$(mem_limit nostr-rs-relay) \
      -p 7047:7047 -v /var/lib/archipelago/nostr-rs-relay:/data \
      scsibug/nostr-rs-relay:latest 2>>"$LOG" || true
  fi
fi
if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'strfry'; then
  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q strfry; then
    log "Creating strfry..."
    mkdir -p /var/lib/archipelago/strfry
    $DOCKER run -d --name strfry --restart unless-stopped --memory=$(mem_limit strfry) \
      -p 7777:7777 -v /var/lib/archipelago/strfry:/data \
      hoytech/strfry:latest 2>>"$LOG" || true
  fi
fi

# 8b. Indeehub (pull from registry, or use local build)
if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q indeedhub; then
  INDEEDHUB_IMAGE=""
  # Try local image first (pre-built or loaded from ISO)
  if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'localhost/indeedhub'; then
    INDEEDHUB_IMAGE="localhost/indeedhub:latest"
  # Try registry image
  elif $DOCKER pull git.tx1138.com/lfg2025/indeedhub:latest 2>>"$LOG"; then
    INDEEDHUB_IMAGE="git.tx1138.com/lfg2025/indeedhub:latest"
  fi
  if [ -n "$INDEEDHUB_IMAGE" ]; then
    log "Creating Indeehub from $INDEEDHUB_IMAGE..."
    $DOCKER run -d --name indeedhub --restart unless-stopped --memory=$(mem_limit indeedhub) \
      --cap-drop ALL --security-opt no-new-privileges:true \
      --read-only --tmpfs /tmp:rw,noexec,nosuid,size=64m --tmpfs /app/.next/cache:rw,noexec,nosuid,size=128m \
      -p 7777:7777 \
      -e NODE_ENV=production -e NEXT_TELEMETRY_DISABLED=1 \
      "$INDEEDHUB_IMAGE" 2>>"$LOG" || true
    # Fix IndeedHub for iframe: remove X-Frame-Options so it loads in Archipelago panel
    sleep 2
    if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q "^indeedhub$"; then
      $DOCKER exec indeedhub sed -i "/X-Frame-Options/d" /etc/nginx/conf.d/default.conf 2>/dev/null || true
      if [ -f /opt/archipelago/web-ui/nostr-provider.js ]; then
        $DOCKER cp /opt/archipelago/web-ui/nostr-provider.js indeedhub:/usr/share/nginx/html/nostr-provider.js 2>/dev/null || true
      fi
      $DOCKER exec indeedhub nginx -s reload 2>/dev/null || true
      log "Applied IndeedHub iframe fix (removed X-Frame-Options)"
    fi
  fi
fi

# 9. Custom UI containers (bitcoin-ui, lnd-ui)
# These are built from Dockerfiles in /opt/archipelago/docker/ or loaded from pre-built images.
for ui in bitcoin-ui lnd-ui; do
  if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q "$ui"; then
    continue
  fi
  case $ui in
    bitcoin-ui) PORT_ARG=""; NET_ARG="--network host" ;;  # host network: proxies Bitcoin RPC at 127.0.0.1:8332
    lnd-ui)     PORT_ARG="-p 8081:80"; NET_ARG="" ;;
  esac
  CONTAINER_NAME="archy-$ui"
  if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q "$ui"; then
    log "Starting $ui from pre-built image..."
    IMG=$($DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep "$ui" | head -1)
    $DOCKER run -d --name "$CONTAINER_NAME" $PORT_ARG --restart unless-stopped --memory=$(mem_limit "$CONTAINER_NAME") $NET_ARG "$IMG" 2>>"$LOG" || true
  elif [ -d "/opt/archipelago/docker/$ui" ]; then
    log "Building $ui from source (/opt/archipelago/docker/$ui)..."
    if $DOCKER build -t "$ui:latest" "/opt/archipelago/docker/$ui" 2>>"$LOG"; then
      $DOCKER run -d --name "$CONTAINER_NAME" $PORT_ARG --restart unless-stopped --memory=$(mem_limit "$CONTAINER_NAME") $NET_ARG "$ui:latest" 2>>"$LOG" || true
    fi
  elif [ -d "/home/archipelago/archy/docker/$ui" ]; then
    log "Building $ui from source (/home/archipelago/archy/docker/$ui)..."
    if $DOCKER build -t "$ui:latest" "/home/archipelago/archy/docker/$ui" 2>>"$LOG"; then
      $DOCKER run -d --name "$CONTAINER_NAME" $PORT_ARG --restart unless-stopped --memory=$(mem_limit "$CONTAINER_NAME") $NET_ARG "$ui:latest" 2>>"$LOG" || true
    fi
  else
    log "$ui: no image or source found, skipping"
  fi
done

# 10. Initialize backend data directories
# tor-config: backend stores tor service configs here (writable by archipelago user)
mkdir -p /var/lib/archipelago/tor-config
SERVICES_JSON=/var/lib/archipelago/tor-config/services.json
if [ ! -f "$SERVICES_JSON" ]; then
  cat > "$SERVICES_JSON" <<'SJSON'
{"services":[
  {"name":"archipelago","local_port":80,"enabled":true},
  {"name":"lnd","local_port":8081,"enabled":true},
  {"name":"btcpay","local_port":23000,"enabled":true},
  {"name":"mempool","local_port":4080,"enabled":true},
  {"name":"fedimint","local_port":8175,"enabled":true}
]}
SJSON
  log "Created initial tor-config/services.json"
fi

# identities: backend identity manager stores DIDs here
mkdir -p /var/lib/archipelago/identities

# Ensure archipelago user can write to these directories
chown -R 1000:1000 /var/lib/archipelago/tor-config /var/lib/archipelago/identities 2>/dev/null || true

# 11. Post-boot validation
log "Validating container creation..."
TOTAL=0; RUNNING=0
for c in bitcoin-knots lnd btcpay-server fedimint homeassistant grafana uptime-kuma; do
  TOTAL=$((TOTAL + 1))
  if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q "$c"; then
    RUNNING=$((RUNNING + 1))
  fi
done
log "Post-boot validation: $RUNNING/$TOTAL core containers running"

# 12. Run container doctor for any remaining issues
log "Running container doctor..."
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [ -x "$SCRIPT_DIR/container-doctor.sh" ]; then
    bash "$SCRIPT_DIR/container-doctor.sh" --local 2>&1 | tee -a "$LOG"
elif [ -x "/opt/archipelago/scripts/container-doctor.sh" ]; then
    bash "/opt/archipelago/scripts/container-doctor.sh" --local 2>&1 | tee -a "$LOG"
fi

log "First-boot container creation complete"