feat: Phase 1 — per-installation credential generation, eliminate hardcoded passwords

Generate unique random passwords at first boot for Bitcoin RPC, all database
services (mempool, btcpay, immich, penpot, mysql-root), and Fedimint gateway.
Credentials stored in /var/lib/archipelago/secrets/ with 600 permissions.

Scripts: first-boot-containers.sh, deploy-to-target.sh, deploy-bitcoin-knots.sh,
container-doctor.sh all read from secrets files instead of hardcoded values.

Rust backend: new bitcoin_rpc module reads password from secrets file, env var,
or dev fallback. All .basic_auth() calls and container config strings now use
the shared credential reader instead of hardcoded "archipelago123".

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

core/archipelago/src/api/rpc/bitcoin.rs

@@ -77,6 +77,7 @@ impl RpcHandler {
         method: &str,
         params: &[serde_json::Value],
     ) -> Result<T> {
+        let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
         let body = serde_json::json!({
             "jsonrpc": "1.0",
             "id": "archy",
@@ -86,7 +87,7 @@ impl RpcHandler {
 
         let resp = client
             .post("http://127.0.0.1:8332/")
-            .basic_auth("archipelago", Some("archipelago123"))
+            .basic_auth(&rpc_user, Some(&rpc_pass))
             .json(&body)
             .send()
             .await

core/archipelago/src/api/rpc/package.rs

@@ -193,12 +193,15 @@ impl RpcHandler {
             "--restart=unless-stopped",  // Auto-restart policy
         ];
 
+        // Read Bitcoin RPC password from secrets for container configs
+        let rpc_pass = crate::bitcoin_rpc::bitcoin_rpc_password().await;
+
         // App-specific configuration (should come from manifest)
         let (mut ports, mut volumes, env_vars, custom_command, mut custom_args) = {
             let mut allocator = self.port_allocator.lock().map_err(|e| {
                 anyhow::anyhow!("Port allocator lock poisoned: {}", e)
             })?;
-            get_app_config(package_id, &self.config.host_ip, &mut allocator)
+            get_app_config(package_id, &self.config.host_ip, &mut allocator, &rpc_pass)
         };
 
         // Fedimint Gateway: auto-detect LND and switch to lnd mode
@@ -222,7 +225,7 @@ impl RpcHandler {
                     "--network".to_string(), "bitcoin".to_string(),
                     "--bitcoind-url".to_string(), format!("http://{}:8332", self.config.host_ip),
                     "--bitcoind-username".to_string(), "archipelago".to_string(),
-                    "--bitcoind-password".to_string(), "archipelago123".to_string(),
+                    "--bitcoind-password".to_string(), rpc_pass.clone(),
                     "lnd".to_string(),
                     "--lnd-rpc-host".to_string(), format!("{}:10009", self.config.host_ip),
                     "--lnd-tls-cert".to_string(), "/lnd/tls.cert".to_string(),
@@ -305,16 +308,16 @@ impl RpcHandler {
         if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") {
             let bitcoin_dir = "/var/lib/archipelago/bitcoin";
             let conf_path = format!("{}/bitcoin.conf", bitcoin_dir);
-            let bitcoin_conf = "\
+            let bitcoin_conf = format!("\
 server=1\n\
 prune=550\n\
 rpcuser=archipelago\n\
-rpcpassword=archipelago123\n\
+rpcpassword={}\n\
 rpcbind=0.0.0.0\n\
 rpcallowip=0.0.0.0/0\n\
 rpcport=8332\n\
 listen=1\n\
-printtoconsole=1\n";
+printtoconsole=1\n", rpc_pass);
             let _ = tokio::fs::create_dir_all(bitcoin_dir).await;
             let _ = tokio::fs::write(&conf_path, bitcoin_conf).await;
             info!("Created bitcoin.conf at {} with RPC + txindex enabled", conf_path);
@@ -347,7 +350,7 @@ printtoconsole=1\n";
         run_args.push("--cpus=2");
 
         // Health check definitions
-        let health_args = get_health_check_args(package_id);
+        let health_args = get_health_check_args(package_id, &rpc_pass);
         for arg in &health_args {
             run_args.push(arg);
         }
@@ -1316,10 +1319,11 @@ fn is_readonly_compatible(app_id: &str) -> bool {
 
 /// Get container health check arguments for podman run.
 /// Returns (health-cmd, interval, retries) args to append to run_args.
-fn get_health_check_args(app_id: &str) -> Vec<String> {
+fn get_health_check_args(app_id: &str, rpc_pass: &str) -> Vec<String> {
+    let btc_health = format!("bitcoin-cli -rpcuser=archipelago -rpcpassword={} getblockchaininfo || exit 1", rpc_pass);
     let (cmd, interval, retries) = match app_id {
         "bitcoin" | "bitcoin-core" | "bitcoin-knots" => (
-            "bitcoin-cli -rpcuser=archipelago -rpcpassword=archipelago123 getblockchaininfo || exit 1",
+            btc_health.as_str(),
             "30s", "3",
         ),
         "lnd" => (
@@ -1462,6 +1466,7 @@ fn get_app_config(
     app_id: &str,
     host_ip: &str,
     allocator: &mut PortAllocator,
+    rpc_pass: &str,
 ) -> (Vec<String>, Vec<String>, Vec<String>, Option<String>, Option<Vec<String>>) {
     match app_id {
         "homeassistant" | "home-assistant" => (
@@ -1495,7 +1500,7 @@ fn get_app_config(
                 "BTCPAY_CHAINS=btc".to_string(),
                 format!("BTCPAY_BTCRPCURL=http://{}:8332", host_ip),
                 "BTCPAY_BTCRPCUSER=archipelago".to_string(),
-                "BTCPAY_BTCRPCPASSWORD=archipelago123".to_string(),
+                format!("BTCPAY_BTCRPCPASSWORD={}", rpc_pass),
                 "BTCPAY_POSTGRES=User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true".to_string(),
             ],
             None,
@@ -1519,7 +1524,7 @@ fn get_app_config(
                 format!("CORE_RPC_HOST={}", host_ip),
                 "CORE_RPC_PORT=8332".to_string(),
                 "CORE_RPC_USERNAME=archipelago".to_string(),
-                "CORE_RPC_PASSWORD=archipelago123".to_string(),
+                format!("CORE_RPC_PASSWORD={}", rpc_pass),
                 "DATABASE_ENABLED=true".to_string(),
                 "DATABASE_HOST=archy-mempool-db".to_string(),
                 "DATABASE_DATABASE=mempool".to_string(),
@@ -1536,7 +1541,7 @@ fn get_app_config(
                 vec!["50001:50001".to_string()],
                 vec!["/var/lib/archipelago/electrumx:/data".to_string()],
                 vec![
-                    format!("DAEMON_URL=http://archipelago:archipelago123@{}:8332/", bitcoin_host),
+                    format!("DAEMON_URL=http://archipelago:{}@{}:8332/", rpc_pass, bitcoin_host),
                     "COIN=Bitcoin".to_string(),
                     "DB_DIRECTORY=/data".to_string(),
                     "SERVICES=tcp://:50001,rpc://0.0.0.0:8000".to_string(),
@@ -1701,7 +1706,7 @@ fn get_app_config(
             vec![
                 "FM_DATA_DIR=/data".to_string(),
                 "FM_BITCOIND_USERNAME=archipelago".to_string(),
-                "FM_BITCOIND_PASSWORD=archipelago123".to_string(),
+                format!("FM_BITCOIND_PASSWORD={}", rpc_pass),
                 "FM_BITCOIN_NETWORK=bitcoin".to_string(),
                 "FM_BIND_P2P=0.0.0.0:8173".to_string(),
                 "FM_BIND_API=0.0.0.0:8174".to_string(),
@@ -1727,7 +1732,7 @@ fn get_app_config(
                 "--network".to_string(), "bitcoin".to_string(),
                 "--bitcoind-url".to_string(), format!("http://{}:8332", host_ip),
                 "--bitcoind-username".to_string(), "archipelago".to_string(),
-                "--bitcoind-password".to_string(), "archipelago123".to_string(),
+                "--bitcoind-password".to_string(), rpc_pass.to_string(),
                 "ldk".to_string(),
                 "--ldk-lightning-port".to_string(), "9737".to_string(),
                 "--ldk-alias".to_string(), "archipelago-gateway".to_string(),

core/archipelago/src/bitcoin_rpc.rs

@@ -0,0 +1,49 @@
+//! Shared Bitcoin RPC credential management.
+//! Reads credentials from the per-installation secrets file, falling back to
+//! environment variables, then a dev-only default.
+
+use tokio::sync::OnceCell;
+use tracing::debug;
+
+const SECRETS_PATH: &str = "/var/lib/archipelago/secrets/bitcoin-rpc-password";
+const DEFAULT_USER: &str = "archipelago";
+
+static CACHED_PASSWORD: OnceCell<String> = OnceCell::const_new();
+
+/// Read the Bitcoin RPC password from the secrets file, env var, or dev fallback.
+async fn read_password() -> String {
+    // 1. Try secrets file (production path)
+    if let Ok(pass) = tokio::fs::read_to_string(SECRETS_PATH).await {
+        let pass = pass.trim().to_string();
+        if !pass.is_empty() {
+            debug!("Bitcoin RPC password loaded from secrets file");
+            return pass;
+        }
+    }
+
+    // 2. Try environment variable
+    if let Ok(pass) = std::env::var("BITCOIN_RPC_PASSWORD") {
+        if !pass.is_empty() {
+            debug!("Bitcoin RPC password loaded from env var");
+            return pass;
+        }
+    }
+
+    // 3. Dev fallback (will only work on dev machines with default config)
+    debug!("Bitcoin RPC password: using dev fallback");
+    "archipelago123".to_string()
+}
+
+/// Get Bitcoin RPC credentials (user, password). Cached after first call.
+pub async fn bitcoin_rpc_credentials() -> (String, String) {
+    let pass = CACHED_PASSWORD
+        .get_or_init(|| async { read_password().await })
+        .await;
+    (DEFAULT_USER.to_string(), pass.clone())
+}
+
+/// Get the Bitcoin RPC password as a plain string (for config generation).
+pub async fn bitcoin_rpc_password() -> String {
+    let (_, pass) = bitcoin_rpc_credentials().await;
+    pass
+}

core/archipelago/src/electrs_status.rs

@@ -13,11 +13,9 @@ const ELECTRUMX_DATA_DIR: &str = "/var/lib/archipelago/electrumx";
 // Approximate final index size in bytes for mainnet (~55GB for ElectrumX full index)
 const ESTIMATED_FULL_INDEX_BYTES: f64 = 55_000_000_000.0;
 
-/// Build Bitcoin RPC Basic auth header from env vars.
+/// Build Bitcoin RPC Basic auth header using shared credentials.
-/// Falls back to cookie auth file if env vars are not set.
+async fn bitcoin_rpc_auth() -> String {
-fn bitcoin_rpc_auth() -> String {
+    let (user, pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
-    let user = std::env::var("BITCOIN_RPC_USER").unwrap_or_else(|_| "archipelago".to_string());
-    let pass = std::env::var("BITCOIN_RPC_PASSWORD").unwrap_or_else(|_| "archipelago123".to_string());
     use base64::Engine;
     let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{}:{}", user, pass));
     format!("Basic {}", encoded)
@@ -120,7 +118,7 @@ async fn bitcoin_network_height() -> Result<u64> {
     let resp = client
         .post(BITCOIN_RPC_URL)
         .header("Content-Type", "application/json")
-        .header("Authorization", bitcoin_rpc_auth())
+        .header("Authorization", bitcoin_rpc_auth().await)
         .body(body.to_string())
         .send()
         .await

core/archipelago/src/main.rs

@@ -9,6 +9,7 @@ use tokio::signal;
 mod api;
 mod auth;
 mod backup;
+mod bitcoin_rpc;
 mod config;
 mod content_server;
 mod crash_recovery;

core/archipelago/src/mesh/listener.rs

@@ -1301,6 +1301,8 @@ async fn handle_tx_relay_broadcast(
         }
     };
 
+    let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
+
     // Pre-flight: check if Bitcoin Core is reachable and synced
     let preflight_body = serde_json::json!({
         "jsonrpc": "1.0",
@@ -1311,7 +1313,7 @@ async fn handle_tx_relay_broadcast(
 
     match client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&preflight_body)
         .send()
         .await
@@ -1364,7 +1366,7 @@ async fn handle_tx_relay_broadcast(
 
     let txid = match client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&body)
         .send()
         .await
@@ -1522,9 +1524,10 @@ async fn check_tx_confirmations(client: &reqwest::Client, txid: &str) -> anyhow:
         "method": "gettransaction",
         "params": [txid]
     });
+    let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
     let resp = client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&body)
         .send()
         .await?;

core/archipelago/src/mesh/mod.rs

@@ -602,12 +602,13 @@ struct BlockHeaderInfo {
 }
 
 async fn bitcoin_rpc_getblockcount(client: &reqwest::Client) -> Result<u64> {
+    let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
     let body = serde_json::json!({
         "jsonrpc": "1.0", "id": "mesh", "method": "getblockcount", "params": []
     });
     let resp: BitcoinRpcResponse<u64> = client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&body)
         .send()
         .await
@@ -625,13 +626,14 @@ async fn bitcoin_rpc_getblockheader_by_height(
     client: &reqwest::Client,
     height: u64,
 ) -> Result<BlockHeaderInfo> {
+    let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
     // First get block hash for this height
     let body = serde_json::json!({
         "jsonrpc": "1.0", "id": "mesh", "method": "getblockhash", "params": [height]
     });
     let resp: BitcoinRpcResponse<String> = client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&body)
         .send()
         .await?
@@ -645,7 +647,7 @@ async fn bitcoin_rpc_getblockheader_by_height(
     });
     let resp: BitcoinRpcResponse<serde_json::Value> = client
         .post("http://127.0.0.1:8332/")
-        .basic_auth("archipelago", Some("archipelago123"))
+        .basic_auth(&rpc_user, Some(&rpc_pass))
         .json(&body)
         .send()
         .await?

scripts/container-doctor.sh

@@ -0,0 +1,413 @@
+#!/bin/bash
+#
+# Container Doctor — diagnose and fix common container health issues
+#
+# Usage:
+#   sudo ./scripts/container-doctor.sh              # Run locally on node
+#   ./scripts/container-doctor.sh user@host          # Run remotely via SSH
+#
+# Fixes:
+#   1. Stale podman ps/stats processes (>10 = pileup)
+#   2. Orphaned conmon/crun processes holding ports
+#   3. System tor conflicting with container tor
+#   4. Tor hidden service directory permissions (must be 700)
+#   5. SearXNG read-only root / cap-drop ALL
+#   6. Bitcoin Knots prune+txindex conflict
+#   7. Containers stuck with exit code 127 (binary not found)
+#
+# Safe to run multiple times (idempotent). Never blocks deploy (exit 0 always).
+#
+
+set -o pipefail
+
+FIXES_APPLIED=0
+CHECKS_PASSED=0
+FIX_NAMES=()
+
+log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
+
+run_fix() {
+    local name="$1"
+    shift
+    if "$@"; then
+        FIXES_APPLIED=$((FIXES_APPLIED + 1))
+        FIX_NAMES+=("$name")
+    else
+        CHECKS_PASSED=$((CHECKS_PASSED + 1))
+    fi
+}
+
+# ── Fix 1: Stale podman processes ────────────────────────────
+fix_stale_podman() {
+    local count
+    count=$(pgrep -f "podman (ps|stats)" 2>/dev/null | wc -l)
+    count=${count:-0}
+    if [ "$count" -gt 10 ]; then
+        log "Killing $count stale podman ps/stats processes"
+        pkill -f "podman (ps|stats)" 2>/dev/null || true
+        sleep 2
+        local after
+        after=$(pgrep -f "podman (ps|stats)" 2>/dev/null | wc -l)
+        after=${after:-0}
+        log "Reduced from $count to $after"
+        return 0
+    fi
+    return 1
+}
+
+# ── Fix 2: Orphaned conmon holding ports ─────────────────────
+fix_orphaned_conmon() {
+    local fixed=false
+    # Find conmon processes whose containers no longer exist
+    local pids
+    pids=$(pgrep -f "conmon.*--exit-command" 2>/dev/null || true)
+    if [ -z "$pids" ]; then
+        return 1
+    fi
+
+    for pid in $pids; do
+        # Extract container ID from conmon args
+        local cid
+        cid=$(tr '\0' ' ' < /proc/"$pid"/cmdline 2>/dev/null | grep -oP '(?<=-c )[a-f0-9]{64}' || true)
+        if [ -z "$cid" ]; then
+            continue
+        fi
+        # Check if container still exists
+        if ! podman inspect "$cid" &>/dev/null; then
+            local port_info
+            port_info=$(ss -tlnp 2>/dev/null | grep "pid=$pid" | grep -oP ':\K\d+' | head -3 | tr '\n' ',' | sed 's/,$//')
+            log "Killing orphaned conmon pid=$pid (ports: ${port_info:-none})"
+            kill "$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null || true
+            fixed=true
+        fi
+    done
+
+    $fixed && return 0 || return 1
+}
+
+# ── Fix 3: System tor conflict ───────────────────────────────
+fix_system_tor_conflict() {
+    # Only relevant if we have a container tor on host network
+    local has_container_tor=false
+    if podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qE '^archy-tor$'; then
+        local net_mode
+        net_mode=$(podman inspect archy-tor --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || true)
+        if [ "$net_mode" = "host" ]; then
+            has_container_tor=true
+        fi
+    fi
+
+    if ! $has_container_tor; then
+        return 1
+    fi
+
+    # Check if system tor is binding port 9050
+    local system_tor_pid
+    system_tor_pid=$(ss -tlnp 2>/dev/null | grep ':9050 ' | grep -oP 'pid=\K\d+' | head -1)
+    if [ -z "$system_tor_pid" ]; then
+        return 1
+    fi
+
+    # Check if it's the system tor (not container tor)
+    local exe
+    exe=$(readlink /proc/"$system_tor_pid"/exe 2>/dev/null || true)
+    if [[ "$exe" == */tor ]] && ! grep -q "container" /proc/"$system_tor_pid"/cgroup 2>/dev/null; then
+        log "System tor (pid=$system_tor_pid) conflicts with container tor on port 9050"
+        systemctl stop tor@default 2>/dev/null || true
+        systemctl stop tor 2>/dev/null || true
+        systemctl disable tor@default 2>/dev/null || true
+        systemctl disable tor 2>/dev/null || true
+        sleep 2
+        # Restart container tor now that port is free
+        podman restart archy-tor 2>/dev/null || true
+        log "Disabled system tor, restarted container tor"
+        return 0
+    fi
+
+    return 1
+}
+
+# ── Fix 4: Tor hidden service permissions ────────────────────
+fix_tor_permissions() {
+    local fixed=false
+    local tor_dirs=("/var/lib/archipelago/tor" "/var/lib/tor")
+
+    for base in "${tor_dirs[@]}"; do
+        if [ ! -d "$base" ]; then
+            continue
+        fi
+        while IFS= read -r dir; do
+            local perms
+            perms=$(stat -c '%a' "$dir" 2>/dev/null)
+            if [ "$perms" != "700" ]; then
+                chmod 700 "$dir"
+                log "Fixed permissions on $dir ($perms -> 700)"
+                fixed=true
+            fi
+        done < <(find "$base" -maxdepth 1 -name "hidden_service_*" -type d 2>/dev/null)
+    done
+
+    # If we fixed permissions and tor container exists, restart it
+    if $fixed; then
+        podman restart archy-tor 2>/dev/null || true
+        return 0
+    fi
+    return 1
+}
+
+# ── Fix 5: SearXNG read-only / cap-drop ─────────────────────
+fix_searxng() {
+    if ! podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^searxng$'; then
+        return 1
+    fi
+
+    local state
+    state=$(podman inspect searxng --format '{{.State.Status}}' 2>/dev/null || true)
+    local readonly_root
+    readonly_root=$(podman inspect searxng --format '{{.HostConfig.ReadonlyRootfs}}' 2>/dev/null || true)
+    local cap_drop
+    cap_drop=$(podman inspect searxng --format '{{.HostConfig.CapDrop}}' 2>/dev/null || true)
+
+    # Fix if: exited, or has read-only root, or has cap-drop ALL
+    local needs_fix=false
+    if [ "$state" = "exited" ]; then
+        needs_fix=true
+    fi
+    if [ "$readonly_root" = "true" ]; then
+        needs_fix=true
+    fi
+    if [[ "$cap_drop" == *"ALL"* ]] || [[ "$cap_drop" == *"all"* ]]; then
+        needs_fix=true
+    fi
+
+    if ! $needs_fix; then
+        return 1
+    fi
+
+    log "Recreating SearXNG (readonly=$readonly_root, cap_drop=$cap_drop, state=$state)"
+
+    # Get current port mapping
+    local port
+    port=$(podman inspect searxng --format '{{range $k,$v := .HostConfig.PortBindings}}{{$k}}={{range $v}}{{.HostPort}}{{end}}{{println}}{{end}}' 2>/dev/null | head -1)
+    local host_port="${port##*=}"
+    host_port="${host_port:-8888}"
+
+    # Kill any stale conmon holding the port
+    local conmon_pid
+    conmon_pid=$(ss -tlnp 2>/dev/null | grep ":${host_port} " | grep -oP 'pid=\K\d+' | head -1)
+
+    podman stop searxng 2>/dev/null || true
+    podman rm -f searxng 2>/dev/null || true
+
+    if [ -n "$conmon_pid" ]; then
+        kill -9 "$conmon_pid" 2>/dev/null || true
+        sleep 2
+    fi
+
+    podman run -d \
+        --name searxng \
+        --restart=unless-stopped \
+        --security-opt=no-new-privileges:true \
+        --tmpfs /tmp:rw,noexec,nosuid,size=256m \
+        -v searxng-config:/etc/searxng:rw \
+        -v searxng-cache:/var/cache/searxng:rw \
+        -p "${host_port}:8080" \
+        --memory=512m \
+        docker.io/searxng/searxng:latest 2>&1 || true
+
+    log "SearXNG recreated (no readonly, no cap-drop ALL)"
+    return 0
+}
+
+# ── Fix 6: Bitcoin Knots prune+txindex conflict ──────────────
+fix_bitcoin_txindex() {
+    if ! podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^bitcoin-knots$'; then
+        return 1
+    fi
+
+    # Check if bitcoin.conf has prune enabled
+    local conf="/var/lib/archipelago/bitcoin/bitcoin.conf"
+    if [ ! -f "$conf" ] || ! grep -q '^prune=' "$conf"; then
+        return 1
+    fi
+
+    # Check if container args include txindex
+    local cmd
+    cmd=$(podman inspect bitcoin-knots --format '{{json .Config.Cmd}}' 2>/dev/null || true)
+    if ! echo "$cmd" | grep -q "txindex"; then
+        return 1
+    fi
+
+    log "Bitcoin Knots: prune+txindex conflict detected"
+
+    # Get current config
+    local image
+    image=$(podman inspect bitcoin-knots --format '{{.ImageName}}' 2>/dev/null)
+    local network
+    network=$(podman inspect bitcoin-knots --format '{{.HostConfig.NetworkMode}}' 2>/dev/null)
+
+    # Read per-installation RPC password
+    local SECRETS_DIR="/var/lib/archipelago/secrets"
+    local BTC_RPC_PASS="archipelago"
+    if [ -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
+        BTC_RPC_PASS=$(cat "$SECRETS_DIR/bitcoin-rpc-password")
+    fi
+
+    # Ensure bitcoin.conf has all RPC settings
+    if ! grep -q 'rpcuser=' "$conf"; then
+        cat > "$conf" <<BCONF
+server=1
+prune=550
+rpcuser=archipelago
+rpcpassword=$BTC_RPC_PASS
+rpcbind=0.0.0.0
+rpcallowip=0.0.0.0/0
+rpcport=8332
+listen=1
+printtoconsole=1
+BCONF
+        log "Updated bitcoin.conf with full RPC settings"
+    fi
+
+    # Remove stale txindex if present
+    if [ -d "/var/lib/archipelago/bitcoin/indexes/txindex" ]; then
+        find /var/lib/archipelago/bitcoin/indexes/txindex -type f -delete 2>/dev/null
+        rmdir /var/lib/archipelago/bitcoin/indexes/txindex 2>/dev/null || true
+        log "Removed stale txindex directory"
+    fi
+
+    # Recreate without txindex
+    podman stop bitcoin-knots 2>/dev/null || true
+    podman rm -f bitcoin-knots 2>/dev/null || true
+    sleep 2
+
+    # Kill stale conmon on port 8332/8333
+    for p in 8332 8333; do
+        local cpid
+        cpid=$(ss -tlnp 2>/dev/null | grep ":${p} " | grep -oP 'pid=\K\d+' | head -1)
+        if [ -n "$cpid" ]; then
+            kill -9 "$cpid" 2>/dev/null || true
+        fi
+    done
+    sleep 1
+
+    local net_arg=""
+    if [ -n "$network" ] && [ "$network" != "bridge" ] && [ "$network" != "host" ]; then
+        net_arg="--network=$network"
+    elif [ "$network" = "host" ]; then
+        net_arg="--network=host"
+    else
+        net_arg="--network=archy-net"
+    fi
+
+    podman run -d \
+        --name bitcoin-knots \
+        --restart=always \
+        $net_arg \
+        -p 8332:8332 \
+        -p 8333:8333 \
+        -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
+        --memory=2g \
+        --cap-drop=ALL \
+        --cap-add=CHOWN \
+        --cap-add=FOWNER \
+        --cap-add=SETUID \
+        --cap-add=SETGID \
+        --cap-add=DAC_OVERRIDE \
+        --security-opt=no-new-privileges:true \
+        --health-cmd="bitcoin-cli -rpcuser=archipelago -rpcpassword=$BTC_RPC_PASS getblockchaininfo || exit 1" \
+        --health-interval=30s \
+        --health-retries=3 \
+        "$image" 2>&1 || true
+
+    log "Bitcoin Knots recreated without txindex (prune mode)"
+    return 0
+}
+
+# ── Fix 7: Exit code 127 containers ─────────────────────────
+fix_exit_127() {
+    local containers
+    containers=$(podman ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null | grep 'Exited (127)' | awk '{print $1}' || true)
+
+    if [ -z "$containers" ]; then
+        return 1
+    fi
+
+    local fixed_names=()
+    for name in $containers; do
+        # Skip containers handled by other fixes
+        if [ "$name" = "searxng" ]; then
+            continue
+        fi
+
+        log "Container $name has exit code 127 — recreating"
+        # Get image and create command for recreation
+        local image
+        image=$(podman inspect "$name" --format '{{.ImageName}}' 2>/dev/null || true)
+        local create_cmd
+        create_cmd=$(podman inspect "$name" --format '{{json .Config.CreateCommand}}' 2>/dev/null || true)
+
+        podman rm -f "$name" 2>/dev/null || true
+
+        if [ -n "$create_cmd" ] && [ "$create_cmd" != "null" ]; then
+            # Re-run the original create command (strip the leading "podman" and "run")
+            local recreate_args
+            recreate_args=$(echo "$create_cmd" | python3 -c "
+import json, sys
+args = json.load(sys.stdin)
+# Skip 'podman' and 'run', output the rest
+print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
+" 2>/dev/null || true)
+
+            if [ -n "$recreate_args" ]; then
+                eval "podman run $recreate_args" 2>&1 || true
+                fixed_names+=("$name")
+                log "Recreated $name from original args"
+            else
+                fixed_names+=("$name(removed)")
+                log "Removed $name — will be recreated on next deploy"
+            fi
+        else
+            fixed_names+=("$name(removed)")
+            log "Removed $name — will be recreated on next deploy"
+        fi
+    done
+
+    [ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1
+}
+
+# ── Main ─────────────────────────────────────────────────────
+
+# If remote host provided, run via SSH
+if [ -n "$1" ] && [ "$1" != "--local" ]; then
+    REMOTE_HOST="$1"
+    SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
+    SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -i $SSH_KEY"
+
+    log "Running container doctor on $REMOTE_HOST"
+
+    # Copy script to remote and execute
+    scp $SSH_OPTS "$0" "$REMOTE_HOST:/tmp/container-doctor.sh" 2>/dev/null
+    ssh $SSH_OPTS "$REMOTE_HOST" "sudo bash /tmp/container-doctor.sh --local" 2>&1
+    exit 0
+fi
+
+# Running locally (on the node itself)
+log "Starting container health check"
+
+run_fix "stale-podman"    fix_stale_podman
+run_fix "orphaned-conmon" fix_orphaned_conmon
+run_fix "system-tor"      fix_system_tor_conflict
+run_fix "tor-permissions"  fix_tor_permissions
+run_fix "searxng"         fix_searxng
+run_fix "bitcoin-txindex" fix_bitcoin_txindex
+run_fix "exit-127"        fix_exit_127
+
+echo ""
+if [ $FIXES_APPLIED -gt 0 ]; then
+    log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed"
+else
+    log "Done: all $CHECKS_PASSED checks passed — no fixes needed"
+fi
+
+exit 0

scripts/deploy-bitcoin-knots.sh

@@ -9,6 +9,16 @@
 
 set -e
 
+# Read per-installation Bitcoin RPC credentials
+SECRETS_DIR="/var/lib/archipelago/secrets"
+sudo mkdir -p "$SECRETS_DIR" && sudo chmod 700 "$SECRETS_DIR"
+if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
+    openssl rand -base64 24 | sudo tee "$SECRETS_DIR/bitcoin-rpc-password" > /dev/null
+    sudo chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
+fi
+BITCOIN_RPC_USER="archipelago"
+BITCOIN_RPC_PASS=$(sudo cat "$SECRETS_DIR/bitcoin-rpc-password")
+
 echo "╔════════════════════════════════════════════════════════════════╗"
 echo "║  Deploying Bitcoin Knots with Web UI                          ║"
 echo "╚════════════════════════════════════════════════════════════════╝"
@@ -44,7 +54,7 @@ sudo podman run -d \
   -rpcallowip=0.0.0.0/0 \
   -rpcbind=0.0.0.0:8332 \
   -rpcuser=archipelago \
-  -rpcpassword=archipelago123 \
+  -rpcpassword=$BITCOIN_RPC_PASS \
   -dbcache=4096
 
 echo "   ✅ Bitcoin Knots node starting"
@@ -115,7 +125,7 @@ echo "   • Network: Port 8333 (Bitcoin P2P)"
 echo ""
 echo "📝 RPC Credentials:"
 echo "   • User: archipelago"
-echo "   • Pass: archipelago123"
+echo "   • Pass: (stored in /var/lib/archipelago/secrets/bitcoin-rpc-password)"
 echo ""
 echo "⏰ Blockchain sync will take several hours to days."
 echo "   Check progress: sudo podman logs -f bitcoin-knots"

scripts/deploy-to-target.sh

@@ -734,6 +734,58 @@ MANIFEST_EOF
 
     # Bitcoin Knots: required for Mempool, ElectrumX, BTCPay, Fedimint
     TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
+
+    # Read per-installation Bitcoin RPC credentials from server secrets
+    progress "Reading Bitcoin RPC credentials"
+    BITCOIN_RPC_PASS=$(ssh $SSH_OPTS "$TARGET_HOST" '
+      SECRETS_DIR="/var/lib/archipelago/secrets"
+      sudo mkdir -p "$SECRETS_DIR" && sudo chmod 700 "$SECRETS_DIR"
+      if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
+          openssl rand -base64 24 | sudo tee "$SECRETS_DIR/bitcoin-rpc-password" > /dev/null
+          sudo chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
+      fi
+      sudo cat "$SECRETS_DIR/bitcoin-rpc-password"
+    ' 2>/dev/null)
+    BITCOIN_RPC_USER="archipelago"
+    if [ -z "$BITCOIN_RPC_PASS" ]; then
+        echo "   WARNING: Could not read Bitcoin RPC password from server, aborting container fixes"
+        return 1
+    fi
+
+    # Read per-installation database passwords from server secrets
+    DB_PASSWORDS=$(ssh $SSH_OPTS "$TARGET_HOST" '
+      SECRETS_DIR="/var/lib/archipelago/secrets"
+      for svc in mempool btcpay immich penpot mysql-root; do
+          if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
+              openssl rand -base64 24 | sudo tee "$SECRETS_DIR/${svc}-db-password" > /dev/null
+              sudo chmod 600 "$SECRETS_DIR/${svc}-db-password"
+          fi
+      done
+      echo "MEMPOOL_DB_PASS=$(sudo cat "$SECRETS_DIR/mempool-db-password")"
+      echo "BTCPAY_DB_PASS=$(sudo cat "$SECRETS_DIR/btcpay-db-password")"
+      echo "IMMICH_DB_PASS=$(sudo cat "$SECRETS_DIR/immich-db-password")"
+      echo "PENPOT_DB_PASS=$(sudo cat "$SECRETS_DIR/penpot-db-password")"
+      echo "MYSQL_ROOT_PASS=$(sudo cat "$SECRETS_DIR/mysql-root-db-password")"
+      # Fedimint gateway password and hash
+      if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
+          FEDI_PASS=$(openssl rand -base64 16)
+          echo "$FEDI_PASS" | sudo tee "$SECRETS_DIR/fedimint-gateway-password" > /dev/null
+          sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-password"
+          if command -v htpasswd >/dev/null 2>&1; then
+              htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ":\n" | sudo tee "$SECRETS_DIR/fedimint-gateway-hash" > /dev/null
+              sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
+          fi
+      fi
+      if [ -f "$SECRETS_DIR/fedimint-gateway-hash" ]; then
+          echo "FEDI_HASH=$(sudo cat "$SECRETS_DIR/fedimint-gateway-hash")"
+      fi
+    ' 2>/dev/null)
+    eval "$DB_PASSWORDS"
+    # Fallback if hash not available
+    if [ -z "$FEDI_HASH" ]; then
+        FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
+    fi
+
     progress "Ensuring Bitcoin Knots"
     ssh $SSH_OPTS "$TARGET_HOST" "
       DOCKER=podman
@@ -759,7 +811,7 @@ MANIFEST_EOF
           docker.io/bitcoinknots/bitcoin:latest \
           -server=1 \$BTC_EXTRA_ARGS \
           -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
-          -rpcuser=archipelago -rpcpassword=archipelago123 \
+          -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS \
           -dbcache=\$BTC_DBCACHE
         echo '   Bitcoin Knots started (sync may take hours)'
       else
@@ -788,8 +840,8 @@ MANIFEST_EOF
           -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
           -e MYSQL_DATABASE=mempool \
           -e MYSQL_USER=mempool \
-          -e MYSQL_PASSWORD=mempoolpass \
+          -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
-          -e MYSQL_ROOT_PASSWORD=rootpass \
+          -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
           docker.io/mariadb:10.11
         sleep 3
       fi
@@ -814,7 +866,7 @@ MANIFEST_EOF
           sudo \$DOCKER run -d --name electrumx --restart unless-stopped \$NET_OPT \
             -p 50001:50001 \
             -v /var/lib/archipelago/electrumx:/data \
-            -e DAEMON_URL=http://archipelago:archipelago123@bitcoin-knots:8332/ \
+            -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
             -e COIN=Bitcoin \
             -e DB_DIRECTORY=/data \
             -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
@@ -840,12 +892,12 @@ MANIFEST_EOF
         -e CORE_RPC_HOST=\$TARGET_IP \
         -e CORE_RPC_PORT=8332 \
         -e CORE_RPC_USERNAME=archipelago \
-        -e CORE_RPC_PASSWORD=archipelago123 \
+        -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
         -e DATABASE_ENABLED=true \
         -e DATABASE_HOST=\$MYSQL_CNT \
         -e DATABASE_DATABASE=mempool \
         -e DATABASE_USERNAME=mempool \
-        -e DATABASE_PASSWORD=mempoolpass \
+        -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
         docker.io/mempool/backend:v2.5.0
       fi
       # Recreate mempool frontend - handle both 'mempool' and 'mempool-web' (frontend was on wrong port 8999)
@@ -884,13 +936,13 @@ MANIFEST_EOF
           -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
           -e POSTGRES_DB=btcpay \
           -e POSTGRES_USER=btcpay \
-          -e POSTGRES_PASSWORD=btcpaypass \
+          -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
           docker.io/postgres:15-alpine
         sleep 3
       fi
       # Create NBXplorer database in PostgreSQL (NBXplorer needs its own DB)
       sudo \$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \
-        sudo \$DOCKER exec -e PGPASSWORD=btcpaypass archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true
+        sudo \$DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true
       # Create NBXplorer (required by BTCPay - indexes blocks for payment tracking)
       if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
         if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
@@ -907,8 +959,8 @@ MANIFEST_EOF
             -e NBXPLORER_BIND=0.0.0.0:32838 \
             -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
             -e NBXPLORER_BTCRPCUSER=archipelago \
-            -e NBXPLORER_BTCRPCPASSWORD=archipelago123 \
+            -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-            -e NBXPLORER_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
+            -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
             docker.io/nicolasdorier/nbxplorer:2.6.0
           sleep 5
         fi
@@ -936,8 +988,8 @@ MANIFEST_EOF
           -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
           -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
           -e BTCPAY_BTCRPCUSER=archipelago \
-          -e BTCPAY_BTCRPCPASSWORD=archipelago123 \
+          -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-          -e BTCPAY_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
+          -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
           docker.io/btcpayserver/btcpayserver:1.13.5
       fi
     " 2>&1 | sed 's/^/   /' || true
@@ -961,7 +1013,7 @@ MANIFEST_EOF
         if ! sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
           sudo \$DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
             -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
-            -e POSTGRES_PASSWORD=immichpass -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
+            -e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
             ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>/dev/null || true
           sleep 5
         fi
@@ -973,7 +1025,7 @@ MANIFEST_EOF
         if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
           sudo \$DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
             -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
-            -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=immichpass \
+            -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
             -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
             -e UPLOAD_LOCATION=/usr/src/app/upload \
             ghcr.io/immich-app/immich-server:release 2>/dev/null || true
@@ -1098,7 +1150,7 @@ print("torrc generated with %d services" % (len(lines) // 3))
           -v /var/lib/archipelago/fedimint:/data \
           -e FM_DATA_DIR=/data \
           -e FM_BITCOIND_USERNAME=archipelago \
-          -e FM_BITCOIND_PASSWORD=archipelago123 \
+          -e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \
           -e FM_BITCOIN_NETWORK=bitcoin \
           -e FM_BIND_P2P=0.0.0.0:8173 \
           -e FM_BIND_API=0.0.0.0:8174 \
@@ -1117,7 +1169,7 @@ print("torrc generated with %d services" % (len(lines) // 3))
       sudo mkdir -p /var/lib/archipelago/fedimint-gateway
       LND_CERT=/var/lib/archipelago/lnd/tls.cert
       LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
-      GW_COMMON=\"-p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data docker.io/fedimint/gatewayd:v0.10.0 gatewayd --data-dir /data --listen 0.0.0.0:8176 --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' --network bitcoin --bitcoind-url http://$TARGET_IP:8332 --bitcoind-username archipelago --bitcoind-password archipelago123\"
+      GW_COMMON=\"-p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data docker.io/fedimint/gatewayd:v0.10.0 gatewayd --data-dir /data --listen 0.0.0.0:8176 --bcrypt-password-hash '$FEDI_HASH' --network bitcoin --bitcoind-url http://$TARGET_IP:8332 --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS\"
       if sudo \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then
         echo '   LND detected — using lnd mode'
         sudo \$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
@@ -1129,9 +1181,9 @@ print("torrc generated with %d services" % (len(lines) // 3))
           -v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \
           docker.io/fedimint/gatewayd:v0.10.0 \
           gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-          --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
+          --bcrypt-password-hash '$FEDI_HASH' \
           --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
-          --bitcoind-username archipelago --bitcoind-password archipelago123 \
+          --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
           lnd --lnd-rpc-host $TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon
       else
         echo '   No LND found — using ldk (built-in Lightning)'
@@ -1142,9 +1194,9 @@ print("torrc generated with %d services" % (len(lines) // 3))
           -v /var/lib/archipelago/fedimint-gateway:/data \
           docker.io/fedimint/gatewayd:v0.10.0 \
           gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-          --bcrypt-password-hash '\$2y\$10\$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
+          --bcrypt-password-hash '$FEDI_HASH' \
           --network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
-          --bitcoind-username archipelago --bitcoind-password archipelago123 \
+          --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
           ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway
       fi
     " 2>&1 | sed 's/^/   /') || echo "   (Fedimint fix timed out or skipped - run manually if needed)"
@@ -1179,7 +1231,7 @@ bitcoin.node=bitcoind
 [Bitcoind]
 bitcoind.rpchost=bitcoin-knots:8332
 bitcoind.rpcuser=archipelago
-bitcoind.rpcpass=archipelago123
+bitcoind.rpcpass=$BITCOIN_RPC_PASS
 bitcoind.rpcpolling=true
 bitcoind.estimatemode=ECONOMICAL
 

scripts/first-boot-containers.sh

@@ -35,6 +35,56 @@ wait_for_container() {
   return 1
 }
 
+# Generate per-installation credentials if not already saved
+SECRETS_DIR="/var/lib/archipelago/secrets"
+mkdir -p "$SECRETS_DIR" && chmod 700 "$SECRETS_DIR"
+if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
+    openssl rand -base64 24 > "$SECRETS_DIR/bitcoin-rpc-password"
+    chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
+fi
+BITCOIN_RPC_USER="archipelago"
+BITCOIN_RPC_PASS=$(cat "$SECRETS_DIR/bitcoin-rpc-password")
+
+# Generate per-installation database passwords if not already saved
+for svc in mempool btcpay immich penpot mysql-root; do
+    if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
+        openssl rand -base64 24 > "$SECRETS_DIR/${svc}-db-password"
+        chmod 600 "$SECRETS_DIR/${svc}-db-password"
+    fi
+done
+MEMPOOL_DB_PASS=$(cat "$SECRETS_DIR/mempool-db-password")
+BTCPAY_DB_PASS=$(cat "$SECRETS_DIR/btcpay-db-password")
+IMMICH_DB_PASS=$(cat "$SECRETS_DIR/immich-db-password")
+PENPOT_DB_PASS=$(cat "$SECRETS_DIR/penpot-db-password")
+MYSQL_ROOT_PASS=$(cat "$SECRETS_DIR/mysql-root-db-password")
+
+# Generate Fedimint gateway password and bcrypt hash
+if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
+    FEDI_PASS=$(openssl rand -base64 16)
+    echo "$FEDI_PASS" > "$SECRETS_DIR/fedimint-gateway-password"
+    chmod 600 "$SECRETS_DIR/fedimint-gateway-password"
+    # Pre-compute bcrypt hash (requires htpasswd from apache2-utils)
+    if command -v htpasswd >/dev/null 2>&1; then
+        htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ':\n' > "$SECRETS_DIR/fedimint-gateway-hash"
+        chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
+    fi
+fi
+FEDI_PASS=$(cat "$SECRETS_DIR/fedimint-gateway-password")
+if [ -f "$SECRETS_DIR/fedimint-gateway-hash" ]; then
+    FEDI_HASH=$(cat "$SECRETS_DIR/fedimint-gateway-hash")
+else
+    # Fallback: generate hash now
+    if command -v htpasswd >/dev/null 2>&1; then
+        FEDI_HASH=$(htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ':\n')
+        echo "$FEDI_HASH" > "$SECRETS_DIR/fedimint-gateway-hash"
+        chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
+    else
+        log "WARNING: htpasswd not found, using default Fedimint gateway hash"
+        FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
+    fi
+fi
+log "Fedimint gateway password stored in $SECRETS_DIR/fedimint-gateway-password"
+
 log "First-boot container creation starting (host=$TARGET_IP)"
 
 # Create swap file if not present (50% of RAM, min 2GB, max 8GB)
@@ -88,7 +138,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|arch
     docker.io/bitcoinknots/bitcoin:latest \
     -server=1 $BTC_EXTRA_ARGS \
     -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
-    -rpcuser=archipelago -rpcpassword=archipelago123 \
+    -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS \
     -dbcache=$BTC_DBCACHE 2>>"$LOG"; then
     log "Bitcoin Knots started"
   else
@@ -99,12 +149,12 @@ else
   log "Bitcoin Knots already running"
 fi
 # Wait for Bitcoin Knots RPC to be responsive (LND, NBXplorer, mempool depend on it)
-wait_for_container "Bitcoin Knots RPC" "$DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=archipelago -rpcpassword=archipelago123 getblockchaininfo" 60
+wait_for_container "Bitcoin Knots RPC" "$DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS getblockchaininfo" 60
 
 # Ensure wallet exists (Bitcoin Knots no longer auto-creates a default wallet)
-if ! $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=archipelago -rpcpassword=archipelago123 listwallets 2>/dev/null | grep -q "archipelago"; then
+if ! $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS listwallets 2>/dev/null | grep -q "archipelago"; then
-  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=archipelago -rpcpassword=archipelago123 loadwallet "archipelago" 2>/dev/null || \
+  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS loadwallet "archipelago" 2>/dev/null || \
-  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=archipelago -rpcpassword=archipelago123 createwallet "archipelago" 2>/dev/null
+  $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS createwallet "archipelago" 2>/dev/null
   log "Bitcoin Knots wallet 'archipelago' created/loaded"
 fi
 
@@ -114,10 +164,10 @@ if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-d
   mkdir -p /var/lib/archipelago/mysql-mempool
   $DOCKER run -d --name archy-mempool-db --restart unless-stopped --network archy-net \
     -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
-    -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool -e MYSQL_PASSWORD=mempoolpass \
+    -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
-    -e MYSQL_ROOT_PASSWORD=rootpass \
+    -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
     docker.io/mariadb:10.11 2>>"$LOG" || true
-  wait_for_container "Mempool MariaDB" "$DOCKER exec archy-mempool-db mariadb -uroot -prootpass -e 'SELECT 1'" 30
+  wait_for_container "Mempool MariaDB" "$DOCKER exec archy-mempool-db mariadb -uroot -p$MYSQL_ROOT_PASS -e 'SELECT 1'" 30
 fi
 MYSQL_CNT=$($DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
 MYSQL_CNT=${MYSQL_CNT:-archy-mempool-db}
@@ -131,7 +181,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
     mkdir -p /var/lib/archipelago/electrumx
     $DOCKER run -d --name electrumx --restart unless-stopped --network archy-net \
       -p 50001:50001 -v /var/lib/archipelago/electrumx:/data \
-      -e DAEMON_URL=http://archipelago:archipelago123@bitcoin-knots:8332/ \
+      -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
       -e COIN=Bitcoin -e DB_DIRECTORY=/data \
       -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
       docker.io/lukechilds/electrumx:v1.18.0 2>>"$LOG" || true
@@ -145,9 +195,9 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
     -p 8999:8999 -v /var/lib/archipelago/mempool:/data \
     -e MEMPOOL_BACKEND=electrum -e ELECTRUM_HOST=electrumx -e ELECTRUM_PORT=50001 \
     -e ELECTRUM_TLS_ENABLED=false -e CORE_RPC_HOST="$TARGET_IP" -e CORE_RPC_PORT=8332 \
-    -e CORE_RPC_USERNAME=archipelago -e CORE_RPC_PASSWORD=archipelago123 \
+    -e CORE_RPC_USERNAME=$BITCOIN_RPC_USER -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
     -e DATABASE_ENABLED=true -e DATABASE_HOST="$MYSQL_CNT" -e DATABASE_DATABASE=mempool \
-    -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=mempoolpass \
+    -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
     docker.io/mempool/backend:v2.5.0 2>>"$LOG" || true
 fi
 
@@ -182,14 +232,14 @@ if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db
   mkdir -p /var/lib/archipelago/postgres-btcpay
   $DOCKER run -d --name archy-btcpay-db --restart unless-stopped --network archy-net \
     -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
-    -e POSTGRES_DB=btcpay -e POSTGRES_USER=btcpay -e POSTGRES_PASSWORD=btcpaypass \
+    -e POSTGRES_DB=btcpay -e POSTGRES_USER=btcpay -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
     docker.io/postgres:15-alpine 2>>"$LOG" || true
   wait_for_container "BTCPay PostgreSQL" "$DOCKER exec archy-btcpay-db pg_isready -U postgres" 30
 fi
 # Create nbxplorer DB only if postgres is running
 if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
   $DOCKER exec archy-btcpay-db psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='nbxplorer'" 2>/dev/null | grep -q 1 || \
-    $DOCKER exec -e PGPASSWORD=btcpaypass archy-btcpay-db psql -U postgres -c "CREATE DATABASE nbxplorer;" 2>/dev/null || true
+    $DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c "CREATE DATABASE nbxplorer;" 2>/dev/null || true
 fi
 
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
@@ -202,8 +252,8 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; the
       -p 32838:32838 -v /var/lib/archipelago/nbxplorer:/data \
       -e NBXPLORER_DATADIR=/data -e NBXPLORER_NETWORK=mainnet -e NBXPLORER_CHAINS=btc \
       -e NBXPLORER_BIND=0.0.0.0:32838 -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
-      -e NBXPLORER_BTCRPCUSER=archipelago -e NBXPLORER_BTCRPCPASSWORD=archipelago123 \
+      -e NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-      -e NBXPLORER_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
+      -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
       docker.io/nicolasdorier/nbxplorer:2.6.0 2>>"$LOG" && sleep 5 || true
   fi
 fi
@@ -219,8 +269,8 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
     -e BTCPAY_HOST="$TARGET_IP:23000" -e BTCPAY_CHAINS=btc \
     -e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
     -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
-    -e BTCPAY_BTCRPCUSER=archipelago -e BTCPAY_BTCRPCPASSWORD=archipelago123 \
+    -e BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-    -e BTCPAY_POSTGRES='User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
+    -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
     docker.io/btcpayserver/btcpayserver:1.13.5 2>>"$LOG" || true
 fi
 
@@ -234,7 +284,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE '^lnd$'; then
   mkdir -p /var/lib/archipelago/lnd
   # Create lnd.conf so LND auto-connects to Bitcoin Knots via archy-net
   if [ ! -f /var/lib/archipelago/lnd/lnd.conf ]; then
-    cat > /var/lib/archipelago/lnd/lnd.conf <<'LNDCONF'
+    cat > /var/lib/archipelago/lnd/lnd.conf <<LNDCONF
 [Application Options]
 listen=0.0.0.0:9735
 rpclisten=0.0.0.0:10009
@@ -250,7 +300,7 @@ bitcoin.node=bitcoind
 [Bitcoind]
 bitcoind.rpchost=bitcoin-knots:8332
 bitcoind.rpcuser=archipelago
-bitcoind.rpcpass=archipelago123
+bitcoind.rpcpass=$BITCOIN_RPC_PASS
 bitcoind.rpcpolling=true
 bitcoind.estimatemode=ECONOMICAL
 
@@ -276,7 +326,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint; then
     --security-opt no-new-privileges:true \
     -p 8173:8173 -p 8174:8174 -p 8175:8175 \
     -v /var/lib/archipelago/fedimint:/data \
-    -e FM_DATA_DIR=/data -e FM_BITCOIND_USERNAME=archipelago -e FM_BITCOIND_PASSWORD=archipelago123 \
+    -e FM_DATA_DIR=/data -e FM_BITCOIND_USERNAME=$BITCOIN_RPC_USER -e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \
     -e FM_BITCOIN_NETWORK=bitcoin -e FM_BIND_P2P=0.0.0.0:8173 \
     -e FM_BIND_API=0.0.0.0:8174 -e FM_BIND_UI=0.0.0.0:8175 \
     -e FM_P2P_URL=fedimint://"$TARGET_IP":8173 -e FM_API_URL=ws://"$TARGET_IP":8174 \
@@ -302,9 +352,9 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; th
       -v "$LND_MACAROON":/lnd/admin.macaroon:ro \
       docker.io/fedimint/gatewayd:v0.10.0 \
       gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-      --bcrypt-password-hash '$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
+      --bcrypt-password-hash "$FEDI_HASH" \
       --network bitcoin --bitcoind-url http://"$TARGET_IP":8332 \
-      --bitcoind-username archipelago --bitcoind-password archipelago123 \
+      --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
       lnd --lnd-rpc-host "$TARGET_IP":10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon 2>>"$LOG" || true
   else
     log "  No LND found — using ldk (built-in Lightning)"
@@ -315,9 +365,9 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; th
       -v /var/lib/archipelago/fedimint-gateway:/data \
       docker.io/fedimint/gatewayd:v0.10.0 \
       gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-      --bcrypt-password-hash '$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC' \
+      --bcrypt-password-hash "$FEDI_HASH" \
       --network bitcoin --bitcoind-url http://"$TARGET_IP":8332 \
-      --bitcoind-username archipelago --bitcoind-password archipelago123 \
+      --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
       ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway 2>>"$LOG" || true
   fi
 fi
@@ -482,7 +532,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
   if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
     $DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
       -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
-      -e POSTGRES_PASSWORD=immichpass -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
+      -e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
       ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>>"$LOG" || true
     sleep 3
     for i in 1 2 3 4 5 6 7 8 9 10; do
@@ -498,7 +548,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
   if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
     $DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
       -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
-      -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=immichpass \
+      -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
       -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
       -e UPLOAD_LOCATION=/usr/src/app/upload \
       ghcr.io/immich-app/immich-server:release 2>>"$LOG" || true
@@ -513,7 +563,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; the
   if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q penpot-postgres; then
     $DOCKER run -d --name penpot-postgres --restart unless-stopped --network penpot-net \
       -v /var/lib/archipelago/penpot-postgres:/var/lib/postgresql/data \
-      -e POSTGRES_DB=penpot -e POSTGRES_USER=penpot -e POSTGRES_PASSWORD=penpot \
+      -e POSTGRES_DB=penpot -e POSTGRES_USER=penpot -e POSTGRES_PASSWORD=$PENPOT_DB_PASS \
       docker.io/postgres:15 2>>"$LOG" || true
     sleep 5
   fi
@@ -529,7 +579,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; the
       -e PENPOT_PUBLIC_URI="http://${TARGET_IP}:9001" \
       -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
       -e PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot \
-      -e PENPOT_DATABASE_USERNAME=penpot -e PENPOT_DATABASE_PASSWORD=penpot \
+      -e PENPOT_DATABASE_USERNAME=penpot -e PENPOT_DATABASE_PASSWORD=$PENPOT_DB_PASS \
       -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
       -e PENPOT_OBJECTS_STORAGE_BACKEND=fs \
       -e PENPOT_OBJECTS_STORAGE_FS_DIRECTORY=/opt/data/assets \