diff --git a/CHANGELOG.md b/CHANGELOG.md index 882357c2..7da99031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v1.7.49-alpha (2026-04-30) + +- Bitcoin Knots/Core UI now reports connection, reconnecting, syncing, and error states from a backend status bridge instead of showing a stale "Unable to connect" message while the node is warming up. +- ElectrumX UI now exposes indexed height, local Bitcoin height, known headers, status, and progress source so indexing/waiting states are readable during long initial sync. +- Added container doctor timer and smoke/lifecycle test coverage for Bitcoin Knots/Core, ElectrumX, Mempool, BTCPay/NBXplorer, and UI surface availability. +- Bitcoin Core and Bitcoin Knots are mutually exclusive variants, with a real Bitcoin Core manifest and corrected install conflict handling. +- IndeeHub now launches only on direct web UI port `7778`; the broken `/app/indeedhub/` path proxy was removed, and port `7777` remains the Nostr relay. +- BTCPay/NBXplorer Postgres environment formatting fixed so installs do not carry malformed connection strings. + ## v1.7.48-alpha (2026-04-29) - archipelago.service no longer fails to start with "Failed to set up mount namespacing: /run/containers: No such file or directory" on nodes where /run/containers wasn't pre-created. ExecStartPre now creates it. Existing nodes need a one-time `systemctl edit archipelago` to add the mkdir; ISO installs from this version forward have the fix baked in. diff --git a/app-catalog/catalog.json b/app-catalog/catalog.json index 5fee9801..3db1472e 100644 --- a/app-catalog/catalog.json +++ b/app-catalog/catalog.json @@ -26,7 +26,7 @@ "id": "bitcoin-core", "title": "Bitcoin Core", "version": "28.4", - "description": "Reference implementation of the Bitcoin protocol. Run a full node validating and relaying blocks.", + "description": "Reference Bitcoin node implementation. Alternative to Bitcoin Knots; uninstall Knots before switching.", "icon": "/assets/img/app-icons/bitcoin-core.svg", "author": "Bitcoin Core contributors", "category": "money", diff --git a/apps/archy-nbxplorer/manifest.yml b/apps/archy-nbxplorer/manifest.yml index c4ddc5e9..0705cae5 100644 --- a/apps/archy-nbxplorer/manifest.yml +++ b/apps/archy-nbxplorer/manifest.yml @@ -47,7 +47,7 @@ app: - NBXPLORER_BIND=0.0.0.0:32838 - NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 - NBXPLORER_BTCRPCUSER=archipelago - - NBXPLORER_POSTGRES=User ID=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true + - NBXPLORER_POSTGRES=Username=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=nbxplorer health_check: type: http diff --git a/apps/bitcoin-core/manifest.yml b/apps/bitcoin-core/manifest.yml index 8abd96ed..69944ef4 100644 --- a/apps/bitcoin-core/manifest.yml +++ b/apps/bitcoin-core/manifest.yml @@ -1,13 +1,13 @@ app: id: bitcoin-core - name: Bitcoin Knots + name: Bitcoin Core version: 28.4.0 - description: Full Bitcoin Knots node with dynamic prune/full-mode startup based on host disk. + description: Reference Bitcoin Core node with dynamic prune/full-mode startup based on host disk. - container_name: bitcoin-knots + container_name: bitcoin-core container: - image: 146.59.87.168:3000/lfg2025/bitcoin-knots:latest + image: 146.59.87.168:3000/lfg2025/bitcoin:28.4 pull_policy: if-not-present network: archy-net entrypoint: ["sh", "-lc"] diff --git a/apps/bitcoin-knots/manifest.yml b/apps/bitcoin-knots/manifest.yml new file mode 100644 index 00000000..089e32ca --- /dev/null +++ b/apps/bitcoin-knots/manifest.yml @@ -0,0 +1,75 @@ +app: + id: bitcoin-knots + name: Bitcoin Knots + version: 28.1.0 + description: Full Bitcoin Knots node with dynamic prune/full-mode startup based on host disk. + + container_name: bitcoin-knots + + container: + image: 146.59.87.168:3000/lfg2025/bitcoin-knots:latest + pull_policy: if-not-present + network: archy-net + entrypoint: ["sh", "-lc"] + custom_args: + # Sync-speed flags: -par=0 uses every core (was capped at 2 by + # --cpus=2, now removed for bitcoin/electrumx). -dbcache sized to + # the IBD sweet spot — 4GB on full nodes, 1GB on pruned. Container + # --memory=8g (config.rs::get_memory_limit) leaves headroom for + # mempool + connections. + - >- + if [ "${DISK_GB:-0}" -lt 1000 ]; then + exec bitcoind -server=1 -prune=550 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 -listen=1 -bind=0.0.0.0:8333 -dbcache=1024 -par=0 -maxconnections=125 -rpcuser="${BITCOIN_RPC_USER}" -rpcpassword="${BITCOIN_RPC_PASS}"; + else + exec bitcoind -server=1 -txindex=1 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 -listen=1 -bind=0.0.0.0:8333 -dbcache=4096 -par=0 -maxconnections=125 -rpcuser="${BITCOIN_RPC_USER}" -rpcpassword="${BITCOIN_RPC_PASS}"; + fi + derived_env: + - key: DISK_GB + template: "{{DISK_GB}}" + secret_env: + - key: BITCOIN_RPC_PASS + secret_file: bitcoin-rpc-password + data_uid: "100101:100101" + + dependencies: + - storage: 500Gi + + resources: + cpu_limit: 0 + memory_limit: 4Gi + disk_limit: 500Gi + + security: + capabilities: [CHOWN, FOWNER, SETUID, SETGID, DAC_OVERRIDE] + readonly_root: false + network_policy: isolated + + ports: + - host: 8332 + container: 8332 + protocol: tcp + - host: 8333 + container: 8333 + protocol: tcp + + volumes: + - type: bind + source: /var/lib/archipelago/bitcoin + target: /home/bitcoin/.bitcoin + options: [rw] + + environment: + - BITCOIN_RPC_USER=archipelago + + health_check: + type: tcp + endpoint: localhost:8332 + interval: 30s + timeout: 5s + retries: 3 + + bitcoin_integration: + rpc_access: admin + sync_required: true + testnet_support: false + pruning_support: true diff --git a/apps/btcpay-server/manifest.yml b/apps/btcpay-server/manifest.yml index dc0fba34..93449482 100644 --- a/apps/btcpay-server/manifest.yml +++ b/apps/btcpay-server/manifest.yml @@ -51,7 +51,7 @@ app: - BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 - BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 - BTCPAY_BTCRPCUSER=archipelago - - BTCPAY_POSTGRES=User ID=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true + - BTCPAY_POSTGRES=Username=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=btcpay health_check: type: http diff --git a/apps/indeedhub/manifest.yml b/apps/indeedhub/manifest.yml index 59fee6ff..4ccbfb88 100644 --- a/apps/indeedhub/manifest.yml +++ b/apps/indeedhub/manifest.yml @@ -27,9 +27,9 @@ app: apparmor_profile: default ports: - - host: 7777 - container: 3000 - protocol: tcp # Web UI (Next.js) + - host: 7778 + container: 7777 + protocol: tcp # Web UI. Port 7777 on the host is reserved for Nostr relay. volumes: - type: tmpfs @@ -57,7 +57,7 @@ app: name: Web UI description: Stream Bitcoin documentaries with Nostr identity type: ui - port: 7777 + port: 7778 protocol: http path: / diff --git a/core/Cargo.lock b/core/Cargo.lock index e99b97b3..7109daca 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "archipelago" -version = "1.7.48-alpha" +version = "1.7.49-alpha" dependencies = [ "anyhow", "archipelago-container", diff --git a/core/archipelago/Cargo.toml b/core/archipelago/Cargo.toml index 5d4c4d2a..f3d852af 100644 --- a/core/archipelago/Cargo.toml +++ b/core/archipelago/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "archipelago" -version = "1.7.48-alpha" +version = "1.7.49-alpha" edition = "2021" description = "Archipelago Bitcoin Node OS - Native backend" authors = ["Archipelago Team"] diff --git a/core/archipelago/src/api/handler/mod.rs b/core/archipelago/src/api/handler/mod.rs index fbb61b92..5ca04f4a 100644 --- a/core/archipelago/src/api/handler/mod.rs +++ b/core/archipelago/src/api/handler/mod.rs @@ -429,6 +429,7 @@ impl ApiHandler { // Electrs status — unauthenticated (read-only sync status) (Method::GET, "/electrs-status") => Self::handle_electrs_status().await, + (Method::GET, "/bitcoin-status") => Self::handle_bitcoin_status().await, // App-catalog proxy — fetches catalog.json from the configured // upstream URLs server-side so the browser doesn't hit CORS diff --git a/core/archipelago/src/api/handler/proxy.rs b/core/archipelago/src/api/handler/proxy.rs index cd299c9f..eb1b5e2d 100644 --- a/core/archipelago/src/api/handler/proxy.rs +++ b/core/archipelago/src/api/handler/proxy.rs @@ -1,5 +1,6 @@ use super::build_response; use crate::api::rpc::RpcHandler; +use crate::bitcoin_status; use crate::electrs_status; use anyhow::Result; use hyper::{Response, StatusCode}; @@ -76,11 +77,23 @@ impl ApiHandler { pub(super) async fn handle_electrs_status() -> Result> { let status = electrs_status::get_electrs_sync_status().await; let body = serde_json::to_vec(&status).unwrap_or_default(); - Ok(build_response( - StatusCode::OK, - "application/json", - hyper::Body::from(body), - )) + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .header("Cache-Control", "no-store") + .body(hyper::Body::from(body)) + .unwrap_or_else(|_| Response::new(hyper::Body::from("{}")))) + } + + pub(super) async fn handle_bitcoin_status() -> Result> { + let status = bitcoin_status::get_bitcoin_status().await; + let body = serde_json::to_vec(&status).unwrap_or_default(); + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .header("Cache-Control", "no-store") + .body(hyper::Body::from(body)) + .unwrap_or_else(|_| Response::new(hyper::Body::from("{}")))) } pub(super) async fn handle_lnd_connect_info( diff --git a/core/archipelago/src/api/rpc/package/install.rs b/core/archipelago/src/api/rpc/package/install.rs index 5c485e93..c38e32f3 100644 --- a/core/archipelago/src/api/rpc/package/install.rs +++ b/core/archipelago/src/api/rpc/package/install.rs @@ -229,6 +229,7 @@ impl RpcHandler { let deps = detect_running_deps().await?; check_install_deps(package_id, &deps)?; log_optional_dep_info(package_id, &deps); + check_bitcoin_implementation_conflict(package_id).await?; // Check if container already exists let check_output = tokio::process::Command::new("podman") @@ -1961,9 +1962,51 @@ fn should_try_orchestrator_install(package_id: &str, orchestrator_available: boo orchestrator_available && uses_orchestrator_install_flow(package_id) } +async fn check_bitcoin_implementation_conflict(package_id: &str) -> Result<()> { + let other = match package_id { + "bitcoin-core" => "bitcoin-knots", + "bitcoin-knots" => "bitcoin-core", + _ => return Ok(()), + }; + + let output = tokio::process::Command::new("podman") + .args([ + "ps", + "-a", + "--format", + "{{.Names}}", + "--filter", + &format!("name=^{}$", other), + ]) + .output() + .await + .context("Failed to check existing Bitcoin node containers")?; + + if String::from_utf8_lossy(&output.stdout).trim().is_empty() { + return Ok(()); + } + + let current = match other { + "bitcoin-core" => "Bitcoin Core", + "bitcoin-knots" => "Bitcoin Knots", + _ => "another Bitcoin node", + }; + let requested = match package_id { + "bitcoin-core" => "Bitcoin Core", + "bitcoin-knots" => "Bitcoin Knots", + _ => "the requested Bitcoin node", + }; + + Err(anyhow::anyhow!( + "{} is already installed. Stop and uninstall {} before installing {}; both implementations use the same Bitcoin data directory and ports.", + current, + current, + requested + )) +} + fn orchestrator_install_app_id(package_id: &str) -> &str { match package_id { - "bitcoin-knots" => "bitcoin-core", "electrs" | "mempool-electrs" => "electrumx", _ => package_id, } @@ -2049,7 +2092,8 @@ mod tests { #[test] fn install_aliases_map_to_manifest_app_ids() { - assert_eq!(orchestrator_install_app_id("bitcoin-knots"), "bitcoin-core"); + assert_eq!(orchestrator_install_app_id("bitcoin-knots"), "bitcoin-knots"); + assert_eq!(orchestrator_install_app_id("bitcoin-core"), "bitcoin-core"); assert_eq!(orchestrator_install_app_id("electrs"), "electrumx"); assert_eq!(orchestrator_install_app_id("mempool-electrs"), "electrumx"); assert_eq!(orchestrator_install_app_id("lnd"), "lnd"); diff --git a/core/archipelago/src/api/rpc/tor/mod.rs b/core/archipelago/src/api/rpc/tor/mod.rs index 05903c6e..77bed2e3 100644 --- a/core/archipelago/src/api/rpc/tor/mod.rs +++ b/core/archipelago/src/api/rpc/tor/mod.rs @@ -355,7 +355,7 @@ pub(in crate::api::rpc) fn known_service_port(name: &str) -> u16 { "penpot" => 9001, "nginx-proxy-manager" => 81, "vaultwarden" => 8343, - "indeedhub" => 7777, + "indeedhub" => 7778, _ => 0, } } diff --git a/core/archipelago/src/bitcoin_status.rs b/core/archipelago/src/bitcoin_status.rs new file mode 100644 index 00000000..d4ecf282 --- /dev/null +++ b/core/archipelago/src/bitcoin_status.rs @@ -0,0 +1,186 @@ +//! Cached Bitcoin node status for browser UIs. +//! +//! The bitcoin-ui should not poll Bitcoin RPC directly for display state. +//! During container restarts, reindexing, and IBD, direct browser RPC polling +//! turns short RPC gaps into visible UI failures. This module owns the RPC +//! polling loop, caches the last successful snapshot, and serves stale-but-known +//! state while the node is reconnecting. + +use anyhow::{Context, Result}; +use serde::Serialize; +use std::sync::OnceLock; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::RwLock; +use tracing::{debug, warn}; + +const CACHE_REFRESH_SECS: u64 = 5; + +#[derive(Debug, Clone, Serialize)] +pub struct BitcoinNodeStatus { + pub ok: bool, + pub stale: bool, + pub updated_at_ms: u64, + pub error: Option, + pub blockchain_info: Option, + pub network_info: Option, + pub index_info: Option, + pub zmq_notifications: Option, +} + +impl Default for BitcoinNodeStatus { + fn default() -> Self { + Self { + ok: false, + stale: false, + updated_at_ms: 0, + error: Some("Connecting to Bitcoin node...".to_string()), + blockchain_info: None, + network_info: None, + index_info: None, + zmq_notifications: None, + } + } +} + +static STATUS_CACHE: OnceLock> = OnceLock::new(); + +fn cache() -> &'static RwLock { + STATUS_CACHE.get_or_init(|| RwLock::new(BitcoinNodeStatus::default())) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +fn transient_error(err_msg: &str) -> bool { + let lower = err_msg.to_lowercase(); + lower.contains("connect") + || lower.contains("reset") + || lower.contains("refused") + || lower.contains("timed out") + || lower.contains("timeout") + || lower.contains("broken pipe") + || lower.contains("eof") + || lower.contains("500 internal server error") +} + +pub fn spawn_status_cache() { + tokio::spawn(async { + loop { + let fresh = fetch_bitcoin_status().await; + let mut cached = cache().write().await; + match fresh { + Ok(mut status) => { + status.ok = true; + status.stale = false; + status.error = None; + *cached = status; + } + Err(e) => { + let err_msg = e.to_string(); + if transient_error(&err_msg) { + debug!("Bitcoin status: transient RPC failure: {}", err_msg); + } else { + warn!("Bitcoin status: RPC failure: {}", err_msg); + } + + if cached.blockchain_info.is_some() { + cached.ok = false; + cached.stale = true; + cached.error = Some(format!( + "Bitcoin node is reconnecting; showing last known state: {}", + err_msg + )); + } else { + *cached = BitcoinNodeStatus { + ok: false, + stale: false, + updated_at_ms: now_ms(), + error: Some(format!("Connecting to Bitcoin node: {}", err_msg)), + ..BitcoinNodeStatus::default() + }; + } + } + } + drop(cached); + tokio::time::sleep(Duration::from_secs(CACHE_REFRESH_SECS)).await; + } + }); +} + +pub async fn get_bitcoin_status() -> BitcoinNodeStatus { + cache().read().await.clone() +} + +async fn fetch_bitcoin_status() -> Result { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(8)) + .build() + .context("build Bitcoin status HTTP client")?; + + let blockchain_info = bitcoin_rpc_call(&client, "getblockchaininfo", serde_json::json!([])) + .await + .context("getblockchaininfo")?; + let network_info = bitcoin_rpc_call(&client, "getnetworkinfo", serde_json::json!([])) + .await + .context("getnetworkinfo") + .ok(); + let index_info = bitcoin_rpc_call(&client, "getindexinfo", serde_json::json!([])) + .await + .context("getindexinfo") + .ok(); + let zmq_notifications = + bitcoin_rpc_call(&client, "getzmqnotifications", serde_json::json!([])) + .await + .context("getzmqnotifications") + .ok(); + + Ok(BitcoinNodeStatus { + ok: true, + stale: false, + updated_at_ms: now_ms(), + error: None, + blockchain_info: Some(blockchain_info), + network_info, + index_info, + zmq_notifications, + }) +} + +async fn bitcoin_rpc_call( + client: &reqwest::Client, + method: &str, + params: serde_json::Value, +) -> Result { + let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await; + let body = serde_json::json!({ + "jsonrpc": "1.0", + "id": "bitcoin-status", + "method": method, + "params": params, + }); + + let resp = client + .post(crate::constants::BITCOIN_RPC_URL) + .basic_auth(rpc_user, Some(rpc_pass)) + .header("Content-Type", "application/json") + .json(&body) + .send() + .await + .context("Bitcoin RPC request failed")?; + + let status = resp.status(); + let json: serde_json::Value = resp.json().await.context("decode Bitcoin RPC JSON")?; + if !status.is_success() { + anyhow::bail!("Bitcoin RPC returned {}: {}", status, json); + } + if let Some(error) = json.get("error").filter(|e| !e.is_null()) { + anyhow::bail!("Bitcoin RPC {} error: {}", method, error); + } + json.get("result") + .cloned() + .context("missing Bitcoin RPC result") +} diff --git a/core/archipelago/src/container/bitcoin_ui_nginx.conf.template b/core/archipelago/src/container/bitcoin_ui_nginx.conf.template index 97db8a08..7ab57dd8 100644 --- a/core/archipelago/src/container/bitcoin_ui_nginx.conf.template +++ b/core/archipelago/src/container/bitcoin_ui_nginx.conf.template @@ -15,5 +15,13 @@ server { add_header Access-Control-Allow-Headers "Content-Type, Authorization"; if ($request_method = OPTIONS) { return 204; } } + location /bitcoin-status { + proxy_pass http://127.0.0.1:5678/bitcoin-status; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + add_header Cache-Control "no-store"; + } location / { try_files $uri $uri/ /index.html; } } diff --git a/core/archipelago/src/electrs_status.rs b/core/archipelago/src/electrs_status.rs index 875957c7..ecc6662c 100644 --- a/core/archipelago/src/electrs_status.rs +++ b/core/archipelago/src/electrs_status.rs @@ -30,9 +30,11 @@ async fn bitcoin_rpc_auth() -> String { #[derive(Debug, Clone, Serialize)] pub struct ElectrsSyncStatus { pub indexed_height: u64, + pub bitcoin_height: u64, pub network_height: u64, pub progress_pct: f64, pub status: String, + pub stale: bool, pub error: Option, /// Index data size in human-readable format (e.g. "11.2 GB") pub index_size: Option, @@ -44,9 +46,11 @@ impl Default for ElectrsSyncStatus { fn default() -> Self { Self { indexed_height: 0, + bitcoin_height: 0, network_height: 0, progress_pct: 0.0, status: "starting".to_string(), + stale: false, error: None, index_size: None, tor_onion: None, @@ -64,15 +68,33 @@ fn cache() -> &'static RwLock { /// Spawn background task that refreshes ElectrumX status every CACHE_REFRESH_SECS. pub fn spawn_status_cache() { tokio::spawn(async { - // Initial delay — let services start up before first query - tokio::time::sleep(Duration::from_secs(5)).await; - - let mut interval = tokio::time::interval(Duration::from_secs(CACHE_REFRESH_SECS)); loop { - interval.tick().await; - let fresh = fetch_electrs_sync_status().await; + let mut fresh = fetch_electrs_sync_status().await; let mut cached = cache().write().await; + if fresh.indexed_height == 0 + && cached.indexed_height > 0 + && matches!(fresh.status.as_str(), "indexing" | "waiting") + { + fresh.indexed_height = cached.indexed_height; + if fresh.network_height == 0 { + fresh.network_height = cached.network_height; + } + if fresh.bitcoin_height == 0 { + fresh.bitcoin_height = cached.bitcoin_height; + } + if fresh.progress_pct <= 0.0 { + fresh.progress_pct = cached.progress_pct; + } + fresh.stale = true; + fresh.error = Some( + fresh + .error + .unwrap_or_else(|| "ElectrumX is reconnecting; showing last known indexed height.".to_string()), + ); + } *cached = fresh; + drop(cached); + tokio::time::sleep(Duration::from_secs(CACHE_REFRESH_SECS)).await; } }); } @@ -187,13 +209,69 @@ async fn electrumx_indexed_height() -> Result { Ok(height) } -/// Fetch Bitcoin network height via JSON-RPC. -async fn bitcoin_network_height() -> Result { +fn parse_electrumx_height_from_logs(logs: &str) -> Option { + let mut height = None; + + for line in logs.lines() { + if let Some(idx) = line.find("BlockProcessor:our height:") { + let rest = &line[idx + "BlockProcessor:our height:".len()..]; + if let Some(parsed) = parse_first_u64_token(rest) { + height = Some(parsed); + } + continue; + } + + if let Some(idx) = line.find("DB:height:") { + let rest = &line[idx + "DB:height:".len()..]; + if let Some(parsed) = parse_first_u64_token(rest) { + height = Some(parsed); + } + } + } + + height +} + +fn parse_first_u64_token(input: &str) -> Option { + let token: String = input + .trim_start() + .chars() + .take_while(|c| c.is_ascii_digit() || *c == ',') + .filter(|c| *c != ',') + .collect(); + + if token.is_empty() { + None + } else { + token.parse().ok() + } +} + +async fn electrumx_log_indexed_height() -> Result { + let output = tokio::process::Command::new("podman") + .args(["logs", "--tail", "500", "electrumx"]) + .output() + .await + .context("Failed to read ElectrumX logs")?; + + if !output.status.success() { + anyhow::bail!( + "podman logs electrumx failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + let logs = String::from_utf8_lossy(&output.stdout); + parse_electrumx_height_from_logs(&logs).context("No ElectrumX indexed height in logs") +} + +/// Fetch Bitcoin local block height and best-known network header height via JSON-RPC. +async fn bitcoin_chain_heights() -> Result<(u64, u64)> { let client = reqwest::Client::new(); let body = serde_json::json!({ "jsonrpc": "1.0", "id": "electrs-status", - "method": "getblockcount", + "method": "getblockchaininfo", "params": [] }); let resp = client @@ -211,11 +289,18 @@ async fn bitcoin_network_height() -> Result { } let json: serde_json::Value = resp.json().await?; - let height = json + let result = json .get("result") - .and_then(|r| r.as_u64()) .context("Missing result in Bitcoin RPC")?; - Ok(height) + let blocks = result + .get("blocks") + .and_then(|h| h.as_u64()) + .context("Missing blocks in Bitcoin RPC")?; + let headers = result + .get("headers") + .and_then(|h| h.as_u64()) + .unwrap_or(blocks); + Ok((blocks, headers.max(blocks))) } /// Fetch fresh ElectrumX sync status (called by background cache task). @@ -260,8 +345,8 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus { onion }; - let network_height = match bitcoin_network_height().await { - Ok(h) => h, + let (bitcoin_blocks, network_height) = match bitcoin_chain_heights().await { + Ok(heights) => heights, Err(e) => { let err_msg = e.to_string(); if is_transient_error(&err_msg) { @@ -271,9 +356,11 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus { } return ElectrsSyncStatus { indexed_height: 0, + bitcoin_height: 0, network_height: 0, progress_pct: 0.0, status: "waiting".to_string(), + stale: false, error: Some("Waiting for Bitcoin node...".to_string()), index_size, tor_onion, @@ -283,7 +370,9 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus { let indexed_height = match electrumx_indexed_height().await { Ok(h) => h, - Err(e) => { + Err(e) => match electrumx_log_indexed_height().await { + Ok(h) if h > 0 => h, + _ => { let err_msg = e.to_string(); if is_transient_error(&err_msg) { // ElectrumX is starting up or busy — estimate from data size @@ -295,9 +384,11 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus { let size_str = index_size.clone().unwrap_or_else(|| "0 MB".to_string()); return ElectrsSyncStatus { indexed_height: 0, + bitcoin_height: bitcoin_blocks, network_height, progress_pct, status: "indexing".to_string(), + stale: false, error: Some(format!( "Building index ({} / ~130 GB estimated). Electrum RPC will be available when complete.", size_str @@ -310,35 +401,85 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus { warn!("ElectrumX status: unexpected error: {}", err_msg); return ElectrsSyncStatus { indexed_height: 0, + bitcoin_height: bitcoin_blocks, network_height, progress_pct: 0.0, status: "error".to_string(), + stale: false, error: Some(format!("ElectrumX: {}", err_msg)), index_size, tor_onion, }; } + }, }; - let progress_pct = if network_height > 0 { - (indexed_height as f64 / network_height as f64) * 100.0 + let observed_header_height = network_height.max(indexed_height); + let bitcoin_catching_up = bitcoin_blocks > 0 && bitcoin_blocks < observed_header_height; + let electrum_waiting_on_bitcoin = + bitcoin_catching_up && indexed_height >= bitcoin_blocks.saturating_sub(1); + let sync_target_height = if bitcoin_blocks > 0 { + bitcoin_blocks + } else { + observed_header_height + }; + + let progress_pct = if electrum_waiting_on_bitcoin && observed_header_height > 0 { + ((bitcoin_blocks as f64 / observed_header_height as f64) * 100.0).min(99.9) + } else if sync_target_height > 0 { + ((indexed_height as f64 / sync_target_height as f64) * 100.0).min(100.0) } else { 0.0 }; - let status = if indexed_height >= network_height.saturating_sub(1) { + let status = if sync_target_height == 0 { + "waiting" + } else if electrum_waiting_on_bitcoin { + "waiting" + } else if indexed_height >= sync_target_height.saturating_sub(1) { "synced" } else { "syncing" }; + let error = if electrum_waiting_on_bitcoin { + Some(format!( + "ElectrumX is indexed to {:}; waiting for the local Bitcoin node to catch up from {:} to known header {:}.", + indexed_height, bitcoin_blocks, observed_header_height + )) + } else if status == "syncing" && bitcoin_blocks < observed_header_height { + Some(format!( + "Indexing local Bitcoin node height {:} of {:}. Bitcoin node is still catching up to known header {:}.", + indexed_height, bitcoin_blocks, observed_header_height + )) + } else { + None + }; + ElectrsSyncStatus { indexed_height, - network_height, + bitcoin_height: bitcoin_blocks, + network_height: observed_header_height, progress_pct, status: status.to_string(), - error: None, + stale: false, + error, index_size, tor_onion, } } + +#[cfg(test)] +mod tests { + use super::parse_electrumx_height_from_logs; + + #[test] + fn parses_latest_electrumx_progress_height_from_logs() { + let logs = r#" +INFO:DB:height: 228,238 +INFO:BlockProcessor:our height: 228,248 daemon: 731,568 UTXOs 1MB hist 1MB +INFO:BlockProcessor:our height: 232,117 daemon: 732,108 UTXOs 281MB hist 83MB +"#; + assert_eq!(parse_electrumx_height_from_logs(logs), Some(232_117)); + } +} diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index f7644810..68d47b41 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -29,6 +29,7 @@ mod auth; mod avatar; mod backup; mod bitcoin_rpc; +mod bitcoin_status; mod blobs; mod bootstrap; mod config; @@ -289,6 +290,7 @@ async fn main() -> Result<()> { // Spawn ElectrumX status cache (refreshes every 15s, serves cached data to avoid race conditions) electrs_status::spawn_status_cache(); + bitcoin_status::spawn_status_cache(); let startup_ms = startup_start.elapsed().as_millis(); info!( diff --git a/docker/bitcoin-ui/index.html b/docker/bitcoin-ui/index.html index ac81d9e9..415fdf85 100644 --- a/docker/bitcoin-ui/index.html +++ b/docker/bitcoin-ui/index.html @@ -606,7 +606,8 @@ console.log('[Bitcoin UI] Script loaded, initializing...'); // RPC Configuration - Use local Nginx proxy within container - const RPC_ENDPOINT = '/bitcoin-rpc/'; + const RPC_ENDPOINT = 'bitcoin-rpc/'; + const STATUS_ENDPOINT = 'bitcoin-status'; console.log('[Bitcoin UI] RPC Endpoint:', RPC_ENDPOINT); // Make RPC call to Bitcoin node via local proxy @@ -645,6 +646,14 @@ } } + async function fetchBitcoinStatus() { + const response = await fetch(STATUS_ENDPOINT, { cache: 'no-store' }); + if (!response.ok) { + throw new Error(`status HTTP ${response.status}`); + } + return response.json(); + } + // Implementation branding — detected from getnetworkinfo.subversion. // Bitcoin Knots identifies as "/Satoshi:/Knots:/", Bitcoin Core as "/Satoshi:/". let brandingApplied = false; @@ -672,22 +681,62 @@ // Track last block count for animations let lastBlockCount = 0; + let consecutiveRpcFailures = 0; + let lastSuccessfulUpdateAt = 0; + + function formatPercent(value) { + if (!Number.isFinite(value) || value <= 0) return '0.00'; + if (value < 0.01) return '<0.01'; + return value.toFixed(2); + } + + function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes <= 0) return null; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let value = bytes; + let unit = 0; + while (value >= 1000 && unit < units.length - 1) { + value /= 1000; + unit += 1; + } + return `${value.toFixed(unit >= 3 ? 1 : 0)} ${units[unit]}`; + } // Update blockchain info async function updateBlockchainInfo() { console.log('[Bitcoin UI] updateBlockchainInfo() called'); try { - const blockchainInfo = await callRPC('getblockchaininfo'); + const status = await fetchBitcoinStatus(); + const blockchainInfo = status.blockchain_info; console.log('[Bitcoin UI] blockchainInfo:', blockchainInfo); if (!blockchainInfo) { console.error('[Bitcoin UI] No blockchain info received'); - document.getElementById('syncStatusText').textContent = 'Unable to connect to Bitcoin node'; - document.getElementById('syncStatusText').className = 'text-red-400 text-sm'; + consecutiveRpcFailures += 1; + const syncStatusText = document.getElementById('syncStatusText'); + const syncIcon = document.getElementById('syncIcon'); + if (syncStatusText) { + if (status.stale) { + syncStatusText.textContent = status.error || 'Bitcoin node is reconnecting... showing last known values'; + syncStatusText.className = 'text-yellow-300 text-sm font-medium'; + } else if (consecutiveRpcFailures < 6) { + syncStatusText.textContent = status.error || 'Connecting to Bitcoin node...'; + syncStatusText.className = 'text-yellow-300 text-sm font-medium'; + } else { + syncStatusText.textContent = status.error || 'Bitcoin node is not responding yet'; + syncStatusText.className = 'text-red-400 text-sm font-medium'; + } + } + if (syncIcon) { + syncIcon.classList.add('animate-spin-slow'); + syncIcon.classList.remove('text-green-500'); + } return; } + consecutiveRpcFailures = 0; + lastSuccessfulUpdateAt = Date.now(); - const networkInfo = await callRPC('getnetworkinfo'); + const networkInfo = status.network_info; applyImplBranding(networkInfo && networkInfo.subversion); @@ -743,44 +792,51 @@ } // Populate Settings — Transaction Index, ZMQ, RPC (fire-and-forget) - (async () => { - const txIndexEl = document.getElementById('settingsTxIndex'); - if (txIndexEl) { - const idx = await callRPC('getindexinfo'); - if (idx && typeof idx === 'object') { - const names = Object.keys(idx); - txIndexEl.textContent = names.length - ? `Enabled: ${names.join(', ')}` - : 'Disabled'; - } else { - txIndexEl.textContent = 'Disabled'; - } + const txIndexEl = document.getElementById('settingsTxIndex'); + if (txIndexEl) { + const idx = status.index_info; + if (idx && typeof idx === 'object') { + const names = Object.keys(idx); + txIndexEl.textContent = names.length + ? `Enabled: ${names.join(', ')}` + : 'Disabled'; + } else { + txIndexEl.textContent = 'Unavailable while node starts'; } - const zmqEl = document.getElementById('settingsZmq'); - if (zmqEl) { - const zmq = await callRPC('getzmqnotifications'); - if (Array.isArray(zmq) && zmq.length) { - zmqEl.textContent = zmq.map(z => `${z.type}@${z.address}`).join('; '); - } else { - zmqEl.textContent = 'Not enabled'; - } + } + const zmqEl = document.getElementById('settingsZmq'); + if (zmqEl) { + const zmq = status.zmq_notifications; + if (Array.isArray(zmq) && zmq.length) { + zmqEl.textContent = zmq.map(z => `${z.type}@${z.address}`).join('; '); + } else if (Array.isArray(zmq)) { + zmqEl.textContent = 'Not enabled'; + } else { + zmqEl.textContent = 'Unavailable while node starts'; } - const rpcEl = document.getElementById('settingsRpc'); - if (rpcEl && networkInfo) { - const port = chain === 'main' ? 8332 : (chain === 'test' ? 18332 : (chain === 'signet' ? 38332 : 18443)); - rpcEl.textContent = `Reachable on port ${port}`; - } - })(); + } + const rpcEl = document.getElementById('settingsRpc'); + if (rpcEl) { + const port = chain === 'main' ? 8332 : (chain === 'test' ? 18332 : (chain === 'signet' ? 38332 : 18443)); + rpcEl.textContent = status.stale + ? `Reconnecting on port ${port}` + : `Reachable on port ${port}`; + } // Update sync status const blocks = blockchainInfo.blocks || 0; const headers = blockchainInfo.headers || 0; const verificationProgress = blockchainInfo.verificationprogress || 0; - const isSynced = blocks >= headers - 1; + const initialBlockDownload = blockchainInfo.initialblockdownload === true; + const isSynced = headers > 0 && blocks >= headers - 1 && !initialBlockDownload; + const diskSize = formatBytes(blockchainInfo.size_on_disk || 0); + const appearsToBeReindexing = initialBlockDownload && blocks === 0 && headers > 0 && (blockchainInfo.size_on_disk || 0) > 1024 * 1024 * 1024; // Calculate actual sync percentage based on blocks/headers - const actualSyncPercentage = headers > 0 ? ((blocks / headers) * 100).toFixed(2) : '0.00'; - const verificationPercentage = (verificationProgress * 100).toFixed(2); + const actualSyncValue = headers > 0 ? (blocks / headers) * 100 : 0; + const actualSyncPercentage = formatPercent(actualSyncValue); + const progressWidth = Math.max(0, Math.min(100, actualSyncValue)); + const verificationPercentage = formatPercent(verificationProgress * 100); // Animate block count if it changed const currentHeightElem = document.getElementById('currentHeight'); @@ -795,16 +851,27 @@ document.getElementById('headers').textContent = headers.toLocaleString(); document.getElementById('verificationProgress').textContent = `${verificationPercentage}%`; document.getElementById('syncPercentage').textContent = `${actualSyncPercentage}%`; - document.getElementById('currentBlock').textContent = `Block ${blocks.toLocaleString()}`; - document.getElementById('syncProgressBar').style.width = `${actualSyncPercentage}%`; + document.getElementById('currentBlock').textContent = appearsToBeReindexing + ? 'Reindexing from disk' + : `Block ${blocks.toLocaleString()}`; + document.getElementById('syncProgressBar').style.width = `${progressWidth}%`; // Update sync status text and icon const syncStatusText = document.getElementById('syncStatusText'); const syncIcon = document.getElementById('syncIcon'); - if (isSynced) { - syncStatusText.textContent = '✓ Fully synchronized with the network'; - syncStatusText.className = 'text-green-400 text-sm font-medium'; + if (appearsToBeReindexing) { + syncStatusText.textContent = `Reindexing local block files${diskSize ? ` (${diskSize} on disk)` : ''}`; + syncStatusText.className = 'text-orange-400 text-sm font-medium'; + if (syncIcon) { + syncIcon.classList.add('animate-spin-slow'); + syncIcon.classList.remove('text-green-500'); + } + } else if (isSynced) { + syncStatusText.textContent = status.stale + ? 'Bitcoin node is reconnecting... showing last known synchronized state' + : '✓ Fully synchronized with the network'; + syncStatusText.className = status.stale ? 'text-yellow-300 text-sm font-medium' : 'text-green-400 text-sm font-medium'; // Stop spinning when synced if (syncIcon) { syncIcon.classList.remove('animate-spin-slow'); @@ -812,8 +879,12 @@ } } else { const remaining = headers - blocks; - syncStatusText.textContent = `Syncing... ${remaining.toLocaleString()} blocks remaining`; - syncStatusText.className = 'text-orange-400 text-sm font-medium'; + syncStatusText.textContent = status.stale + ? 'Bitcoin node is reconnecting... showing last known sync state' + : initialBlockDownload + ? `Initial block download... ${remaining.toLocaleString()} blocks remaining` + : `Syncing... ${remaining.toLocaleString()} blocks remaining`; + syncStatusText.className = status.stale ? 'text-yellow-300 text-sm font-medium' : 'text-orange-400 text-sm font-medium'; // Keep spinning while syncing if (syncIcon) { syncIcon.classList.add('animate-spin-slow'); @@ -834,8 +905,15 @@ } catch (error) { console.error('Failed to update blockchain info:', error); - document.getElementById('syncStatusText').textContent = 'Unable to fetch blockchain data'; - document.getElementById('syncStatusText').className = 'text-red-400 text-sm'; + consecutiveRpcFailures += 1; + const syncStatusText = document.getElementById('syncStatusText'); + if (syncStatusText) { + const hasRecentData = lastSuccessfulUpdateAt > 0 && Date.now() - lastSuccessfulUpdateAt < 120000; + syncStatusText.textContent = hasRecentData + ? 'Bitcoin status bridge is reconnecting... keeping last known values' + : 'Connecting to Bitcoin status bridge...'; + syncStatusText.className = 'text-yellow-300 text-sm font-medium'; + } } } diff --git a/docker/electrs-ui/index.html b/docker/electrs-ui/index.html index 539ae344..dc43ec47 100644 --- a/docker/electrs-ui/index.html +++ b/docker/electrs-ui/index.html @@ -68,6 +68,7 @@ @media (min-width: 768px) { .md-flex-row { flex-direction: row; } .md-grid-cols-4 { grid-template-columns: repeat(4, 1fr); } + .md-grid-cols-5 { grid-template-columns: repeat(5, 1fr); } } /* Connection details */ @@ -147,13 +148,17 @@ -
+
-

Indexed Height

+

Electrum Indexed

-

-

Network Height

+

Bitcoin Node

+

-

+
+
+

Known Headers

-

@@ -370,15 +375,36 @@ } var indexedH = data.indexed_height || 0; - var networkH = data.network_height || 0; + var bitcoinH = data.bitcoin_height || 0; + var reportedNetworkH = data.network_height || 0; + var knownHeaderH = Math.max(reportedNetworkH, indexedH, bitcoinH); + var targetH = bitcoinH > 0 ? bitcoinH : knownHeaderH; var pct = data.progress_pct || 0; + var hasIndexedHeight = indexedH > 0 || data.stale; + var indexedLabel = hasIndexedHeight + ? indexedH.toLocaleString() + : (data.status === 'indexing' ? 'Pending' : '-'); + var currentBlockLabel; + if (hasIndexedHeight && bitcoinH > 0 && indexedH > bitcoinH) { + currentBlockLabel = 'Bitcoin node ' + bitcoinH.toLocaleString() + + (knownHeaderH > 0 ? ' of known headers ' + knownHeaderH.toLocaleString() : '') + + '; Electrum index ' + indexedH.toLocaleString(); + } else if (hasIndexedHeight) { + currentBlockLabel = 'Indexed ' + indexedH.toLocaleString() + ' of ' + + (targetH > 0 ? targetH.toLocaleString() : 'Bitcoin node height'); + } else { + currentBlockLabel = data.index_size + ? 'Index building from disk (' + data.index_size + ')' + : 'Waiting for Electrum index height'; + } - document.getElementById('indexedHeight').textContent = indexedH > 0 ? indexedH.toLocaleString() : (data.status === 'indexing' ? 'Building...' : '-'); - document.getElementById('networkHeight').textContent = networkH > 0 ? networkH.toLocaleString() : '-'; + document.getElementById('indexedHeight').textContent = indexedLabel; + document.getElementById('bitcoinHeight').textContent = bitcoinH > 0 ? bitcoinH.toLocaleString() : 'Checking...'; + document.getElementById('networkHeight').textContent = knownHeaderH > 0 ? knownHeaderH.toLocaleString() : 'Checking...'; document.getElementById('indexSize').textContent = data.index_size || '-'; - document.getElementById('progressPct').textContent = pct > 0 ? pct.toFixed(1) + '%' : '-'; - document.getElementById('currentBlock').textContent = indexedH > 0 ? 'Block ' + indexedH.toLocaleString() : (data.index_size ? 'Index: ' + data.index_size : 'Block 0'); - document.getElementById('syncPercentage').textContent = pct > 0 ? pct.toFixed(1) + '%' : '0%'; + document.getElementById('progressPct').textContent = (knownHeaderH > 0 || pct > 0) ? pct.toFixed(1) + '%' : '-'; + document.getElementById('currentBlock').textContent = currentBlockLabel; + document.getElementById('syncPercentage').textContent = (knownHeaderH > 0 || pct > 0) ? pct.toFixed(1) + '%' : '0%'; document.getElementById('syncProgressBar').style.width = Math.max(pct, 0.5) + '%'; var statusTextEl = document.getElementById('syncStatusText'); @@ -389,14 +415,16 @@ statusTextEl.textContent = data.error || 'Starting up...'; statusTextEl.style.color = '#fbbf24'; statusDot.className = 'status-dot bg-yellow animate-pulse'; - document.getElementById('statusText').textContent = 'Starting'; + document.getElementById('statusText').textContent = data.status === 'waiting' ? 'Waiting' : 'Starting'; syncIcon.classList.add('animate-spin-slow'); document.getElementById('connSubtitle').textContent = 'Connections will be available once ElectrumX has completed syncing.'; } else if (data.status === 'indexing') { - statusTextEl.textContent = data.error || 'Building index...'; + statusTextEl.textContent = data.stale + ? (data.error || 'ElectrumX is reconnecting; showing last known indexed height.') + : (data.error || 'Building index. Indexed height will appear when Electrum RPC is ready.'); statusTextEl.style.color = '#fbbf24'; statusDot.className = 'status-dot bg-amber animate-pulse'; - document.getElementById('statusText').textContent = 'Indexing'; + document.getElementById('statusText').textContent = data.stale ? 'Reconnecting' : 'Indexing'; syncIcon.classList.add('animate-spin-slow'); document.getElementById('connSubtitle').textContent = 'Connections will be available once ElectrumX has completed syncing.'; } else if (data.status === 'error') { @@ -414,8 +442,10 @@ syncIcon.style.color = '#4ade80'; document.getElementById('connSubtitle').textContent = 'Use the following details to connect your wallet or application to ElectrumX.'; } else { - var remaining = networkH - indexedH; - statusTextEl.textContent = 'Syncing... ' + remaining.toLocaleString() + ' blocks remaining'; + var remaining = Math.max(targetH - indexedH, 0); + statusTextEl.textContent = data.error || (targetH > 0 + ? 'Syncing... ' + remaining.toLocaleString() + ' blocks remaining' + : 'Waiting for Bitcoin network height...'); statusTextEl.style.color = '#fb923c'; statusDot.className = 'status-dot bg-yellow'; document.getElementById('statusText').textContent = 'Syncing'; diff --git a/image-recipe/configs/archipelago-doctor.service b/image-recipe/configs/archipelago-doctor.service index 667e133b..6112febb 100644 --- a/image-recipe/configs/archipelago-doctor.service +++ b/image-recipe/configs/archipelago-doctor.service @@ -7,6 +7,6 @@ Type=oneshot # Runs as root: needs to kill orphaned conmon processes, fix permissions User=root ExecStart=/home/archipelago/archy/scripts/container-doctor.sh --local -TimeoutStartSec=120 +TimeoutStartSec=300 StandardOutput=journal StandardError=journal diff --git a/image-recipe/configs/archipelago-doctor.timer b/image-recipe/configs/archipelago-doctor.timer index 26906149..a72cf2f9 100644 --- a/image-recipe/configs/archipelago-doctor.timer +++ b/image-recipe/configs/archipelago-doctor.timer @@ -2,9 +2,11 @@ Description=Archipelago container doctor (periodic) [Timer] -# First run 5 minutes after boot, then every 30 minutes -OnBootSec=5min -OnUnitActiveSec=30min +# First run 2 minutes after boot, then every 5 minutes. The doctor is +# idempotent and exits quickly when no drift exists; this keeps vanished +# rootless port listeners and stopped containers from remaining broken. +OnBootSec=2min +OnUnitActiveSec=5min # Jitter to avoid load spikes RandomizedDelaySec=60 diff --git a/neode-ui/package-lock.json b/neode-ui/package-lock.json index 02df9c47..cde8444a 100644 --- a/neode-ui/package-lock.json +++ b/neode-ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "neode-ui", - "version": "1.7.44-alpha", + "version": "1.7.49-alpha", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "neode-ui", - "version": "1.7.44-alpha", + "version": "1.7.49-alpha", "dependencies": { "@types/dompurify": "^3.0.5", "@vue-leaflet/vue-leaflet": "^0.10.1", diff --git a/neode-ui/package.json b/neode-ui/package.json index 7c45d1b3..a8a108da 100644 --- a/neode-ui/package.json +++ b/neode-ui/package.json @@ -1,7 +1,7 @@ { "name": "neode-ui", "private": true, - "version": "1.7.48-alpha", + "version": "1.7.49-alpha", "type": "module", "scripts": { "start": "./start-dev.sh", diff --git a/neode-ui/public/catalog.json b/neode-ui/public/catalog.json index 5fee9801..3db1472e 100644 --- a/neode-ui/public/catalog.json +++ b/neode-ui/public/catalog.json @@ -26,7 +26,7 @@ "id": "bitcoin-core", "title": "Bitcoin Core", "version": "28.4", - "description": "Reference implementation of the Bitcoin protocol. Run a full node validating and relaying blocks.", + "description": "Reference Bitcoin node implementation. Alternative to Bitcoin Knots; uninstall Knots before switching.", "icon": "/assets/img/app-icons/bitcoin-core.svg", "author": "Bitcoin Core contributors", "category": "money", diff --git a/neode-ui/src/stores/appLauncher.ts b/neode-ui/src/stores/appLauncher.ts index c6b35235..a6b01c86 100644 --- a/neode-ui/src/stores/appLauncher.ts +++ b/neode-ui/src/stores/appLauncher.ts @@ -13,7 +13,7 @@ const NEW_TAB_PORTS = new Set([ '8085', // Nextcloud — X-Frame-Options: SAMEORIGIN '3002', // Uptime Kuma — X-Frame-Options: SAMEORIGIN '9001', // Penpot — not reachable - // IndeedHub (7777) uses proxy path for NIP-07 nostr-provider.js — NOT new tab + // Port 7777 is the Nostr relay; IndeeHub's web UI is exposed on 7778. ]) const NEW_TAB_APP_IDS = new Set([ @@ -34,6 +34,7 @@ function mustOpenInNewTab(url: string): boolean { function inferAppIdFromTitle(title?: string): string | null { const t = (title || '').toLowerCase() if (!t) return null + if (t.includes('indeehub') || t.includes('indeedhub')) return 'indeedhub' if ((t.includes('uptime') && t.includes('kuma')) || t.includes('uptime-kuma')) return 'uptime-kuma' if ((t.includes('nginx') && t.includes('proxy') && t.includes('manager')) || t.includes('nginx-proxy-manager')) return 'nginx-proxy-manager' if (t.includes('gitea')) return 'gitea' @@ -47,6 +48,10 @@ function normalizeLaunchUrl(urlStr: string, appIdHint?: string | null): string { const normalizedPath = u.pathname === '/' ? '' : u.pathname const rebuilt = (port: string) => `${u.protocol}//${u.hostname}:${port}${normalizedPath}${u.search}${u.hash}` + if (sameHost && appIdHint === 'indeedhub' && u.port === '7777') { + return rebuilt('7778') + } + if (sameHost && appIdHint === 'uptime-kuma' && u.port === '3001') { return rebuilt('3002') } @@ -87,7 +92,7 @@ const PORT_TO_APP_ID: Record = { '8175': 'fedimint', '8176': 'fedimint-gateway', '3100': 'dwn', - '7777': 'indeedhub', + '7778': 'indeedhub', '50002': 'electrumx', '3010': 'thunderhub', } diff --git a/neode-ui/src/utils/dummyApps.ts b/neode-ui/src/utils/dummyApps.ts index 5420287e..6a87a754 100644 --- a/neode-ui/src/utils/dummyApps.ts +++ b/neode-ui/src/utils/dummyApps.ts @@ -504,7 +504,7 @@ export const dummyApps: Record = { 'interface-addresses': { main: { 'tor-address': '', - 'lan-address': 'http://localhost:8190' + 'lan-address': 'http://localhost:7778' } }, status: ServiceStatus.Running @@ -749,4 +749,3 @@ export const dummyApps: Record = { } } } - diff --git a/neode-ui/src/views/Kiosk.vue b/neode-ui/src/views/Kiosk.vue index 0cff1e0d..3553e065 100644 --- a/neode-ui/src/views/Kiosk.vue +++ b/neode-ui/src/views/Kiosk.vue @@ -107,7 +107,7 @@ const launchableApps = computed(() => { 'fedimint': '/app/fedimint/', 'fedimint-gateway': '/app/fedimint-gateway/', 'dwn': '/app/dwn/', - 'indeedhub': 'http://localhost:8190', + 'indeedhub': 'http://localhost:7778', 'botfights': 'http://localhost:9100', 'nwnn': 'https://nwnn.l484.com', '484-kitchen': 'https://484.kitchen', diff --git a/neode-ui/src/views/appSession/appSessionConfig.ts b/neode-ui/src/views/appSession/appSessionConfig.ts index 99bf55be..5d88d015 100644 --- a/neode-ui/src/views/appSession/appSessionConfig.ts +++ b/neode-ui/src/views/appSession/appSessionConfig.ts @@ -38,15 +38,14 @@ export const APP_PORTS: Record = { 'fedimint': 8175, 'fedimintd': 8175, 'fedimint-gateway': 8176, - 'indeedhub': 7777, + 'indeedhub': 7778, 'botfights': 9100, 'dwn': 3100, 'endurain': 8080, } /** Apps that need nginx proxy for iframe embedding. - * IndeedHub loads via /app/indeedhub/ proxy for nostr-provider.js injection - * from the container's internal nginx so iframe works on all servers. */ + * IndeeHub web UI is on 7778. Port 7777 is the Nostr relay. */ export const PROXY_APPS: Record = { 'gitea': '/app/gitea/', 'nginx-proxy-manager': '/app/nginx-proxy-manager/', @@ -60,9 +59,10 @@ export const HTTPS_PROXY_PATHS: Record = { 'bitcoin-core': '/app/bitcoin-ui/', 'bitcoin-ui': '/app/bitcoin-ui/', 'lnd': '/app/lnd/', - 'electrumx': '/app/electrs/', - 'electrs': '/app/electrs/', - 'mempool-electrs': '/app/electrs/', + 'electrumx': '/app/electrumx/', + 'electrs': '/app/electrumx/', + 'archy-electrs-ui': '/app/electrumx/', + 'mempool-electrs': '/app/electrumx/', 'mempool': '/app/mempool/', 'mempool-web': '/app/mempool/', 'archy-mempool-web': '/app/mempool/', @@ -87,7 +87,6 @@ export const HTTPS_PROXY_PATHS: Record = { 'btcpay-server': '/app/btcpay/', 'nextcloud': '/app/nextcloud/', 'grafana': '/app/grafana/', - 'indeedhub': '/app/indeedhub/', 'botfights': '/app/botfights/', 'gitea': '/app/gitea/', } diff --git a/scripts/app-surface-smoke-test.sh b/scripts/app-surface-smoke-test.sh new file mode 100755 index 00000000..a014a021 --- /dev/null +++ b/scripts/app-surface-smoke-test.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# App surface smoke test. +# +# Verifies that installed containers have their published host ports listening +# and that known nginx app proxy paths return a non-5xx response. This catches +# the common "container is running but UI disappeared" failure mode. +# +# Usage: +# scripts/app-surface-smoke-test.sh --target archipelago@192.168.1.228 --ssh-key /path/key + +set -euo pipefail + +TARGET="" +SSH_KEY="${ARCHIPELAGO_SSH_KEY:-}" +SSH_EXTRA=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --target) TARGET="${2:-}"; shift 2 ;; + --ssh-key) SSH_KEY="${2:-}"; shift 2 ;; + --ssh-option) SSH_EXTRA+=("-o" "${2:-}"); shift 2 ;; + -h|--help) sed -n '1,12p' "$0"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +[ -n "$TARGET" ] || { echo "--target is required" >&2; exit 2; } + +SSH_OPTS=(-F /dev/null -o BatchMode=yes -o PreferredAuthentications=publickey -o PasswordAuthentication=no) +[ -n "$SSH_KEY" ] && SSH_OPTS+=(-i "$SSH_KEY") +SSH_OPTS+=("${SSH_EXTRA[@]}") + +ssh_run() { + ssh "${SSH_OPTS[@]}" "$TARGET" "$@" +} + +ssh_run 'bash -s' <<'REMOTE' +set -u + +pass=0 +fail=0 + +ok() { echo " PASS $*"; pass=$((pass + 1)); } +bad() { echo " FAIL $*"; fail=$((fail + 1)); } + +container_exists() { + podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$1" +} + +port_listening() { + ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$1$" +} + +http_code() { + local url="$1" code + for _ in 1 2 3; do + code=$(curl -ksS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true) + [ -n "$code" ] || code=000 + [ "$code" != "000" ] && { echo "$code"; return; } + sleep 2 + done + echo "$code" +} + +http_post_code() { + local url="$1" code + for _ in 1 2 3; do + code=$(curl -ksS -o /dev/null -w '%{http_code}' --max-time 25 \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"getblockchaininfo","params":[]}' \ + "$url" 2>/dev/null || true) + [ -n "$code" ] || code=000 + [ "$code" != "000" ] && { echo "$code"; return; } + sleep 2 + done + echo "$code" +} + +assert_http() { + local label="$1" url="$2" code + code=$(http_code "$url") + case "$code" in + 200|204|301|302|307|308|401|403) ok "$label HTTP $code" ;; + *) bad "$label HTTP $code ($url)" ;; + esac +} + +assert_http_post() { + local label="$1" url="$2" code + code=$(http_post_code "$url") + case "$code" in + 200|204|401|403) ok "$label HTTP POST $code" ;; + *) bad "$label HTTP POST $code ($url)" ;; + esac +} + +assert_container_ports() { + local name="$1" ports port missing=0 + container_exists "$name" || return 0 + ports=$(podman inspect "$name" --format '{{range $p,$bindings := .NetworkSettings.Ports}}{{if $bindings}}{{range $bindings}}{{.HostPort}}{{"\n"}}{{end}}{{end}}{{end}}' 2>/dev/null | sort -u) + [ -n "$ports" ] || return 0 + while IFS= read -r port; do + [ -n "$port" ] || continue + if port_listening "$port"; then + ok "$name port $port listening" + else + bad "$name port $port missing listener" + missing=1 + fi + done <<< "$ports" + return "$missing" +} + +assert_env_contains() { + local name="$1" key="$2" needle="$3" val + container_exists "$name" || return 0 + val=$(podman inspect "$name" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n "s/^${key}=//p" | head -n 1) + if [ -n "$val" ] && printf '%s' "$val" | grep -qF "$needle"; then + ok "$name env $key" + else + bad "$name env $key missing $needle" + fi +} + +echo "[surface] host=$(hostname) ip=$(hostname -I 2>/dev/null | awk '{print $1}')" + +for c in $(podman ps -a --format '{{.Names}}' 2>/dev/null | sort); do + assert_container_ports "$c" || true +done + +container_exists archy-bitcoin-ui && { + assert_http "bitcoin-ui" "http://127.0.0.1/app/bitcoin-ui/" + assert_http "bitcoin status" "http://127.0.0.1/app/bitcoin-ui/bitcoin-status" + assert_http_post "bitcoin rpc proxy" "http://127.0.0.1/app/bitcoin-ui/bitcoin-rpc/" +} + +container_exists archy-electrs-ui && { + assert_http "electrumx ui" "http://127.0.0.1/app/electrumx/" + assert_http "electrumx status" "http://127.0.0.1/app/electrumx/electrs-status" + assert_http "electrs legacy status" "http://127.0.0.1/app/electrs/electrs-status" +} + +container_exists mempool && assert_http "mempool ui" "http://127.0.0.1/app/mempool/" +container_exists indeedhub && assert_http "indeedhub ui" "http://127.0.0.1:7778/" +container_exists uptime-kuma && assert_http "uptime-kuma" "http://127.0.0.1/app/uptime-kuma/" +container_exists filebrowser && assert_http "filebrowser" "http://127.0.0.1/app/filebrowser/" +container_exists searxng && assert_http "searxng" "http://127.0.0.1/app/searxng/" +container_exists grafana && assert_http "grafana" "http://127.0.0.1/app/grafana/" +container_exists portainer && assert_http "portainer" "http://127.0.0.1/app/portainer/" +container_exists vaultwarden && assert_http "vaultwarden" "http://127.0.0.1/app/vaultwarden/" +container_exists nextcloud && assert_http "nextcloud" "http://127.0.0.1/app/nextcloud/" +container_exists archy-nbxplorer && assert_env_contains "archy-nbxplorer" "NBXPLORER_POSTGRES" "Database=nbxplorer" +container_exists btcpay-server && { + assert_env_contains "btcpay-server" "BTCPAY_POSTGRES" "Database=btcpay" + assert_http "btcpay" "http://127.0.0.1/app/btcpay/" +} + +echo "[surface] summary: pass=$pass fail=$fail" +[ "$fail" -eq 0 ] +REMOTE diff --git a/scripts/bitcoin-stack-lifecycle-test.sh b/scripts/bitcoin-stack-lifecycle-test.sh new file mode 100755 index 00000000..34cae1f4 --- /dev/null +++ b/scripts/bitcoin-stack-lifecycle-test.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# +# Bitcoin stack lifecycle test. +# +# Exercises the production Bitcoin stack under repeated stop/start and +# remove/recreate cycles while asserting the actual user-facing surfaces: +# Bitcoin RPC, bitcoin-ui /bitcoin-rpc, ElectrumX status, and electrs-ui. +# +# This intentionally removes containers but not data volumes. It is safe for +# installed nodes, but it will briefly interrupt Bitcoin/ElectrumX service. +# +# Usage: +# scripts/bitcoin-stack-lifecycle-test.sh --target archipelago@192.168.1.228 +# scripts/bitcoin-stack-lifecycle-test.sh --target archipelago@192.168.1.116 --cycles 5 + +set -euo pipefail + +TARGET="" +SSH_KEY="${ARCHIPELAGO_SSH_KEY:-}" +CYCLES=3 +SSH_EXTRA=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --target) + TARGET="${2:-}" + shift 2 + ;; + --ssh-key) + SSH_KEY="${2:-}" + shift 2 + ;; + --cycles) + CYCLES="${2:-}" + shift 2 + ;; + --ssh-option) + SSH_EXTRA+=("-o" "${2:-}") + shift 2 + ;; + -h|--help) + sed -n '1,22p' "$0" + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +if [ -z "$TARGET" ]; then + echo "--target is required, for example archipelago@192.168.1.228" >&2 + exit 2 +fi + +SSH=(ssh -F /dev/null -o BatchMode=yes -o PreferredAuthentications=publickey -o PasswordAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) +if [ -n "$SSH_KEY" ]; then + SSH+=("-i" "$SSH_KEY") +fi +SSH+=("${SSH_EXTRA[@]}") + +"${SSH[@]}" "$TARGET" "CYCLES='$CYCLES' bash -s" <<'REMOTE' +set -euo pipefail + +PODMAN="${PODMAN:-podman}" +SCRIPTS_DIR="/opt/archipelago/scripts" +if [ ! -x "$SCRIPTS_DIR/reconcile-containers.sh" ]; then + SCRIPTS_DIR="$HOME/archy/scripts" +fi +RECONCILE="$SCRIPTS_DIR/reconcile-containers.sh" + +pass_count=0 +fail_count=0 + +log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; } +pass() { pass_count=$((pass_count + 1)); printf ' PASS %s\n' "$*"; } +fail() { fail_count=$((fail_count + 1)); printf ' FAIL %s\n' "$*" >&2; } + +retry() { + local timeout="$1" label="$2" + shift 2 + local end=$((SECONDS + timeout)) + local out rc + while [ "$SECONDS" -lt "$end" ]; do + set +e + out=$("$@" 2>&1) + rc=$? + set -e + if [ "$rc" -eq 0 ]; then + pass "$label" + return 0 + fi + sleep 2 + done + fail "$label: $out" + return 1 +} + +rpc_pass() { + cat /var/lib/archipelago/secrets/bitcoin-rpc-password +} + +json_rpc_reachable_or_warming() { + local url="$1" auth_arg=() body rc + if [ "${2:-}" = "auth" ]; then + auth_arg=(--user "archipelago:$(rpc_pass)") + fi + set +e + body=$(curl --connect-timeout 3 --max-time 20 -sS "${auth_arg[@]}" \ + -H "Content-Type: application/json" \ + --data-binary '{"jsonrpc":"1.0","id":"lifecycle-test","method":"getblockchaininfo","params":[]}' \ + "$url" 2>&1) + rc=$? + set -e + [ "$rc" -eq 0 ] || { + echo "$body" + return 1 + } + echo "$body" | grep -q '"result"' && return 0 + echo "$body" | grep -q '"code":-28' && return 0 + echo "$body" + return 1 +} + +bitcoin_status_usable() { + local url="$1" + local body + body=$(curl --connect-timeout 3 --max-time 20 -fsS "$url") + echo "$body" | grep -q '"ok":\(true\|false\)' || { + echo "$body" + return 1 + } + echo "$body" | grep -q '"blockchain_info"' || echo "$body" | grep -q '"error"' +} + +http_ok() { + local url="$1" + curl --connect-timeout 3 --max-time 20 -fsS -o /dev/null "$url" +} + +electrs_status_ok() { + local url="${1:-http://127.0.0.1:50002/electrs-status}" + local body + body=$(curl --connect-timeout 3 --max-time 20 -fsS "$url") + echo "$body" | grep -q '"network_height":[1-9]' || { + echo "$body" + return 1 + } + echo "$body" | grep -q '"status":"\(indexing\|syncing\|synced\|waiting\)"' +} + +container_running() { + local name="$1" + [ "$($PODMAN inspect "$name" --format '{{.State.Status}}' 2>/dev/null || true)" = "running" ] +} + +container_healthy_or_starting() { + local name="$1" + local health + health=$($PODMAN inspect "$name" --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' 2>/dev/null || true) + [ "$health" = "healthy" ] || [ "$health" = "starting" ] || [ -z "$health" ] +} + +assert_bitcoin_stack() { + retry 90 "bitcoin-knots running" container_running bitcoin-knots + retry 90 "bitcoin-knots healthy/starting" container_healthy_or_starting bitcoin-knots + retry 90 "host Bitcoin RPC reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1:8332/ auth + retry 90 "backend Bitcoin status bridge usable" bitcoin_status_usable http://127.0.0.1:5678/bitcoin-status + retry 90 "bitcoin-ui page" http_ok http://127.0.0.1:8334/ + retry 90 "bitcoin-ui status bridge usable" bitcoin_status_usable http://127.0.0.1:8334/bitcoin-status + retry 90 "bitcoin-ui app-session status bridge usable" bitcoin_status_usable http://127.0.0.1/app/bitcoin-ui/bitcoin-status + retry 90 "bitcoin-ui RPC proxy reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1:8334/bitcoin-rpc/ + retry 90 "bitcoin-ui app-session RPC proxy reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1/app/bitcoin-ui/bitcoin-rpc/ +} + +assert_electrum_stack() { + retry 120 "electrumx running" container_running electrumx + retry 120 "electrumx healthy/starting" container_healthy_or_starting electrumx + retry 90 "electrs-ui page" http_ok http://127.0.0.1:50002/ + retry 120 "electrs status has network height" electrs_status_ok + retry 120 "electrs app-session status has network height" electrs_status_ok http://127.0.0.1/app/electrumx/electrs-status + retry 120 "electrs legacy app-session status has network height" electrs_status_ok http://127.0.0.1/app/electrs/electrs-status +} + +reconcile_one() { + local name="$1" + "$RECONCILE" --container="$name" --force --force-recreate --create-missing +} + +restart_container() { + local name="$1" + log "restart $name" + $PODMAN restart "$name" >/dev/null || { + log "podman restart failed for $name; using stop/start" + $PODMAN stop "$name" >/dev/null 2>&1 || true + sleep 3 + $PODMAN start "$name" >/dev/null + } +} + +remove_and_reconcile() { + local name="$1" + log "remove/recreate $name" + $PODMAN rm -f "$name" >/dev/null 2>&1 || true + reconcile_one "$name" +} + +log "target $(hostname) cycles=$CYCLES" +log "using reconciler: $RECONCILE" + +assert_bitcoin_stack +assert_electrum_stack + +for i in $(seq 1 "$CYCLES"); do + log "cycle $i/$CYCLES: bitcoin restart" + restart_container bitcoin-knots + assert_bitcoin_stack + assert_electrum_stack + + log "cycle $i/$CYCLES: bitcoin remove/reconcile" + remove_and_reconcile bitcoin-knots + assert_bitcoin_stack + assert_electrum_stack + + log "cycle $i/$CYCLES: bitcoin UI remove/reconcile" + remove_and_reconcile archy-bitcoin-ui + assert_bitcoin_stack + + log "cycle $i/$CYCLES: electrumx restart" + restart_container electrumx + assert_electrum_stack + + log "cycle $i/$CYCLES: electrumx remove/reconcile" + remove_and_reconcile electrumx + assert_electrum_stack + + log "cycle $i/$CYCLES: electrs UI remove/reconcile" + remove_and_reconcile archy-electrs-ui + assert_electrum_stack +done + +log "final container state" +$PODMAN ps -a --format 'table {{.Names}}\t{{.State}}\t{{.Status}}' \ + | grep -E 'bitcoin-knots|electrumx|archy-bitcoin-ui|archy-electrs-ui' || true + +log "summary: pass=$pass_count fail=$fail_count" +[ "$fail_count" -eq 0 ] +REMOTE diff --git a/scripts/container-doctor.sh b/scripts/container-doctor.sh index 4cbe5a8b..0be90f9b 100755 --- a/scripts/container-doctor.sh +++ b/scripts/container-doctor.sh @@ -15,6 +15,7 @@ # 6. Bitcoin Knots prune+txindex conflict # 7. Containers stuck with exit code 127 (binary not found) # 8. Stopped core containers (rootless restart policy workaround) +# 9. Missing rootless port listeners while Podman still shows published ports # # Safe to run multiple times (idempotent). Never blocks deploy (exit 0 always). # @@ -31,6 +32,21 @@ FIX_NAMES=() log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; } +podman_rootless() { + if [ "$(id -u)" = "0" ] && id archipelago >/dev/null 2>&1; then + local archi_uid + archi_uid=$(id -u archipelago) + sudo -u archipelago env XDG_RUNTIME_DIR="/run/user/$archi_uid" podman "$@" + else + podman "$@" + fi +} + +port_is_listening() { + local port="$1" + ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$" +} + run_fix() { local name="$1" shift @@ -374,6 +390,11 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]])) # at 0 peers; package pulls fail. The only reliable repair is a stop-all/ # start-all cycle so pasta + aardvark-dns rebuild the netns from scratch. fix_rootless_netns_egress() { + # Needs root for nsenter. When doctor runs as the rootless container owner, + # a failed nsenter probe is a permissions artifact, not evidence of broken + # egress; do not cycle the fleet from that context. + [ "$(id -u)" = "0" ] || return 1 + local archi_uid archi_uid=$(id -u archipelago 2>/dev/null) || return 1 @@ -453,6 +474,44 @@ fix_stopped_core_containers() { [ ${#restarted[@]} -gt 0 ] && return 0 || return 1 } +# ── Fix 10: Missing rootless port listeners ───────────────── +# Rootless Podman can leave a container running with PortBindings still present +# while the host-side rootlessport process has disappeared. Nginx then returns +# 502 and direct app ports refuse connections even though `podman ps` looks OK. +fix_missing_rootless_ports() { + local containers + containers=$(podman_rootless ps --format '{{.Names}}' 2>/dev/null || true) + [ -n "$containers" ] || return 1 + + local fixed=false + local name + for name in $containers; do + local ports + ports=$(podman_rootless inspect "$name" --format '{{range $p,$bindings := .NetworkSettings.Ports}}{{if $bindings}}{{range $bindings}}{{.HostPort}}{{"\n"}}{{end}}{{end}}{{end}}' 2>/dev/null | sort -u) + [ -n "$ports" ] || continue + + local missing=() + local port + for port in $ports; do + [ -n "$port" ] || continue + if ! port_is_listening "$port"; then + missing+=("$port") + fi + done + + if [ ${#missing[@]} -gt 0 ]; then + log "Restarting $name: missing rootlessport listener(s): ${missing[*]}" + if podman_rootless restart "$name" >/dev/null 2>&1; then + fixed=true + else + log "WARN: failed to restart $name for missing rootlessport listener(s)" + fi + fi + done + + $fixed && return 0 || return 1 +} + # ── Main ───────────────────────────────────────────────────── # If remote host provided, run via SSH @@ -481,6 +540,7 @@ run_fix "bitcoin-txindex" fix_bitcoin_txindex run_fix "exit-127" fix_exit_127 run_fix "netns-egress" fix_rootless_netns_egress run_fix "stopped-core" fix_stopped_core_containers +run_fix "rootless-ports" fix_missing_rootless_ports echo "" if [ $FIXES_APPLIED -gt 0 ]; then diff --git a/scripts/container-specs.sh b/scripts/container-specs.sh index 6596910d..a33a718a 100755 --- a/scripts/container-specs.sh +++ b/scripts/container-specs.sh @@ -252,7 +252,7 @@ load_spec_archy-nbxplorer() { SPEC_VOLUMES="/var/lib/archipelago/nbxplorer:/data" SPEC_MEMORY="$(mem_limit archy-nbxplorer)" SPEC_HEALTH_CMD="curl -sf http://localhost:32838/ || exit 1" - SPEC_ENV="NBXPLORER_DATADIR=/data NBXPLORER_NETWORK=mainnet NBXPLORER_CHAINS=btc NBXPLORER_BIND=0.0.0.0:32838 NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS NBXPLORER_POSTGRES=User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true" + SPEC_ENV="NBXPLORER_DATADIR=/data NBXPLORER_NETWORK=mainnet NBXPLORER_CHAINS=btc NBXPLORER_BIND=0.0.0.0:32838 NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS NBXPLORER_POSTGRES=Username=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer" SPEC_TIER="2" SPEC_DATA_DIR="/var/lib/archipelago/nbxplorer" SPEC_DEPENDS="bitcoin-knots archy-btcpay-db" @@ -268,7 +268,7 @@ load_spec_btcpay-server() { SPEC_VOLUMES="/var/lib/archipelago/btcpay:/datadir" SPEC_MEMORY="$(mem_limit btcpay-server)" SPEC_HEALTH_CMD="curl -sf http://localhost:49392/ || exit 1" - SPEC_ENV="ASPNETCORE_URLS=http://0.0.0.0:49392 BTCPAY_PROTOCOL=http BTCPAY_HOST=$HOST_IP:23000 BTCPAY_CHAINS=btc BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS BTCPAY_POSTGRES=User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true" + SPEC_ENV="ASPNETCORE_URLS=http://0.0.0.0:49392 BTCPAY_PROTOCOL=http BTCPAY_HOST=$HOST_IP:23000 BTCPAY_CHAINS=btc BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS BTCPAY_POSTGRES=Username=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay" SPEC_TIER="2" SPEC_DATA_DIR="/var/lib/archipelago/btcpay" SPEC_DEPENDS="archy-nbxplorer archy-btcpay-db" @@ -344,7 +344,7 @@ load_spec_homeassistant() { SPEC_ENV="TZ=UTC" SPEC_TIER="3" SPEC_DATA_DIR="/var/lib/archipelago/home-assistant" - SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE" + SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE" SPEC_OPTIONAL="true" } @@ -362,7 +362,7 @@ load_spec_grafana() { SPEC_TIER="3" SPEC_DATA_DIR="/var/lib/archipelago/grafana" SPEC_DATA_UID="100472:100472" - SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE" + SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE" SPEC_OPTIONAL="true" } @@ -370,7 +370,7 @@ load_spec_uptime-kuma() { reset_spec SPEC_NAME="uptime-kuma" SPEC_IMAGE="${UPTIME_KUMA_IMAGE}" - SPEC_PORTS="3001:3001" + SPEC_PORTS="3002:3001" SPEC_VOLUMES="/var/lib/archipelago/uptime-kuma:/app/data" SPEC_MEMORY="$(mem_limit uptime-kuma)" SPEC_HEALTH_CMD="curl -sf http://localhost:3001/ || exit 1" @@ -434,7 +434,7 @@ load_spec_nextcloud() { SPEC_HEALTH_CMD="curl -sf http://localhost:80/ || exit 1" SPEC_TIER="3" SPEC_DATA_DIR="/var/lib/archipelago/nextcloud" - SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE" + SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE" SPEC_OPTIONAL="true" } @@ -539,6 +539,7 @@ load_spec_archy-bitcoin-ui() { SPEC_NAME="archy-bitcoin-ui" SPEC_IMAGE="localhost/bitcoin-ui:local" SPEC_NETWORK="host" + SPEC_VOLUMES="/var/lib/archipelago/bitcoin-ui/nginx.conf:/etc/nginx/conf.d/default.conf:ro" SPEC_MEMORY="$(mem_limit archy-bitcoin-ui)" SPEC_TIER="4" SPEC_LOCAL_IMAGE="true" diff --git a/scripts/nginx-https-app-proxies.conf b/scripts/nginx-https-app-proxies.conf index db73cfae..0257a1ba 100644 --- a/scripts/nginx-https-app-proxies.conf +++ b/scripts/nginx-https-app-proxies.conf @@ -183,6 +183,26 @@ location /app/electrs/ { proxy_hide_header X-Frame-Options; proxy_hide_header Content-Security-Policy; } +location /app/electrumx/ { + proxy_pass http://127.0.0.1:50002/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_hide_header X-Frame-Options; + proxy_hide_header Content-Security-Policy; +} +location /app/electrs-ui/ { + proxy_pass http://127.0.0.1:50002/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_hide_header X-Frame-Options; + proxy_hide_header Content-Security-Policy; +} location /app/nginx-proxy-manager/ { proxy_pass http://127.0.0.1:81/; proxy_http_version 1.1; diff --git a/scripts/reconcile-containers.sh b/scripts/reconcile-containers.sh index aa8a8b2b..a60e93b5 100755 --- a/scripts/reconcile-containers.sh +++ b/scripts/reconcile-containers.sh @@ -8,6 +8,7 @@ # sudo ./reconcile-containers.sh # Fix everything # sudo ./reconcile-containers.sh --check-only # Audit only, no changes # sudo ./reconcile-containers.sh --force # Override user-stopped +# sudo ./reconcile-containers.sh --force-recreate # Recreate matched containers # sudo ./reconcile-containers.sh --tier=2 # Only reconcile tier 2 # sudo ./reconcile-containers.sh --container=lnd # Only reconcile lnd # @@ -18,6 +19,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # ── Parse arguments ────────────────────────────────────────────────── CHECK_ONLY=false FORCE=false +FORCE_RECREATE=false CREATE_MISSING=false FILTER_TIER="" FILTER_CONTAINER="" @@ -25,14 +27,18 @@ for arg in "$@"; do case "$arg" in --check-only) CHECK_ONLY=true ;; --force) FORCE=true ;; + --force-recreate) FORCE_RECREATE=true ;; --create-missing) CREATE_MISSING=true ;; --tier=*) FILTER_TIER="${arg#*=}" ;; --container=*) FILTER_CONTAINER="${arg#*=}" ;; -h|--help) - echo "Usage: $0 [--check-only] [--force] [--create-missing] [--tier=N] [--container=NAME]" + echo "Usage: $0 [--check-only] [--force] [--force-recreate] [--create-missing] [--tier=N] [--container=NAME]" echo "" echo " --check-only Audit only, no changes." echo " --force Override user-stopped state." + echo " --force-recreate Recreate matched existing containers even if they" + echo " otherwise match the spec. Use with --container or" + echo " --tier for scoped image/config refreshes." echo " --create-missing Override SPEC_OPTIONAL for containers that have on-disk" echo " data but no live container (recovery from failed updates)." echo " --tier=N Only reconcile containers in tier N." @@ -110,6 +116,14 @@ container_image() { $PODMAN inspect "$1" --format '{{.ImageName}}' 2>/dev/null } +container_image_id() { + $PODMAN inspect "$1" --format '{{.Image}}' 2>/dev/null +} + +spec_image_id() { + $PODMAN image inspect "$SPEC_IMAGE" --format '{{.Id}}' 2>/dev/null +} + container_network() { # Use actual Networks map — NetworkMode is unreliable (always shows 'bridge' in rootless) local nets @@ -122,6 +136,34 @@ container_memory() { $PODMAN inspect "$1" --format '{{.HostConfig.Memory}}' 2>/dev/null } +container_health_cmd() { + $PODMAN inspect "$1" --format '{{with .Config.Healthcheck}}{{range .Test}}{{println .}}{{end}}{{end}}' 2>/dev/null \ + | awk 'NR > 1 { print }' \ + | paste -sd ' ' - +} + +normalize_health_cmd() { + printf '%s' "$1" | sed 's/\\"/"/g; s/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' +} + +host_port_listening() { + local port="$1" + ss -ltn 2>/dev/null | awk -v p=":$port" ' + $4 == p || $4 ~ p "$" { found=1 } + END { exit found ? 0 : 1 } + ' +} + +container_has_mount() { + local name="$1" source="$2" target="$3" + $PODMAN inspect "$name" --format '{{range .Mounts}}{{println .Source "|" .Destination}}{{end}}' 2>/dev/null \ + | awk -F'|' -v src="$source" -v dst="$target" ' + { gsub(/[[:space:]]+$/, "", $1); gsub(/^[[:space:]]+/, "", $2); } + $1 == src && $2 == dst { found=1 } + END { exit found ? 0 : 1 } + ' +} + # Read one environment variable's current value from a running/stopped container. # Returns empty string if the var is not set. container_env_val() { @@ -153,6 +195,36 @@ image_exists() { echo "$images" | grep -qF "$1" } +resolve_spec_image() { + image_exists "$SPEC_IMAGE" && return + + local image_path image_name image_tag candidate repo + image_path="${SPEC_IMAGE#*/}" + image_name="${SPEC_IMAGE##*/}" + image_tag="${image_name#*:}" + image_name="${image_name%%:*}" + + for candidate in \ + "${ARCHY_REGISTRY_FALLBACK:-}/${image_path}" \ + "80.71.235.15:3000/archipelago/${image_name}:${image_tag}" \ + "80.71.235.15:3000/lfg2025/${image_name}:${image_tag}"; do + [ "$candidate" = "/" ] && continue + if image_exists "$candidate"; then + info "$SPEC_NAME — using local image alias $candidate" + SPEC_IMAGE="$candidate" + return + fi + done + + repo=$($PODMAN images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null \ + | grep -E "/${image_name}:${image_tag}$" \ + | head -1 || true) + if [ -n "$repo" ]; then + info "$SPEC_NAME — using local image alias $repo" + SPEC_IMAGE="$repo" + fi +} + # Convert memory string to bytes for comparison mem_to_bytes() { local m="$1" @@ -262,6 +334,10 @@ reconcile() { return fi + # Resolve registry aliases before create/recreate. ISOs and older installers + # may seed the same image under a fallback registry tag. + resolve_spec_image + # Local images: skip if image doesn't exist and container doesn't exist if [ "$SPEC_LOCAL_IMAGE" = "true" ]; then if ! image_exists "$SPEC_IMAGE" && ! container_exists "$name"; then @@ -284,14 +360,28 @@ reconcile() { local reasons="" if container_exists "$name"; then - local cur_image cur_network cur_memory + local cur_image cur_image_id want_image_id cur_network cur_memory cur_image=$(container_image "$name") + cur_image_id=$(container_image_id "$name") + want_image_id=$(spec_image_id) cur_network=$(container_network "$name") cur_memory=$(container_memory "$name") local spec_memory_bytes expected_network spec_memory_bytes=$(mem_to_bytes "$SPEC_MEMORY") + if [ "$FORCE_RECREATE" = "true" ]; then + action="RECREATE" + reasons+="force-recreate " + fi + + # Same-tag local rebuilds leave running containers on the old image ID. + # Recreate when the currently tagged spec image points at a different ID. + if [ "$action" = "OK" ] && [ -n "$want_image_id" ] && [ -n "$cur_image_id" ] && [ "$cur_image_id" != "$want_image_id" ]; then + action="RECREATE" + reasons+="image-id " + fi + # Check network mismatch # For archy-net and host: exact match required # For bridge/default: accept any non-archy-net, non-host network @@ -319,6 +409,19 @@ reconcile() { reasons+="memory(none→$SPEC_MEMORY) " fi + # Healthcheck drift matters: a stale check can leave an otherwise working + # service permanently unhealthy (for example ElectrumX images do not ship + # curl, so the healthcheck must use python's socket module). + if [ "$action" = "OK" ] && [ -n "$SPEC_HEALTH_CMD" ]; then + local cur_health spec_health + cur_health=$(normalize_health_cmd "$(container_health_cmd "$name")") + spec_health=$(normalize_health_cmd "$SPEC_HEALTH_CMD") + if [ "$cur_health" != "$spec_health" ]; then + action="RECREATE" + reasons+="healthcheck " + fi + fi + # Check URL/HOST env drift — catches stale network topology baked into # container env (fedimint April-11 bug: FM_P2P_URL pointed at old IP). # Only checks URL-shaped keys; other env drift (passwords rotated, etc.) @@ -342,6 +445,40 @@ reconcile() { done fi + # Check bind mounts. This catches companion UIs recreated from older specs, + # especially bitcoin-ui: its image intentionally does not bake nginx.conf, + # so the rendered RPC proxy config must be mounted from the host. + if [ "$action" = "OK" ] && [ -n "$SPEC_VOLUMES" ]; then + for v in $SPEC_VOLUMES; do + local mount_source mount_rest mount_target + mount_source="${v%%:*}" + mount_rest="${v#*:}" + mount_target="${mount_rest%%:*}" + [ -n "$mount_source" ] && [ -n "$mount_target" ] || continue + if ! container_has_mount "$name" "$mount_source" "$mount_target"; then + action="RECREATE" + reasons+="mount($mount_target) " + break + fi + done + fi + + # Rootless Podman can occasionally leave a container running while its + # rootlessport listener is gone. The container still looks healthy in + # `podman ps`, but host-network UIs and backend status probes fail against + # 127.0.0.1. Treat missing host listeners as spec drift. + if [ "$action" = "OK" ] && [ -n "$SPEC_PORTS" ]; then + for p in $SPEC_PORTS; do + local host_port="${p%%:*}" + [ -n "$host_port" ] || continue + if ! host_port_listening "$host_port"; then + action="RECREATE" + reasons+="port($host_port-not-listening) " + break + fi + done + fi + # Check if running if ! container_running "$name" && [ "$action" = "OK" ]; then action="START" @@ -476,7 +613,7 @@ ensure_secrets() { ensure_bitcoin_conf() { local BITCOIN_CONF="/var/lib/archipelago/bitcoin/bitcoin.conf" sudo mkdir -p /var/lib/archipelago/bitcoin 2>/dev/null - if [ ! -f "$BITCOIN_CONF" ] || ! grep -q "^rpcauth=" "$BITCOIN_CONF" 2>/dev/null; then + if [ ! -f "$BITCOIN_CONF" ] || ! sudo grep -q "^rpcauth=" "$BITCOIN_CONF" 2>/dev/null; then if ! $CHECK_ONLY && [ -n "$BITCOIN_RPC_PASS" ]; then local salt hash rpcauth salt=$(openssl rand -hex 16) @@ -491,10 +628,14 @@ BTCEOF info "Generated bitcoin.conf" fi fi - # Strip duplicate server/rpc/listen lines from existing conf to avoid conflicts with custom args - if [ -f "$BITCOIN_CONF" ]; then - sudo sed -i '/^server=/d; /^rpcbind=/d; /^rpcallowip=/d; /^rpcport=/d; /^listen=/d' "$BITCOIN_CONF" 2>/dev/null - fi + # Strip duplicate server/rpc/listen lines from existing conf files to avoid + # conflicts with custom args. Knots can persist runtime args in + # bitcoin_rw.conf, so clean both files. + for conf in "$BITCOIN_CONF" "/var/lib/archipelago/bitcoin/bitcoin_rw.conf"; do + if [ -f "$conf" ]; then + sudo sed -i '/^server=/d; /^txindex=/d; /^rpcbind=/d; /^rpcallowip=/d; /^rpcport=/d; /^listen=/d; /^bind=/d; /^dbcache=/d' "$conf" 2>/dev/null + fi + done sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null } @@ -531,6 +672,63 @@ LNDEOF fi } +# ── Ensure bitcoin-ui nginx.conf ──────────────────────────────────── +ensure_bitcoin_ui_nginx_conf() { + local CONF_DIR="/var/lib/archipelago/bitcoin-ui" + local CONF_PATH="$CONF_DIR/nginx.conf" + [ -n "$BITCOIN_RPC_PASS" ] || return + if $CHECK_ONLY; then + [ -f "$CONF_PATH" ] || info "Would generate bitcoin-ui nginx.conf" + return + fi + + local auth_b64 tmp + auth_b64=$(printf '%s' "${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASS}" | base64 | tr -d '\n') + sudo mkdir -p "$CONF_DIR" 2>/dev/null + tmp="${CONF_PATH}.tmp.$$" + sudo tee "$tmp" >/dev/null << EOF +server { + listen 8334; + server_name _; + root /usr/share/nginx/html; + index index.html; + + location /bitcoin-rpc/ { + proxy_pass http://127.0.0.1:8332/; + proxy_http_version 1.1; + proxy_set_header Host \$host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header Authorization "Basic ${auth_b64}"; + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "POST, GET, OPTIONS"; + add_header Access-Control-Allow-Headers "Content-Type, Authorization"; + if (\$request_method = OPTIONS) { return 204; } + } + + location /bitcoin-status { + proxy_pass http://127.0.0.1:5678/bitcoin-status; + proxy_http_version 1.1; + proxy_set_header Host \$host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + add_header Cache-Control "no-store"; + } + + location / { + try_files \$uri \$uri/ /index.html; + } +} +EOF + if ! sudo cmp -s "$tmp" "$CONF_PATH" 2>/dev/null; then + sudo mv "$tmp" "$CONF_PATH" + sudo chmod 644 "$CONF_PATH" + info "Generated bitcoin-ui nginx.conf" + else + sudo rm -f "$tmp" + fi +} + # ── Ensure BTCPay databases ───────────────────────────────────────── ensure_btcpay_db() { if container_running "archy-btcpay-db"; then @@ -548,8 +746,10 @@ START_TIME=$(date +%s) header "Phase 0: Prerequisites" ensure_secrets +detect_environment ensure_bitcoin_conf ensure_lnd_conf +ensure_bitcoin_ui_nginx_conf TIER_NAMES=("Databases" "Core Infrastructure" "Services" "Applications" "Frontend UIs") diff --git a/scripts/resilience/lib.sh b/scripts/resilience/lib.sh index 207842b1..89d2ec26 100755 --- a/scripts/resilience/lib.sh +++ b/scripts/resilience/lib.sh @@ -136,7 +136,7 @@ expected_containers_for() { ui_proxy_path_for() { case "$1" in bitcoin-knots|bitcoin-core) echo "/app/bitcoin-ui/" ;; - electrumx|electrs) echo "/app/electrs-ui/" ;; + electrumx|electrs) echo "/app/electrumx/" ;; lnd) echo "/app/lnd-ui/" ;; btcpay-server) echo "/app/btcpay/" ;; *) echo "/app/$1/" ;; diff --git a/scripts/self-update.sh b/scripts/self-update.sh index 34a49848..362f26c4 100755 --- a/scripts/self-update.sh +++ b/scripts/self-update.sh @@ -186,7 +186,7 @@ fi # for backward compatibility with older binaries that still look there. SCRIPTS_DEST="/opt/archipelago/scripts" sudo mkdir -p "$SCRIPTS_DEST" -for script in image-versions.sh reconcile-containers.sh container-specs.sh; do +for script in image-versions.sh reconcile-containers.sh container-specs.sh container-doctor.sh app-surface-smoke-test.sh bitcoin-stack-lifecycle-test.sh; do src="$REPO_DIR/scripts/$script" if [ -f "$src" ]; then sudo install -m 755 "$src" "$SCRIPTS_DEST/$script" @@ -299,6 +299,25 @@ if [ -f "$REPO_DIR/image-recipe/configs/archipelago.service" ]; then fi fi +# Keep the doctor timer/service current too. Container uptime fixes rely on +# these units as much as on the helper scripts themselves. +DOCTOR_UNITS_CHANGED=false +for unit in archipelago-doctor.service archipelago-doctor.timer; do + src="$REPO_DIR/image-recipe/configs/$unit" + dst="/etc/systemd/system/$unit" + [ -f "$src" ] || continue + if [ ! -f "$dst" ] || ! diff -q "$src" "$dst" &>/dev/null; then + sudo install -m 644 "$src" "$dst" + DOCTOR_UNITS_CHANGED=true + ok "Updated $unit" + fi +done +if [ "$DOCTOR_UNITS_CHANGED" = "true" ]; then + sudo systemctl daemon-reload + sudo systemctl enable --now archipelago-doctor.timer 2>>"$LOG_FILE" || \ + warn "Failed to enable archipelago-doctor.timer" +fi + # Install/refresh tmpfiles.d rules. The logs rule creates # /var/log/archipelago/ + container-installs.log with archipelago:archipelago # ownership so the non-root backend can append install audit lines.