diff --git a/apps/immich/manifest.yml b/apps/immich/manifest.yml index e29c7c01..73fff674 100644 --- a/apps/immich/manifest.yml +++ b/apps/immich/manifest.yml @@ -30,7 +30,13 @@ app: disk_limit: 200Gi security: - capabilities: [] + # Runs as container root over a data tree the legacy installer chowned + # to the subuid range (host 100000 = container uid 1). Without + # DAC_OVERRIDE the server EACCESes writing upload/encoded-video the + # moment the container is recreated against this manifest (latent until + # the 2026-07-05 secret-env migration recreated it). Same cap set as + # immich-postgres minus the setuid pair it doesn't use. + capabilities: [CHOWN, DAC_OVERRIDE, FOWNER] readonly_root: false network_policy: isolated diff --git a/core/Cargo.lock b/core/Cargo.lock index af604ddc..68425044 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -169,6 +169,7 @@ dependencies = [ "async-trait", "chrono", "futures", + "hex", "hyper 0.14.32", "indexmap", "log", @@ -176,6 +177,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2 0.10.9", "thiserror 1.0.69", "tokio", "tracing", diff --git a/core/archipelago/src/api/rpc/auth.rs b/core/archipelago/src/api/rpc/auth.rs index 5b275ce4..b68519f2 100644 --- a/core/archipelago/src/api/rpc/auth.rs +++ b/core/archipelago/src/api/rpc/auth.rs @@ -1,4 +1,6 @@ -use super::{RpcHandler, DEV_DEFAULT_PASSWORD}; +use super::RpcHandler; +#[cfg(debug_assertions)] +use super::DEV_DEFAULT_PASSWORD; use anyhow::Result; impl RpcHandler { @@ -14,7 +16,10 @@ impl RpcHandler { let is_setup = self.auth_manager.is_setup().await?; if !is_setup { - // Dev mode: allow default password so UI can log in without running setup + // Dev BUILDS only: allow the default password so the UI can log + // in without running setup. cfg-gated so no release binary can + // carry the bypass, whatever its runtime config says. + #[cfg(debug_assertions)] if self.config.dev_mode && password == DEV_DEFAULT_PASSWORD { tracing::info!("[onboarding] login via dev default password"); return Ok(serde_json::Value::Null); diff --git a/core/archipelago/src/api/rpc/middleware.rs b/core/archipelago/src/api/rpc/middleware.rs index 0c9bd6ae..da078543 100644 --- a/core/archipelago/src/api/rpc/middleware.rs +++ b/core/archipelago/src/api/rpc/middleware.rs @@ -179,13 +179,91 @@ pub(super) fn extract_cookie(headers: &hyper::HeaderMap, name: &str) -> Option IpAddr { +/// The TCP peer address of the connection a request arrived on, injected +/// into request extensions by the server accept loop. +#[derive(Debug, Clone, Copy)] +pub struct PeerAddr(pub std::net::SocketAddr); + +/// Extract the client IP for rate limiting. +/// +/// `X-Real-IP`/`X-Forwarded-For` are only honored when the connection +/// itself comes from loopback — i.e. from our local nginx, which sets +/// `X-Real-IP $remote_addr`. On a direct connection (the FIPS peer +/// listener, or anything that isn't the local proxy) the headers are +/// client-supplied, so trusting them let an attacker rotate per-request +/// "IPs" and defeat the login rate limiter; there we use the socket +/// address instead. +pub(super) fn extract_client_ip(parts: &hyper::http::request::Parts) -> IpAddr { + let socket_ip = parts.extensions.get::().map(|p| p.0.ip()); + match socket_ip { + Some(ip) if ip.is_loopback() => forwarded_client_ip(&parts.headers).unwrap_or(ip), + Some(ip) => ip, + // No socket info recorded (shouldn't happen in the server path); + // fall back to the pre-extension behavior. + None => forwarded_client_ip(&parts.headers) + .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + } +} + +/// The proxy-reported client IP, if a forwarded header carries one. +fn forwarded_client_ip(headers: &hyper::HeaderMap) -> Option { headers .get("x-real-ip") .or_else(|| headers.get("x-forwarded-for")) .and_then(|v| v.to_str().ok()) .and_then(|s| s.split(',').next()) .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)) +} + +#[cfg(test)] +mod client_ip_tests { + use super::*; + use std::net::SocketAddr; + + fn parts_with( + peer: Option<&str>, + real_ip: Option<&str>, + ) -> hyper::http::request::Parts { + let mut builder = hyper::Request::builder().uri("/rpc/v1"); + if let Some(ip) = real_ip { + builder = builder.header("x-real-ip", ip); + } + let (mut parts, _) = builder.body(()).unwrap().into_parts(); + if let Some(addr) = peer { + parts + .extensions + .insert(PeerAddr(addr.parse::().unwrap())); + } + parts + } + + #[test] + fn loopback_connection_trusts_forwarded_header() { + // nginx on loopback forwards the real client IP — use it. + let parts = parts_with(Some("127.0.0.1:44412"), Some("192.168.1.50")); + assert_eq!( + extract_client_ip(&parts), + "192.168.1.50".parse::().unwrap() + ); + } + + #[test] + fn direct_connection_ignores_spoofed_header() { + // A direct (non-proxy) client rotating X-Real-IP per request must + // still bucket under its socket address. + let parts = parts_with(Some("203.0.113.9:9999"), Some("10.0.0.1")); + assert_eq!( + extract_client_ip(&parts), + "203.0.113.9".parse::().unwrap() + ); + } + + #[test] + fn loopback_connection_without_header_uses_socket_ip() { + let parts = parts_with(Some("127.0.0.1:5000"), None); + assert_eq!( + extract_client_ip(&parts), + "127.0.0.1".parse::().unwrap() + ); + } } diff --git a/core/archipelago/src/api/rpc/mod.rs b/core/archipelago/src/api/rpc/mod.rs index e474cbe6..d0328b82 100644 --- a/core/archipelago/src/api/rpc/mod.rs +++ b/core/archipelago/src/api/rpc/mod.rs @@ -58,9 +58,13 @@ use middleware::{ derive_csrf_token, extract_client_ip, extract_cookie, sanitize_error_message, CACHEABLE_METHODS, UNAUTHENTICATED_METHODS, }; +pub use middleware::PeerAddr; use response::{cookie_header, json_response, ResponseCache, RpcError, RpcRequest, RpcResponse}; /// Default dev password when no user is set up (matches mock-backend). +/// Dev builds only — the pre-setup login bypass that reads this is +/// cfg-gated out of release binaries. +#[cfg(debug_assertions)] pub(crate) const DEV_DEFAULT_PASSWORD: &str = "password123"; pub struct RpcHandler { @@ -369,7 +373,7 @@ impl RpcHandler { // Rate limit login attempts if rpc_req.method == "auth.login" { - let client_ip = extract_client_ip(&parts.headers); + let client_ip = extract_client_ip(&parts); if !self.login_rate_limiter.check(client_ip).await { return Ok(self.rate_limit_response()); } @@ -377,7 +381,7 @@ impl RpcHandler { // Rate limit sensitive endpoints { - let client_ip = extract_client_ip(&parts.headers); + let client_ip = extract_client_ip(&parts); if !self .endpoint_rate_limiter .check(&rpc_req.method, client_ip) @@ -451,7 +455,7 @@ impl RpcHandler { let mut response = json_response(StatusCode::OK, &resp_body); // Post-dispatch: set cookies for auth-related methods - let client_ip = extract_client_ip(&parts.headers); + let client_ip = extract_client_ip(&parts); self.apply_auth_cookies( &rpc_req.method, &mut rpc_resp, diff --git a/core/archipelago/src/api/rpc/package/config.rs b/core/archipelago/src/api/rpc/package/config.rs index 214e3ec6..c41f63dd 100644 --- a/core/archipelago/src/api/rpc/package/config.rs +++ b/core/archipelago/src/api/rpc/package/config.rs @@ -94,35 +94,11 @@ async fn dynamic_app_config( )) } -/// Trusted Docker registries. Only images from these sources are allowed. -#[allow(dead_code)] -pub(super) const TRUSTED_REGISTRIES: &[&str] = &[ - "docker.io/", - "ghcr.io/", - "localhost/", - "git.tx1138.com/", - "146.59.87.168:3000/", -]; - -/// Validate Docker image against trusted registry allowlist. +/// Validate a Docker image reference. Delegates to the shared policy in +/// `container::image_policy` — the same rules the orchestrator enforces at +/// its pull sites, so the two layers can't drift apart. pub(super) fn is_valid_docker_image(image: &str) -> bool { - if image.is_empty() || image.len() > 256 { - return false; - } - // Reject shell metacharacters - let dangerous_chars = ['&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r']; - if image.chars().any(|c| dangerous_chars.contains(&c)) { - return false; - } - // Must come from a trusted registry — match the exact domain, not just prefix - let registry = match image.split('/').next() { - Some(r) => r, - None => return false, - }; - matches!( - registry, - "docker.io" | "ghcr.io" | "localhost" | "git.tx1138.com" | "146.59.87.168:3000" - ) + crate::container::image_policy::is_valid_docker_image(image) } /// Per-app Linux capabilities needed beyond the default cap-drop=ALL. diff --git a/core/archipelago/src/api/rpc/package/dependencies.rs b/core/archipelago/src/api/rpc/package/dependencies.rs index 05a89fa7..ccb8e752 100644 --- a/core/archipelago/src/api/rpc/package/dependencies.rs +++ b/core/archipelago/src/api/rpc/package/dependencies.rs @@ -472,7 +472,7 @@ pub(super) async fn check_bitcoin_pruning_compatibility(package_id: &str) -> Res tokio::time::sleep(std::time::Duration::from_secs(2)).await; } - if detect_disk_gb() < ARCHIVAL_BITCOIN_DISK_GB { + if detect_disk_gb().await < ARCHIVAL_BITCOIN_DISK_GB { anyhow::bail!(archival_bitcoin_required_message(package_id)); } @@ -497,10 +497,11 @@ fn check_blockchain_info_for_pruning(package_id: &str, json: &serde_json::Value) Ok(()) } -fn detect_disk_gb() -> u64 { - let output = std::process::Command::new("df") +async fn detect_disk_gb() -> u64 { + let output = tokio::process::Command::new("df") .args(["-BG", "/var/lib/archipelago"]) - .output(); + .output() + .await; let Ok(output) = output else { return u64::MAX; }; diff --git a/core/archipelago/src/api/rpc/package/install.rs b/core/archipelago/src/api/rpc/package/install.rs index e01598ab..8dc43714 100644 --- a/core/archipelago/src/api/rpc/package/install.rs +++ b/core/archipelago/src/api/rpc/package/install.rs @@ -2196,13 +2196,14 @@ async fn ensure_host_port_listener( container_name: &str, runtime_ports: &[String], ) -> Result<()> { - let Some(port) = runtime_ports + let mut port = runtime_ports .first() .and_then(|p| p.split(':').next()) - .and_then(|p| p.parse::().ok()) - .or_else(|| published_host_port(container_name)) - .or_else(|| required_host_port(package_id)) - else { + .and_then(|p| p.parse::().ok()); + if port.is_none() { + port = published_host_port(container_name).await; + } + let Some(port) = port.or_else(|| required_host_port(package_id)) else { return Ok(()); }; @@ -2248,10 +2249,11 @@ async fn ensure_host_port_listener( )) } -fn published_host_port(container_name: &str) -> Option { - let output = std::process::Command::new("podman") +async fn published_host_port(container_name: &str) -> Option { + let output = tokio::process::Command::new("podman") .args(["port", container_name]) .output() + .await .ok()?; if !output.status.success() { return None; diff --git a/core/archipelago/src/api/rpc/system/handlers.rs b/core/archipelago/src/api/rpc/system/handlers.rs index 512f2f3a..5ddfa4a4 100644 --- a/core/archipelago/src/api/rpc/system/handlers.rs +++ b/core/archipelago/src/api/rpc/system/handlers.rs @@ -575,7 +575,7 @@ impl RpcHandler { // Restart the service via systemd tokio::spawn(async { tokio::time::sleep(std::time::Duration::from_secs(2)).await; - let _ = std::process::Command::new("sudo") + let _ = tokio::process::Command::new("sudo") .args(["systemctl", "restart", "archipelago"]) .spawn(); }); diff --git a/core/archipelago/src/config.rs b/core/archipelago/src/config.rs index dea0a949..ad1a1b2d 100644 --- a/core/archipelago/src/config.rs +++ b/core/archipelago/src/config.rs @@ -81,10 +81,11 @@ pub struct Config { impl Config { /// Detect primary host IP (first non-loopback IPv4) - fn detect_host_ip() -> Result { - let output = std::process::Command::new("hostname") + async fn detect_host_ip() -> Result { + let output = tokio::process::Command::new("hostname") .args(["-I"]) .output() + .await .context("Failed to run hostname -I")?; let s = String::from_utf8_lossy(&output.stdout); let ip = s @@ -210,7 +211,9 @@ impl Config { if let Ok(ip) = std::env::var("ARCHIPELAGO_HOST_IP") { config.host_ip = ip; } else { - config.host_ip = Self::detect_host_ip().unwrap_or_else(|_| "127.0.0.1".to_string()); + config.host_ip = Self::detect_host_ip() + .await + .unwrap_or_else(|_| "127.0.0.1".to_string()); } // Ensure data directory exists diff --git a/core/archipelago/src/container/image_policy.rs b/core/archipelago/src/container/image_policy.rs new file mode 100644 index 00000000..e8baf589 --- /dev/null +++ b/core/archipelago/src/container/image_policy.rs @@ -0,0 +1,95 @@ +//! Trusted-registry policy for container image references — the single +//! source of truth. The RPC boundary (`api::rpc::package::config`) and the +//! orchestrator's pull sites both validate against this, so a catalog- or +//! manifest-supplied ref can't reach `pull_image` unchecked (§A of the +//! 1.8.0 hardening plan). + +/// Registries images may be pulled from with an explicit host part. +pub const TRUSTED_REGISTRIES: &[&str] = &[ + "docker.io", + "ghcr.io", + "localhost", + "git.tx1138.com", + "146.59.87.168:3000", +]; + +/// Validate a container image reference. +/// +/// Accepts: +/// * refs whose explicit registry host is on [`TRUSTED_REGISTRIES`] +/// (`docker.io/grafana/grafana`, `146.59.87.168:3000/archy/x:1`), and +/// * registry-less Docker Hub shorthand (`nginx`, `grafana/grafana`) — +/// the first segment has no `.`/`:` so it cannot name an attacker host; +/// resolution follows the host's registries.conf search order. +/// +/// Rejects empty/oversized refs, shell metacharacters, and any ref whose +/// explicit registry host is not on the allowlist. +pub fn is_valid_docker_image(image: &str) -> bool { + if image.is_empty() || image.len() > 256 { + return false; + } + // Reject shell metacharacters + let dangerous_chars = [ + '&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r', ' ', '\t', + ]; + if image.chars().any(|c| dangerous_chars.contains(&c)) { + return false; + } + let first_segment = match image.split('/').next() { + Some(r) if !r.is_empty() => r, + _ => return false, + }; + if TRUSTED_REGISTRIES.contains(&first_segment) { + return true; + } + // No dot/colon in the first segment ⇒ it's a Docker Hub namespace or a + // bare repo name, not a registry host — allowed. Anything that *looks* + // like a host (has a dot or port) but isn't allowlisted is rejected. + !first_segment.contains('.') && !first_segment.contains(':') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_trusted_registries() { + for img in [ + "docker.io/library/nginx:1.25", + "ghcr.io/owner/app:latest", + "localhost/archy-dev:1", + "git.tx1138.com/lfg2025/x:2", + "146.59.87.168:3000/archy/bitcoin-knots:28.1", + ] { + assert!(is_valid_docker_image(img), "{img} should be accepted"); + } + } + + #[test] + fn accepts_docker_hub_shorthand() { + for img in ["nginx", "grafana/grafana:11.2.0", "lightninglabs/lnd:v0.18"] { + assert!(is_valid_docker_image(img), "{img} should be accepted"); + } + } + + #[test] + fn rejects_untrusted_registry_hosts() { + for img in [ + "evil.com/backdoor:latest", + "203.0.113.7:5000/x", + "registry.gitlab.com/x/y", + "quay.io/x/y", + ] { + assert!(!is_valid_docker_image(img), "{img} should be rejected"); + } + } + + #[test] + fn rejects_malformed_refs() { + assert!(!is_valid_docker_image("")); + assert!(!is_valid_docker_image(&"a".repeat(257))); + assert!(!is_valid_docker_image("docker.io/x; rm -rf /")); + assert!(!is_valid_docker_image("docker.io/$(curl evil)")); + assert!(!is_valid_docker_image("/leading-slash")); + } +} diff --git a/core/archipelago/src/container/mod.rs b/core/archipelago/src/container/mod.rs index 1ee128e7..f1281f9f 100644 --- a/core/archipelago/src/container/mod.rs +++ b/core/archipelago/src/container/mod.rs @@ -7,6 +7,7 @@ pub mod dev_orchestrator; pub mod docker_packages; pub mod filebrowser; pub mod hooks; +pub mod image_policy; pub mod image_versions; pub mod lnd; pub mod prod_orchestrator; diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 5825afa6..f72a3d8a 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -32,7 +32,6 @@ use async_trait::async_trait; use std::collections::{HashMap, HashSet}; use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; -use std::process::Command; use std::sync::Arc; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::{Mutex, RwLock}; @@ -1065,6 +1064,11 @@ pub struct ProdContainerOrchestrator { /// false so the legacy path remains the production path until the /// 5× lifecycle harness goes green against the new path. use_quadlet_backends: bool, + /// app_id → last secret-env content hash pushed to the runtime's + /// secret store. Makes steady-state reconciles free of podman + /// secret calls; a rotation (hash change) falls through and + /// re-registers. + env_secret_cache: Mutex>, #[cfg(test)] test_disk_gb: Option, #[cfg(test)] @@ -1127,6 +1131,7 @@ impl ProdContainerOrchestrator { lnd_paths: lnd::EnsurePaths::default(), secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"), use_quadlet_backends: config.use_quadlet_backends, + env_secret_cache: Mutex::new(HashMap::new()), #[cfg(test)] test_disk_gb: None, #[cfg(test)] @@ -1148,6 +1153,7 @@ impl ProdContainerOrchestrator { lnd_paths: lnd::EnsurePaths::default(), secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"), use_quadlet_backends: false, + env_secret_cache: Mutex::new(HashMap::new()), test_disk_gb: None, test_bitcoin_host: None, } @@ -1440,7 +1446,7 @@ impl ProdContainerOrchestrator { .collect() }; let mut report = ReconcileReport::default(); - let disk_gb = self.disk_gb(); + let disk_gb = self.disk_gb().await; // Register every candidate before the (sequential, possibly slow) // pass so the scanner overlays queued-but-down apps as Restarting // instead of Stopped. Each app is deregistered as its turn finishes, @@ -1620,7 +1626,7 @@ impl ProdContainerOrchestrator { } let mut resolved_manifest = lm.manifest.clone(); - self.resolve_dynamic_env(&mut resolved_manifest)?; + self.resolve_dynamic_env(&mut resolved_manifest).await?; let name = compute_container_name(&lm.manifest); // An explicitly user-stopped app MUST stay stopped. The reconcile filter @@ -1970,7 +1976,7 @@ impl ProdContainerOrchestrator { async fn install_fresh(&self, lm: &LoadedManifest) -> Result<()> { self.ensure_app_secrets(&lm.manifest.app.id).await?; let mut resolved_manifest = lm.manifest.clone(); - self.resolve_dynamic_env(&mut resolved_manifest)?; + self.resolve_dynamic_env(&mut resolved_manifest).await?; resolve_catalog_image(&mut resolved_manifest); let resolved = resolved_manifest.app.container.resolve().ok_or_else(|| { @@ -1986,6 +1992,16 @@ impl ProdContainerOrchestrator { image_signature, .. } => { + // §A: validate at the pull site, not just the RPC boundary — + // catalog/manifest-supplied refs reach here without ever + // passing package::config's check. + if !crate::container::image_policy::is_valid_docker_image(&image) { + anyhow::bail!( + "refusing to pull image {:?} for {}: not from a trusted registry", + image, + lm.manifest.app.id + ); + } self.runtime .pull_image(&image, image_signature.as_deref()) .await @@ -2263,7 +2279,7 @@ impl ProdContainerOrchestrator { // Re-render the manifest with dynamic env baked in, then go // through the same install path a fresh install would. let mut resolved = lm.manifest.clone(); - self.resolve_dynamic_env(&mut resolved)?; + self.resolve_dynamic_env(&mut resolved).await?; self.install_via_quadlet(&resolved, name) .await .with_context(|| format!("Phase 3.3: re-install {name} via Quadlet"))?; @@ -2329,7 +2345,7 @@ impl ProdContainerOrchestrator { let restart_required = quadlet::contains_stale_health_gate(&old_body); let mut resolved = lm.manifest.clone(); - self.resolve_dynamic_env(&mut resolved)?; + self.resolve_dynamic_env(&mut resolved).await?; // Same catalog/pinned-version image resolution the installer applies, so // the drift re-render doesn't revert a pinned version back to the // manifest's shipped `:latest` tag on the next reconcile tick. @@ -2404,7 +2420,7 @@ impl ProdContainerOrchestrator { for dep in dependencies { let mut resolved = dep.manifest.clone(); - self.resolve_dynamic_env(&mut resolved)?; + self.resolve_dynamic_env(&mut resolved).await?; let name = compute_container_name(&dep.manifest); if self.runtime.get_container_status(&name).await.is_err() { continue; @@ -2431,6 +2447,14 @@ impl ProdContainerOrchestrator { image_signature, .. } => { + // Same trusted-registry gate as install_fresh (§A). + if !crate::container::image_policy::is_valid_docker_image(&image) { + anyhow::bail!( + "refusing to pull image {:?} for {}: not from a trusted registry", + image, + lm.manifest.app.id + ); + } let exists = match self.runtime.image_exists(&image).await { Ok(exists) => exists, Err(err) => { @@ -2588,7 +2612,7 @@ impl ProdContainerOrchestrator { } .read("bitcoin-rpc-password") .context("lnd pre-start: read bitcoin RPC password")?; - let bitcoin_host = self.bitcoin_host(); + let bitcoin_host = self.bitcoin_host().await; let outcome = lnd::ensure_config(&self.lnd_paths, &rpc_pass, &bitcoin_host) .await .context("lnd pre-start: ensure lnd.conf")?; @@ -2706,36 +2730,58 @@ impl ProdContainerOrchestrator { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } - fn detect_host_facts(&self) -> HostFacts { - let host_ip = Self::detect_host_ip().unwrap_or_else(|| "127.0.0.1".to_string()); - let host_mdns = Self::detect_host_mdns(); - let disk_gb = self.disk_gb(); - HostFacts { - host_ip, - host_mdns, - disk_gb, - // Cheap default; resolve_dynamic_env fills the real node name on - // demand (it costs a podman call) only for manifests that use - // {{BITCOIN_HOST}}, rather than every app on every reconcile. - bitcoin_host: "bitcoin-knots".to_string(), + async fn detect_host_facts(&self) -> HostFacts { + // Unit tests run under tokio's paused clock; awaiting a real + // subprocess there deadlocks against auto-advanced timers (the old + // BLOCKING detection only worked by never yielding). Canned facts. + #[cfg(test)] + { + return HostFacts { + host_ip: "127.0.0.1".to_string(), + host_mdns: "test.local".to_string(), + disk_gb: self.test_disk_gb.unwrap_or(1000), + bitcoin_host: "bitcoin-knots".to_string(), + }; + } + #[allow(unreachable_code)] + { + let host_ip = Self::detect_host_ip() + .await + .unwrap_or_else(|| "127.0.0.1".to_string()); + let host_mdns = Self::detect_host_mdns().await; + let disk_gb = self.disk_gb().await; + HostFacts { + host_ip, + host_mdns, + disk_gb, + // Cheap default; resolve_dynamic_env fills the real node name on + // demand (it costs a podman call) only for manifests that use + // {{BITCOIN_HOST}}, rather than every app on every reconcile. + bitcoin_host: "bitcoin-knots".to_string(), + } } } /// Container name of the running Bitcoin node (`bitcoin-knots` or /// `bitcoin-core`) for the `{{BITCOIN_HOST}}` derived-env placeholder. - /// Synchronous `podman ps` to match the surrounding host-fact detection; - /// defaults to `bitcoin-knots` when none is running (B12). - fn bitcoin_host(&self) -> String { + /// Defaults to `bitcoin-knots` when none is running (B12). + async fn bitcoin_host(&self) -> String { + // No real podman under the tests' paused clock (see detect_host_facts). #[cfg(test)] - if let Some(host) = &self.test_bitcoin_host { - return host.clone(); + { + return self + .test_bitcoin_host + .clone() + .unwrap_or_else(|| "bitcoin-knots".to_string()); } + #[allow(unreachable_code)] // Mirrors api::rpc::package::dependencies (the legacy install path); // both Bitcoin node variants are reachable on archy-net by name. const BITCOIN_NAMES: &[&str] = &["bitcoin-knots", "bitcoin-core", "bitcoin"]; - let names = Command::new("podman") + let names = tokio::process::Command::new("podman") .args(["ps", "--format", "{{.Names}}"]) .output() + .await .ok() .filter(|o| o.status.success()) .map(|o| String::from_utf8_lossy(&o.stdout).into_owned()) @@ -2753,8 +2799,12 @@ impl ProdContainerOrchestrator { self.test_bitcoin_host = Some(host.to_string()); } - fn detect_host_ip() -> Option { - let output = Command::new("hostname").arg("-I").output().ok()?; + async fn detect_host_ip() -> Option { + let output = tokio::process::Command::new("hostname") + .arg("-I") + .output() + .await + .ok()?; if !output.status.success() { return None; } @@ -2762,9 +2812,10 @@ impl ProdContainerOrchestrator { stdout.split_whitespace().next().map(|s| s.to_string()) } - fn detect_host_mdns() -> String { - let hostname = Command::new("hostname") + async fn detect_host_mdns() -> String { + let hostname = tokio::process::Command::new("hostname") .output() + .await .ok() .and_then(|o| { if o.status.success() { @@ -2782,13 +2833,18 @@ impl ProdContainerOrchestrator { } } - fn detect_disk_gb() -> u64 { + async fn detect_disk_gb() -> u64 { let target = if Path::new("/var/lib/archipelago").exists() { "/var/lib/archipelago" } else { "/" }; - let output = match Command::new("df").arg("-k").arg(target).output() { + let output = match tokio::process::Command::new("df") + .arg("-k") + .arg(target) + .output() + .await + { Ok(o) if o.status.success() => o, _ => return 0, }; @@ -2808,12 +2864,16 @@ impl ProdContainerOrchestrator { kb / 1_000_000 } - fn disk_gb(&self) -> u64 { + async fn disk_gb(&self) -> u64 { + // No real df under the tests' paused clock (see detect_host_facts). #[cfg(test)] - if let Some(disk_gb) = self.test_disk_gb { - return disk_gb; + { + return self.test_disk_gb.unwrap_or(1000); + } + #[allow(unreachable_code)] + { + Self::detect_disk_gb().await } - Self::detect_disk_gb() } /// Ensure app-specific secrets exist *before* env resolution. The Bitcoin @@ -2838,14 +2898,21 @@ impl ProdContainerOrchestrator { Ok(()) } - fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> { + async fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> { + // Idempotency guard: partitioning already ran on this instance. + // Re-running would re-taint against an environment that no longer + // contains the composite entries and silently drop them. Callers + // always resolve a fresh clone, so this only trips on misuse. + if !manifest.app.container.secret_env_refs.is_empty() { + return Ok(()); + } // Materialise any manifest-declared generated secrets before they're // read below. This is the single chokepoint every install/reconcile // path funnels through, so an app's secrets exist by the time its // `secret_env` resolves — no per-app code, no host provisioning. crate::container::secrets::ensure_generated_secrets(&self.secrets_dir, manifest)?; - let mut facts = self.detect_host_facts(); + let mut facts = self.detect_host_facts().await; // Only pay the podman cost to detect Knots-vs-Core when this manifest // actually templates the Bitcoin node into its env (mempool — B12). if manifest @@ -2855,18 +2922,23 @@ impl ProdContainerOrchestrator { .iter() .any(|e| e.template.contains("{{BITCOIN_HOST}}")) { - facts.bitcoin_host = self.bitcoin_host(); + facts.bitcoin_host = self.bitcoin_host().await; } let mut env = manifest.app.environment.clone(); env.extend(manifest.app.container.resolve_derived_env(&facts)); + if manifest.app.id == "fedimint" || manifest.app.id == "fedimintd" { + env.retain(|entry| !entry.starts_with("FM_BITCOIND_URL=")); + env.push("FM_BITCOIND_URL=http://bitcoin-knots:8332".to_string()); + } + let provider = FileSecretsProvider { root: self.secrets_dir.clone(), }; - let secrets = manifest + let secret_pairs = manifest .app .container - .resolve_secret_env(&provider) + .resolve_secret_env_pairs(&provider) .map_err(|e| anyhow::anyhow!(e.to_string())) .with_context(|| { format!( @@ -2875,36 +2947,57 @@ impl ProdContainerOrchestrator { self.secrets_dir.display() ) })?; - env.extend(secrets); - if manifest.app.id == "fedimint" || manifest.app.id == "fedimintd" { - env.retain(|entry| !entry.starts_with("FM_BITCOIND_URL=")); - env.push("FM_BITCOIND_URL=http://bitcoin-knots:8332".to_string()); - } - Self::expand_env_placeholders(&mut env); - manifest.app.environment = env; - Ok(()) - } - fn expand_env_placeholders(env: &mut Vec) { - let values: HashMap = env - .iter() - .filter_map(|entry| { - let (key, value) = entry.split_once('=')?; - Some((key.to_string(), value.to_string())) - }) - .collect(); + // Secret values never merge into `environment` — they'd land in + // `podman inspect` output and Quadlet unit files on disk. Instead: + // expand ${KEY} placeholders (a plain entry that interpolates a + // secret is tainted and travels as a secret itself — btcpay's + // Password=${BTCPAY_DB_PASS} connection strings), keep the plain + // remainder as env, and hand the secret-bearing pairs to the + // runtime's secret store by reference. + let (plain, secret_bearing) = + archipelago_container::manifest::expand_and_partition_env(env, secret_pairs); + manifest.app.environment = plain; + if secret_bearing.is_empty() { + manifest.app.container.secret_env_refs = Vec::new(); + manifest.app.container.secret_env_hash = None; + } else { + let hash = + archipelago_container::manifest::secret_env_content_hash(&secret_bearing); + let app_id = manifest.app.id.clone(); + manifest.app.container.secret_env_refs = secret_bearing + .into_iter() + .map(|(key, value)| archipelago_container::manifest::SecretEnvRef { + secret_name: format!( + "archy-env-{}-{}", + app_id, + key.to_ascii_lowercase() + ), + env_key: key, + value, + }) + .collect(); + manifest.app.container.secret_env_hash = Some(hash.clone()); - for entry in env.iter_mut() { - let Some((key, value)) = entry.split_once('=') else { - continue; - }; - let mut expanded = value.to_string(); - for (placeholder_key, placeholder_value) in &values { - expanded = - expanded.replace(&format!("${{{}}}", placeholder_key), placeholder_value); + // Register/refresh the podman secrets. A per-app hash cache makes + // the steady-state reconcile free: podman is only consulted when + // the resolved content actually changed (or on first touch after + // boot). Mock runtimes no-op via the trait default. + let cached = self + .env_secret_cache + .lock() + .await + .get(&app_id) + .cloned(); + if cached.as_deref() != Some(hash.as_str()) { + self.runtime + .ensure_env_secrets(&manifest.app.container.secret_env_refs) + .await + .with_context(|| format!("registering env secrets for {app_id}"))?; + self.env_secret_cache.lock().await.insert(app_id, hash); } - *entry = format!("{}={}", key, expanded); } + Ok(()) } async fn container_env_drifted(&self, name: &str, manifest: &AppManifest) -> bool { @@ -2940,12 +3033,40 @@ impl ProdContainerOrchestrator { }) .collect(); - manifest.app.environment.iter().any(|entry| { + if manifest.app.environment.iter().any(|entry| { let Some((key, expected)) = entry.split_once('=') else { return false; }; current.get(key).map_or(true, |actual| actual != expected) - }) + }) { + return true; + } + + // Secret-backed env never appears in Config.Env (that's the point) — + // rotation is detected via the content-hash label stamped at create + // time. A pre-upgrade container has no label, which reads as drift + // and triggers the one-time recreate that scrubs its plaintext + // secrets out of `podman inspect`. + if let Some(expected_hash) = &manifest.app.container.secret_env_hash { + let fmt = format!( + "{{{{ index .Config.Labels \"{}\" }}}}", + archipelago_container::manifest::SECRET_ENV_HASH_LABEL + ); + let inspect = tokio::process::Command::new("podman") + .args(["inspect", name, "--format", &fmt]) + .output() + .await; + let Ok(output) = inspect else { + return false; + }; + if !output.status.success() { + return false; + } + if String::from_utf8_lossy(&output.stdout).trim() != expected_hash { + return true; + } + } + false } async fn container_command_drifted(&self, name: &str, manifest: &AppManifest) -> bool { @@ -3208,7 +3329,7 @@ impl ProdContainerOrchestrator { ) -> Result { let mut out = content.to_string(); if out.contains("{{HOST_IP}}") || out.contains("{{HOST_MDNS}}") { - let facts = self.detect_host_facts(); + let facts = self.detect_host_facts().await; out = out .replace("{{HOST_IP}}", &facts.host_ip) .replace("{{HOST_MDNS}}", &facts.host_mdns); @@ -3297,7 +3418,7 @@ impl ProdContainerOrchestrator { /// however the box is reached locally. (Generalised from the old per-app /// netbird TLS helper, deleted in #20 ph4: rsa:2048, 10-year, no per-app Rust.) async fn ensure_manifest_certs(&self, manifest: &AppManifest) -> Result<()> { - let facts = self.detect_host_facts(); + let facts = self.detect_host_facts().await; let render = |s: &str| { s.replace("{{HOST_IP}}", &facts.host_ip) .replace("{{HOST_MDNS}}", &facts.host_mdns) @@ -3594,7 +3715,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { self.ensure_app_secrets(app_id).await?; let name = compute_container_name(&lm.manifest); let mut resolved_manifest = lm.manifest.clone(); - self.resolve_dynamic_env(&mut resolved_manifest)?; + self.resolve_dynamic_env(&mut resolved_manifest).await?; let service = format!("{name}.service"); if self.quadlet_unit_exists(&name).await? { @@ -3830,6 +3951,9 @@ mod tests { images: StdMutex>, /// container_name -> env that create_container received. created_env: StdMutex>>, + /// container_name -> secret env refs that create_container received. + created_secret_refs: + StdMutex>>, /// If set, the next `build_image` call fails with this message. fail_build: StdMutex>, /// If set, `image_exists` fails for this image reference. @@ -3868,6 +3992,17 @@ mod tests { .cloned() .unwrap_or_default() } + fn created_secret_refs_for( + &self, + name: &str, + ) -> Vec { + self.created_secret_refs + .lock() + .unwrap() + .get(name) + .cloned() + .unwrap_or_default() + } } #[async_trait] @@ -3889,6 +4024,10 @@ mod tests { .lock() .unwrap() .insert(name.to_string(), manifest.app.environment.clone()); + self.created_secret_refs.lock().unwrap().insert( + name.to_string(), + manifest.app.container.secret_env_refs.clone(), + ); Ok(name.to_string()) } async fn start_container(&self, name: &str) -> Result<()> { @@ -4328,7 +4467,7 @@ app: orch.set_bitcoin_host_for_test(node); let mut manifest = AppManifest::parse(yaml).unwrap(); - orch.resolve_dynamic_env(&mut manifest).unwrap(); + orch.resolve_dynamic_env(&mut manifest).await.unwrap(); assert!( manifest @@ -4528,7 +4667,17 @@ app: assert!(env .iter() .any(|e| e.starts_with("FM_API_URL=ws://") && e.ends_with(":8174"))); - assert!(env.iter().any(|e| e == "FM_BITCOIND_PASSWORD=secret-pass")); + // The secret must NOT ride in plain env (that's the podman-inspect / + // quadlet-unit-file leak this pipeline exists to close) — it travels + // as a secret ref with the value bound for the podman secret store. + assert!(!env.iter().any(|e| e.starts_with("FM_BITCOIND_PASSWORD="))); + let refs = rt.created_secret_refs_for("fedimint"); + let r = refs + .iter() + .find(|r| r.env_key == "FM_BITCOIND_PASSWORD") + .expect("secret env must arrive as a ref"); + assert_eq!(r.value, "secret-pass"); + assert_eq!(r.secret_name, "archy-env-fedimint-fm_bitcoind_password"); } #[tokio::test] diff --git a/core/archipelago/src/container/quadlet.rs b/core/archipelago/src/container/quadlet.rs index 79aa6a99..cfe1f42b 100644 --- a/core/archipelago/src/container/quadlet.rs +++ b/core/archipelago/src/container/quadlet.rs @@ -142,6 +142,14 @@ pub struct QuadletUnit { // companion's rendered bytes are unchanged from before this PR. pub ports: Vec<(u16, u16, String)>, pub environment: Vec, + /// Secret-backed env: (env_key, podman secret name). Rendered as + /// `Secret=,type=env,target=` so the VALUE never lands in + /// this unit file on disk — only a reference to the podman secret + /// store. The orchestrator registers the secrets before writing units. + pub secret_env: Vec<(String, String)>, + /// Container labels (`Label=k=v`). Carries the secret-env content hash + /// for rotation-drift detection. + pub labels: Vec<(String, String)>, pub devices: Vec, pub add_hosts: Vec<(String, String)>, pub network_aliases: Vec, @@ -247,6 +255,12 @@ impl QuadletUnit { // accepts that form on a single Environment= line per pair. let _ = writeln!(s, "Environment={}", quote_environment(env)); } + for (key, secret_name) in &self.secret_env { + let _ = writeln!(s, "Secret={secret_name},type=env,target={key}"); + } + for (k, v) in &self.labels { + let _ = writeln!(s, "Label={k}={v}"); + } for dev in &self.devices { let _ = writeln!(s, "AddDevice={dev}"); } @@ -415,6 +429,23 @@ impl QuadletUnit { .map(|p| (p.host, p.container, p.protocol.clone())) .collect(), environment: app.environment.clone(), + secret_env: app + .container + .secret_env_refs + .iter() + .map(|r| (r.env_key.clone(), r.secret_name.clone())) + .collect(), + labels: app + .container + .secret_env_hash + .iter() + .map(|h| { + ( + archipelago_container::manifest::SECRET_ENV_HASH_LABEL.to_string(), + h.clone(), + ) + }) + .collect(), devices: app.devices.clone(), add_hosts: vec![("host.archipelago".into(), "10.89.0.1".into())], // Container always answers to its own name; manifest extras add the @@ -847,6 +878,24 @@ mod tests { use super::*; use tempfile::tempdir; + #[test] + fn render_emits_secret_env_by_reference_never_value() { + let u = QuadletUnit { + name: "t".into(), + description: "t".into(), + image: "img".into(), + secret_env: vec![("DB_PASS".into(), "archy-env-app-db_pass".into())], + labels: vec![("io.archipelago.secret-env-hash".into(), "abc123".into())], + ..QuadletUnit::default() + }; + let s = u.render(); + assert!(s.contains("Secret=archy-env-app-db_pass,type=env,target=DB_PASS")); + assert!(s.contains("Label=io.archipelago.secret-env-hash=abc123")); + // the secret VALUE never had a path into this unit — but guard the + // env channel anyway: no Environment= line may mention the key + assert!(!s.contains("Environment=DB_PASS")); + } + fn sample_unit() -> QuadletUnit { QuadletUnit { name: "archy-bitcoin-ui".into(), diff --git a/core/archipelago/src/mesh/listener/decode.rs b/core/archipelago/src/mesh/listener/decode.rs index 3eb764cf..babc3c77 100644 --- a/core/archipelago/src/mesh/listener/decode.rs +++ b/core/archipelago/src/mesh/listener/decode.rs @@ -563,7 +563,9 @@ pub(super) async fn handle_identity_received( // Update peer record let peer = MeshPeer { contact_id, - advert_name: format!("Archy-{}", &did[8..16.min(did.len())]), + // .get(): a malformed DID shorter than the "did:key:" prefix must + // not panic the listener on a radio-supplied string. + advert_name: format!("Archy-{}", did.get(8..16.min(did.len())).unwrap_or(did)), did: Some(did.to_string()), pubkey_hex: Some(ed_pubkey_hex.to_string()), // The advert signature was verified above, so this is an authenticated diff --git a/core/archipelago/src/mesh/message_types.rs b/core/archipelago/src/mesh/message_types.rs index 9b9b36b7..43cee9ec 100644 --- a/core/archipelago/src/mesh/message_types.rs +++ b/core/archipelago/src/mesh/message_types.rs @@ -258,7 +258,31 @@ impl TypedEnvelope { } } - /// Verify signature if present. + /// Signing preimage v2: binds the anti-replay `seq` so a radio MITM + /// can't reorder/replay a signed message under a different sequence + /// number. v1 (legacy) covers only (t, v, ts). + fn signing_preimage_v2(&self) -> Vec { + let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4 + 8); + sign_data.push(self.t); + sign_data.extend_from_slice(&self.v); + sign_data.extend_from_slice(&self.ts.to_le_bytes()); + sign_data.extend_from_slice(&self.seq.to_le_bytes()); + sign_data + } + + fn signing_preimage_v1(&self) -> Vec { + let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4); + sign_data.push(self.t); + sign_data.extend_from_slice(&self.v); + sign_data.extend_from_slice(&self.ts.to_le_bytes()); + sign_data + } + + /// Verify signature if present. Accepts the seq-binding v2 preimage OR + /// the legacy (t, v, ts) preimage — senders still emit v1 until the + /// whole fleet verifies v2 (receivers hard-drop bad signatures, so + /// flipping the send side first would break mixed-fleet alerts). The + /// seq-tampering window closes only when the v1 arm is removed. pub fn verify_signature(&self, verifying_key: &ed25519_dalek::VerifyingKey) -> Result { let Some(sig_bytes) = &self.sig else { return Ok(false); @@ -266,13 +290,14 @@ impl TypedEnvelope { let signature = ed25519_dalek::Signature::from_slice(sig_bytes).context("Invalid signature bytes")?; - let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4); - sign_data.push(self.t); - sign_data.extend_from_slice(&self.v); - sign_data.extend_from_slice(&self.ts.to_le_bytes()); - + if verifying_key + .verify_strict(&self.signing_preimage_v2(), &signature) + .is_ok() + { + return Ok(true); + } verifying_key - .verify_strict(&sign_data, &signature) + .verify_strict(&self.signing_preimage_v1(), &signature) .context("Signature verification failed")?; Ok(true) } @@ -284,12 +309,25 @@ impl TypedEnvelope { /// Set the outbound sequence number. Called by the send path after the /// target's counter has been incremented. Safe to call AFTER `new_signed` - /// because the signature covers `(t, v, ts)` — not `seq`. + /// because the v1 signature covers `(t, v, ts)` — not `seq`. Once the + /// fleet is on a build whose `verify_signature` accepts the v2 preimage, + /// flip senders to sign AFTER seq allocation via `signed_with_seq`. pub fn with_seq(mut self, seq: u64) -> Self { self.seq = seq; self } + /// v2 sender path (NOT yet wired — see `verify_signature` for the fleet + /// migration order): set seq first, then sign binding it. + #[allow(dead_code)] + pub fn signed_with_seq(mut self, seq: u64, signing_key: &ed25519_dalek::SigningKey) -> Self { + use ed25519_dalek::Signer; + self.seq = seq; + let signature = signing_key.sign(&self.signing_preimage_v2()); + self.sig = Some(signature.to_bytes().to_vec()); + self + } + /// Encode to wire format: [0x02] [CBOR envelope]. pub fn to_wire(&self) -> Result> { let mut buf = Vec::new(); @@ -816,6 +854,37 @@ mod tests { assert!(envelope.verify_signature(&key.verifying_key()).is_err()); } + #[test] + fn test_v2_seq_bound_signature() { + use ed25519_dalek::SigningKey; + use rand::rngs::OsRng; + + let key = SigningKey::generate(&mut OsRng); + let envelope = TypedEnvelope::new(MeshMessageType::Alert, b"test".to_vec()) + .signed_with_seq(42, &key); + assert!(envelope.verify_signature(&key.verifying_key()).unwrap()); + + // v2 binds seq: replaying the signed envelope under a different + // sequence number must fail verification. + let mut replayed = envelope.clone(); + replayed.seq = 43; + assert!(replayed.verify_signature(&key.verifying_key()).is_err()); + } + + #[test] + fn test_v1_signature_survives_seq_set_after_signing() { + use ed25519_dalek::SigningKey; + use rand::rngs::OsRng; + + // Mixed-fleet compatibility: current senders sign first (v1 + // preimage, no seq) and allocate seq afterwards; verify must still + // accept that. + let key = SigningKey::generate(&mut OsRng); + let envelope = TypedEnvelope::new_signed(MeshMessageType::Alert, b"test".to_vec(), &key) + .with_seq(7); + assert!(envelope.verify_signature(&key.verifying_key()).unwrap()); + } + #[test] fn test_invoice_payload_roundtrip() { let invoice = InvoicePayload { diff --git a/core/archipelago/src/server.rs b/core/archipelago/src/server.rs index 807a27f7..4a52851d 100644 --- a/core/archipelago/src/server.rs +++ b/core/archipelago/src/server.rs @@ -1034,9 +1034,13 @@ async fn accept_loop( let permit = active_connections.clone().acquire_owned().await; tokio::spawn(async move { let _permit = permit; - let service = service_fn(move |req: hyper::Request| { + let service = service_fn(move |mut req: hyper::Request| { let handler = handler.clone(); async move { + // Record the TCP peer so rate limiting only trusts + // forwarded headers on loopback (nginx) connections. + req.extensions_mut() + .insert(crate::api::rpc::PeerAddr(peer_addr)); if peer_only && !is_peer_allowed_path(req.uri().path()) { let resp = hyper::Response::builder() .status(hyper::StatusCode::NOT_FOUND) diff --git a/core/archipelago/src/transport/fips.rs b/core/archipelago/src/transport/fips.rs index 15a28620..a922969f 100644 --- a/core/archipelago/src/transport/fips.rs +++ b/core/archipelago/src/transport/fips.rs @@ -23,26 +23,36 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; /// TTL keeps the result responsive to daemon flaps without pounding DBus. const AVAILABILITY_CACHE_TTL: Duration = Duration::from_secs(10); +/// Availability cache shared with the background probe thread, so the +/// sync `is_available()` hot path never blocks on `systemctl`. +struct AvailabilityCache { + available: AtomicBool, + probed_at_ms: AtomicU64, + probe_in_flight: AtomicBool, +} + pub struct FipsTransport { identity_dir: PathBuf, - available_cached: AtomicBool, - available_cached_at_ms: AtomicU64, + availability: std::sync::Arc, } impl FipsTransport { pub fn new(identity_dir: &Path) -> Self { Self { identity_dir: identity_dir.to_path_buf(), - available_cached: AtomicBool::new(false), - available_cached_at_ms: AtomicU64::new(0), + availability: std::sync::Arc::new(AvailabilityCache { + available: AtomicBool::new(false), + probed_at_ms: AtomicU64::new(0), + probe_in_flight: AtomicBool::new(false), + }), } } fn probe_daemon_active() -> bool { - // Cheap blocking probe: spawn `systemctl is-active` synchronously. - // Short-circuit if either the archipelago-managed unit or the - // upstream fips.service is active — legacy/dev nodes run only the - // upstream unit. + // Blocking probe — only ever run on a dedicated background thread + // (see is_available), never on a tokio worker. Short-circuit if + // either the archipelago-managed unit or the upstream fips.service + // is active — legacy/dev nodes run only the upstream unit. for unit in [ crate::fips::SERVICE_UNIT, crate::fips::UPSTREAM_SERVICE_UNIT, @@ -70,14 +80,30 @@ impl NodeTransport for FipsTransport { .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as u64) .unwrap_or(0); - let cached_at = self.available_cached_at_ms.load(Ordering::Relaxed); + let cached_at = self.availability.probed_at_ms.load(Ordering::Relaxed); + let cached = self.availability.available.load(Ordering::Relaxed); if now_ms.saturating_sub(cached_at) < AVAILABILITY_CACHE_TTL.as_millis() as u64 { - return self.available_cached.load(Ordering::Relaxed); + return cached; } - let val = Self::probe_daemon_active(); - self.available_cached.store(val, Ordering::Relaxed); - self.available_cached_at_ms.store(now_ms, Ordering::Relaxed); - val + // Cache is stale. This sync trait method is called from async + // route(), so running the ~50ms systemctl probe inline stalls a + // tokio worker. Serve the stale value and refresh on a background + // thread instead — the transport supervisor's warm loop keeps this + // fresh in steady state, so staleness is bounded to one probe round. + let cache = std::sync::Arc::clone(&self.availability); + if !cache.probe_in_flight.swap(true, Ordering::Relaxed) { + std::thread::spawn(move || { + let val = Self::probe_daemon_active(); + let probed_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + cache.available.store(val, Ordering::Relaxed); + cache.probed_at_ms.store(probed_ms, Ordering::Relaxed); + cache.probe_in_flight.store(false, Ordering::Relaxed); + }); + } + cached } fn send<'a>( diff --git a/core/archipelago/src/update.rs b/core/archipelago/src/update.rs index 5a0744d4..848e464a 100644 --- a/core/archipelago/src/update.rs +++ b/core/archipelago/src/update.rs @@ -513,8 +513,64 @@ async fn probe_frontend_once() -> Result<()> { anyhow::bail!("frontend probe returned HTTP {}", status); } +/// Probe the backend RPC API through nginx. An unauthenticated call is +/// EXPECTED to get 401/403 — any such response proves the Rust HTTP stack +/// is alive behind nginx. 5xx (backend dead → nginx 502), 404 (proxy +/// misroute), or connection errors mean the API is down even though the +/// static frontend may still serve — exactly the failure mode the plain +/// `GET /` probe waved through. +async fn probe_backend_once() -> Result<()> { + let client = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .timeout(std::time::Duration::from_secs(5)) + .build() + .context("build probe client")?; + let body = serde_json::json!({ "method": "update.status" }); + let resp = match client + .post("https://127.0.0.1/rpc/v1") + .json(&body) + .send() + .await + { + Ok(resp) => resp, + Err(e) if e.is_connect() => client + .post("http://127.0.0.1/rpc/v1") + .json(&body) + .send() + .await + .context("probe POST http://127.0.0.1/rpc/v1 (https not bound on loopback)")?, + Err(e) => return Err(e).context("probe POST https://127.0.0.1/rpc/v1"), + }; + let status = resp.status(); + if status.is_server_error() || status == reqwest::StatusCode::NOT_FOUND { + anyhow::bail!("backend RPC probe returned HTTP {}", status); + } + Ok(()) +} + +/// Probe that the rootless container runtime is reachable from the new +/// binary (uid mapping / podman socket intact after the swap). A healthy +/// node answers `podman ps` in well under a second. +async fn probe_container_runtime_once() -> Result<()> { + let out = tokio::process::Command::new("podman") + .args(["ps", "--format", "{{.Names}}"]) + .output() + .await + .context("spawn podman ps")?; + if !out.status.success() { + anyhow::bail!( + "podman ps exited {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(()) +} + /// Called from main.rs startup. If a pending-verification marker is -/// present, probe the frontend; on failure, trigger rollback and +/// present, verify the node actually works on the new version — binary +/// version matches the marker, frontend serves, backend RPC answers, +/// rootless podman is reachable. On failure, trigger rollback and /// restart the service so the OLD binary boots. /// /// This is the "post-OTA auto-rollback" guardrail. If ANY problem in @@ -547,34 +603,59 @@ pub async fn verify_pending_update(data_dir: &Path) { info!( new_version = %marker.new_version, previous_version = %marker.previous_version, - "Post-OTA verification: probing frontend at https://127.0.0.1/" + "Post-OTA verification: probing frontend, backend RPC, and container runtime" ); - // Give the new service time to bind its listeners + nginx to - // pick up any config changes. 15s matches what we observed on - // .116 during the v1.7.40 rollout recovery. - tokio::time::sleep(std::time::Duration::from_secs(15)).await; - - let deadline = - std::time::Instant::now() + std::time::Duration::from_secs(PENDING_VERIFY_WINDOW_SECS); + // Binary identity check: if the running binary's version isn't the one + // the marker says we applied, the swap silently failed (or half-applied + // — new frontend with old binary). Deterministic, so no retry loop: + // fall through straight to rollback to restore a matched pair. + let running = env!("CARGO_PKG_VERSION"); let mut attempt = 0u32; let mut last_err: Option = None; + let version_ok = running == marker.new_version; + if !version_ok { + last_err = Some(format!( + "running binary is {} but marker says {} was applied — binary swap failed", + running, marker.new_version + )); + } else { + // Give the new service time to bind its listeners + nginx to + // pick up any config changes. 15s matches what we observed on + // .116 during the v1.7.40 rollout recovery. + tokio::time::sleep(std::time::Duration::from_secs(15)).await; - while std::time::Instant::now() < deadline { - attempt += 1; - match probe_frontend_once().await { - Ok(()) => { - info!(attempt, "Post-OTA verification succeeded — clearing marker"); - clear_pending_verification(data_dir).await; - return; - } - Err(e) => { - let msg = e.to_string(); - tracing::warn!(attempt, error = %msg, "Post-OTA probe failed, retrying"); - last_err = Some(msg); + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(PENDING_VERIFY_WINDOW_SECS); + + while std::time::Instant::now() < deadline { + attempt += 1; + // All three must pass in the same attempt: static frontend via + // nginx, backend RPC liveness, and rootless-podman reachability. + // (Individual app containers are NOT asserted — the pre-Quadlet + // service restart legitimately takes them down and the boot + // reconciler can need minutes to bring heavy apps back.) + let result = match probe_frontend_once().await { + Ok(()) => match probe_backend_once().await { + Ok(()) => probe_container_runtime_once().await, + Err(e) => Err(e), + }, + Err(e) => Err(e), + }; + match result { + Ok(()) => { + info!(attempt, "Post-OTA verification succeeded — clearing marker"); + clear_pending_verification(data_dir).await; + return; + } + Err(e) => { + let msg = format!("{e:#}"); + tracing::warn!(attempt, error = %msg, "Post-OTA probe failed, retrying"); + last_err = Some(msg); + } } + tokio::time::sleep(std::time::Duration::from_secs(5)).await; } - tokio::time::sleep(std::time::Duration::from_secs(5)).await; } tracing::error!( diff --git a/core/container/Cargo.toml b/core/container/Cargo.toml index 217de3e4..3100d40f 100644 --- a/core/container/Cargo.toml +++ b/core/container/Cargo.toml @@ -19,6 +19,8 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1.0", features = ["v4"] } log = "0.4" tracing = "0.1" +sha2 = "0.10" +hex = "0.4" [lib] name = "archipelago_container" diff --git a/core/container/src/image_verify.rs b/core/container/src/image_verify.rs new file mode 100644 index 00000000..64b4e1f9 --- /dev/null +++ b/core/container/src/image_verify.rs @@ -0,0 +1,207 @@ +//! Container image signature verification (cosign). +//! +//! The manifest/catalog `image_signature` field is a *claim* that the image +//! is signed with the fleet's cosign key. Verification runs at the pull +//! choke points (`PodmanClient::pull_image`, `DockerRuntime::pull_image`); +//! a declared signature that cannot be verified hard-fails the pull. +//! +//! Every manifest has carried the literal placeholder `cosign://...` since +//! the field was introduced — that means "not signed yet" and is treated as +//! no claim, so enforcement stays dormant until the signing ceremony +//! publishes real signatures AND nodes carry the pinned cosign public key. +//! Ship order matters: key + cosign binary reach the fleet first, real +//! signature values in the catalog come after. + +use anyhow::{bail, Context, Result}; +use std::path::PathBuf; + +/// The literal placeholder every pre-ceremony manifest carries. +pub const SIGNATURE_PLACEHOLDER: &str = "cosign://..."; + +/// Env override for the pinned cosign public key path (tests, staging). +pub const COSIGN_PUBKEY_ENV: &str = "ARCHIPELAGO_COSIGN_PUBKEY"; + +const DEFAULT_PUBKEY_PATH: &str = "/etc/archipelago/cosign.pub"; + +const COSIGN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SignatureClaim { + /// No signature declared (field absent or empty). + None, + /// The literal `cosign://...` placeholder — manifest predates real signing. + Placeholder, + /// A real declared signature reference; MUST verify or the pull fails. + Declared(String), +} + +pub fn classify_signature(signature: Option<&str>) -> SignatureClaim { + match signature.map(str::trim) { + None | Some("") => SignatureClaim::None, + Some(SIGNATURE_PLACEHOLDER) => SignatureClaim::Placeholder, + Some(s) => SignatureClaim::Declared(s.to_string()), + } +} + +fn pinned_pubkey_path() -> PathBuf { + std::env::var(COSIGN_PUBKEY_ENV) + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(DEFAULT_PUBKEY_PATH)) +} + +/// Verify a declared image signature with the fleet's pinned cosign key. +/// Any failure — missing key, missing cosign binary, verification error — +/// is a hard error: an image that CLAIMS to be signed must never be pulled +/// on a node that can't prove the claim. +pub async fn verify_declared_signature( + image: &str, + sig_ref: &str, + allow_insecure_registry: bool, +) -> Result<()> { + verify_with_key_path(image, sig_ref, &pinned_pubkey_path(), allow_insecure_registry).await +} + +async fn verify_with_key_path( + image: &str, + sig_ref: &str, + key_path: &std::path::Path, + allow_insecure_registry: bool, +) -> Result<()> { + if !key_path.exists() { + bail!( + "Image '{image}' declares signature '{sig_ref}' but the pinned cosign \ + public key is missing at {} (override with {COSIGN_PUBKEY_ENV}). \ + Refusing to pull an image whose signature claim cannot be verified.", + key_path.display() + ); + } + + // Self-managed key => signatures aren't in the public Rekor transparency + // log, so tlog verification must be disabled explicitly (cosign v2 + // defaults it on and would fail every private-key signature otherwise). + let mut cmd = tokio::process::Command::new("cosign"); + cmd.arg("verify") + .arg("--key") + .arg(key_path) + .arg("--insecure-ignore-tlog=true"); + if allow_insecure_registry { + // podman's --tls-verify=false covers both plain HTTP and bad TLS; + // cosign splits those into two flags — pass both to match. + cmd.arg("--allow-insecure-registry"); + cmd.arg("--allow-http-registry"); + } + cmd.arg(image); + + let output = tokio::time::timeout(COSIGN_TIMEOUT, cmd.output()) + .await + .map_err(|_| { + anyhow::anyhow!( + "cosign verify timed out after {}s for image '{image}'", + COSIGN_TIMEOUT.as_secs() + ) + })? + .with_context(|| { + format!( + "Failed to run cosign for image '{image}' which declares signature \ + '{sig_ref}' — is cosign installed? A declared signature cannot be \ + skipped." + ) + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!( + "Signature verification FAILED for image '{image}' (declared: '{sig_ref}'): \ + {stderr}" + ); + } + + tracing::info!("cosign signature verified for image {image}"); + Ok(()) +} + +/// Shared pull-site gate: decide whether the pull may proceed. +/// Returns Ok(()) for unsigned/placeholder claims (with a log line) and only +/// after successful cosign verification for declared ones. +pub async fn enforce_signature_claim( + image: &str, + signature: Option<&str>, + allow_insecure_registry: bool, +) -> Result<()> { + match classify_signature(signature) { + SignatureClaim::None => { + tracing::debug!("image {image}: no signature declared, pulling unverified"); + Ok(()) + } + SignatureClaim::Placeholder => { + tracing::debug!( + "image {image}: signature is the pre-ceremony placeholder, pulling unverified" + ); + Ok(()) + } + SignatureClaim::Declared(sig_ref) => { + verify_declared_signature(image, &sig_ref, allow_insecure_registry).await + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn absent_and_empty_signatures_are_no_claim() { + assert_eq!(classify_signature(None), SignatureClaim::None); + assert_eq!(classify_signature(Some("")), SignatureClaim::None); + assert_eq!(classify_signature(Some(" ")), SignatureClaim::None); + } + + #[test] + fn literal_placeholder_is_not_a_claim() { + assert_eq!( + classify_signature(Some("cosign://...")), + SignatureClaim::Placeholder + ); + assert_eq!( + classify_signature(Some(" cosign://... ")), + SignatureClaim::Placeholder + ); + } + + #[test] + fn real_values_are_declared_claims() { + assert_eq!( + classify_signature(Some("cosign://sha256-abc.sig")), + SignatureClaim::Declared("cosign://sha256-abc.sig".to_string()) + ); + // Unknown schemes still count as a claim — better to fail closed on + // a value we don't understand than to pull unverified. + assert_eq!( + classify_signature(Some("sigstore://whatever")), + SignatureClaim::Declared("sigstore://whatever".to_string()) + ); + } + + #[tokio::test] + async fn declared_signature_without_pinned_key_hard_fails() { + let err = verify_with_key_path( + "registry.example/app:1.0", + "cosign://sha256-abc.sig", + std::path::Path::new("/nonexistent/cosign.pub"), + false, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("pinned cosign public key is missing")); + } + + #[tokio::test] + async fn unsigned_and_placeholder_claims_pass_the_gate() { + enforce_signature_claim("registry.example/app:1.0", None, false) + .await + .unwrap(); + enforce_signature_claim("registry.example/app:1.0", Some("cosign://..."), false) + .await + .unwrap(); + } +} diff --git a/core/container/src/lib.rs b/core/container/src/lib.rs index 05d337e6..02f01e23 100644 --- a/core/container/src/lib.rs +++ b/core/container/src/lib.rs @@ -1,5 +1,6 @@ pub mod bitcoin_simulator; pub mod health_monitor; +pub mod image_verify; pub mod manifest; pub mod podman_client; pub mod port_manager; diff --git a/core/container/src/manifest.rs b/core/container/src/manifest.rs index 9746f93c..d365ac32 100644 --- a/core/container/src/manifest.rs +++ b/core/container/src/manifest.rs @@ -243,6 +243,22 @@ pub struct ContainerConfig { /// Example: `"100070:100070"` for Postgres' mapped subuid. #[serde(default)] pub data_uid: Option, + + /// Runtime-resolved secret env entries (never serialized, never in a + /// manifest file). Populated by the orchestrator's env-resolution + /// chokepoint; the backends turn each ref into a podman secret + /// reference (`secret_env` in the API spec / `Secret=…,type=env` in + /// Quadlet) so the VALUE never lands in `podman inspect` output or a + /// unit file on disk. The dev-only docker fallback injects `value` as + /// plain env — docker has no rootless secret store. + #[serde(skip)] + pub secret_env_refs: Vec, + + /// sha256 over all resolved secret env pairs, stamped onto the + /// container as a label so rotation is detectable as drift without + /// exposing values. None when the app has no secret env. + #[serde(skip)] + pub secret_env_hash: Option, } /// Derived-env entry. The template is rendered against `HostFacts` at @@ -265,6 +281,75 @@ pub struct SecretEnv { pub secret_file: String, } +/// A fully resolved secret env entry, produced at apply time. `value` lives +/// only in memory on its way to the podman secret store (or, on the dev +/// docker fallback, straight into the container env). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SecretEnvRef { + pub env_key: String, + /// Podman secret object name: `archy-env--`. + pub secret_name: String, + pub value: String, +} + +/// Container label carrying the combined secret-env content hash, used by +/// the reconciler to detect secret rotation as env drift. +pub const SECRET_ENV_HASH_LABEL: &str = "io.archipelago.secret-env-hash"; + +/// Podman secret label carrying the individual secret's content hash, used +/// to skip re-registration when nothing changed. +pub const SECRET_HASH_LABEL: &str = "io.archipelago.hash"; + +/// Expand `${KEY}` placeholders across plain + secret values, then split +/// the result into (plain env, secret-bearing pairs). Any plain entry whose +/// value interpolates a secret key is *tainted* — it embeds the secret and +/// must travel as a secret itself (btcpay's +/// `BTCPAY_POSTGRES=…Password=${BTCPAY_DB_PASS};…` pattern). Secret values +/// themselves are taken verbatim: a generated secret that happens to +/// contain `${` must not be mangled by expansion. +pub fn expand_and_partition_env( + plain: Vec, + secrets: Vec<(String, String)>, +) -> (Vec, Vec<(String, String)>) { + let plain_values: std::collections::HashMap = plain + .iter() + .filter_map(|entry| { + let (key, value) = entry.split_once('=')?; + Some((key.to_string(), value.to_string())) + }) + .collect(); + + let mut out_plain = Vec::with_capacity(plain.len()); + let mut out_secret: Vec<(String, String)> = Vec::with_capacity(secrets.len()); + + for entry in plain { + let Some((key, value)) = entry.split_once('=') else { + out_plain.push(entry); + continue; + }; + let mut expanded = value.to_string(); + let mut tainted = false; + for (k, v) in &plain_values { + expanded = expanded.replace(&format!("${{{k}}}"), v); + } + for (k, v) in &secrets { + let placeholder = format!("${{{k}}}"); + if expanded.contains(&placeholder) { + expanded = expanded.replace(&placeholder, v); + tainted = true; + } + } + if tainted { + out_secret.push((key.to_string(), expanded)); + } else { + out_plain.push(format!("{key}={expanded}")); + } + } + + out_secret.extend(secrets); + (out_plain, out_secret) +} + /// How a [`GeneratedSecret`] is produced. Each kind is deterministic in shape /// (so the orchestrator knows which files to expect) but random in value. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] @@ -1209,6 +1294,19 @@ impl ContainerConfig { &self, provider: &dyn SecretsProvider, ) -> Result, ManifestError> { + Ok(self + .resolve_secret_env_pairs(provider)? + .into_iter() + .map(|(k, v)| format!("{k}={v}")) + .collect()) + } + + /// Like `resolve_secret_env` but returns (key, value) pairs — the shape + /// the podman-secret pipeline needs. + pub fn resolve_secret_env_pairs( + &self, + provider: &dyn SecretsProvider, + ) -> Result, ManifestError> { let mut out = Vec::with_capacity(self.secret_env.len()); for e in &self.secret_env { let v = provider.read(&e.secret_file)?; @@ -1220,12 +1318,28 @@ impl ContainerConfig { e.key, e.secret_file ))); } - out.push(format!("{}={}", e.key, v)); + out.push((e.key.clone(), v)); } Ok(out) } } +/// Deterministic content hash over resolved secret env pairs (sorted by +/// key), used for the container drift label and per-secret labels. +pub fn secret_env_content_hash(pairs: &[(String, String)]) -> String { + use sha2::{Digest, Sha256}; + let mut sorted: Vec<&(String, String)> = pairs.iter().collect(); + sorted.sort_by(|a, b| a.0.cmp(&b.0)); + let mut hasher = Sha256::new(); + for (k, v) in sorted { + hasher.update(k.as_bytes()); + hasher.update([0u8]); + hasher.update(v.as_bytes()); + hasher.update([0u8]); + } + hex::encode(hasher.finalize()) +} + #[cfg(test)] mod tests { use super::*; @@ -1233,6 +1347,53 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; + #[test] + fn partition_taints_plain_entries_that_interpolate_secrets() { + let plain = vec![ + "PLAIN=1".to_string(), + "COMPOSED=${BASE}/x".to_string(), + "DB_URL=postgres://u:${DB_PASS}@db/x".to_string(), + "BASE=/srv".to_string(), + "UNKNOWN=${NOT_DEFINED}".to_string(), + ]; + let secrets = vec![("DB_PASS".to_string(), "s3cret".to_string())]; + let (p, s) = expand_and_partition_env(plain, secrets); + + // plain-from-plain expansion stays plain; unknown placeholders stay + // literal (legacy expander parity) + assert!(p.contains(&"PLAIN=1".to_string())); + assert!(p.contains(&"COMPOSED=/srv/x".to_string())); + assert!(p.contains(&"UNKNOWN=${NOT_DEFINED}".to_string())); + // the tainted entry moved out of plain, fully expanded + assert!(!p.iter().any(|e| e.starts_with("DB_URL="))); + assert!(s.contains(&("DB_URL".to_string(), "postgres://u:s3cret@db/x".to_string()))); + // the original secret rides along verbatim + assert!(s.contains(&("DB_PASS".to_string(), "s3cret".to_string()))); + } + + #[test] + fn secret_values_are_never_expanded() { + // A generated secret containing `${` must pass through untouched. + let secrets = vec![("WEIRD".to_string(), "pa${PLAIN}ss".to_string())]; + let (_, s) = expand_and_partition_env(vec!["PLAIN=1".to_string()], secrets); + assert!(s.contains(&("WEIRD".to_string(), "pa${PLAIN}ss".to_string()))); + } + + #[test] + fn secret_env_hash_is_order_independent() { + let a = vec![ + ("K1".to_string(), "v1".to_string()), + ("K2".to_string(), "v2".to_string()), + ]; + let b = vec![ + ("K2".to_string(), "v2".to_string()), + ("K1".to_string(), "v1".to_string()), + ]; + assert_eq!(secret_env_content_hash(&a), secret_env_content_hash(&b)); + let c = vec![("K1".to_string(), "CHANGED".to_string())]; + assert_ne!(secret_env_content_hash(&a), secret_env_content_hash(&c)); + } + #[test] fn hooks_parse_and_validate() { let yaml = r#" @@ -1765,6 +1926,8 @@ app: generated_secrets: vec![], generated_certs: vec![], data_uid: None, + secret_env_refs: vec![], + secret_env_hash: None, }; let facts = HostFacts { host_ip: "192.168.1.116".to_string(), @@ -1817,6 +1980,8 @@ app: generated_secrets: vec![], generated_certs: vec![], data_uid: None, + secret_env_refs: vec![], + secret_env_hash: None, }; let p = MapSecretsProvider { data: HashMap::from([ @@ -1855,6 +2020,8 @@ app: generated_secrets: vec![], generated_certs: vec![], data_uid: None, + secret_env_refs: vec![], + secret_env_hash: None, }; let p = MapSecretsProvider { data: HashMap::from([("bitcoin-rpc-password".to_string(), " \n".to_string())]), diff --git a/core/container/src/podman_client.rs b/core/container/src/podman_client.rs index 927d409c..1a965ea5 100644 --- a/core/container/src/podman_client.rs +++ b/core/container/src/podman_client.rs @@ -252,7 +252,17 @@ impl PodmanClient { // ─── Container Operations ──────────────────────────────────── - pub async fn pull_image(&self, image: &str, _signature: Option<&str>) -> Result<()> { + pub async fn pull_image(&self, image: &str, signature: Option<&str>) -> Result<()> { + // A declared (non-placeholder) signature must verify before we fetch + // anything; placeholder/absent claims pull unverified until the + // signing ceremony ships real signatures (see image_verify). + crate::image_verify::enforce_signature_claim( + image, + signature, + image_uses_insecure_registry(image), + ) + .await?; + // Image pull uses CLI — it's a streaming operation that the API handles differently let mut cmd = tokio::process::Command::new("podman"); cmd.arg("pull"); @@ -372,12 +382,32 @@ impl PodmanClient { manifest.app.security.network_policy.as_str(), ); + // Secret env travels by reference (podman injects the value at + // start), so it never shows up in `podman inspect` output. The + // combined content hash rides as a label for rotation-drift checks. + let mut secret_env_map = serde_json::Map::new(); + for r in &manifest.app.container.secret_env_refs { + secret_env_map.insert( + r.env_key.clone(), + serde_json::Value::String(r.secret_name.clone()), + ); + } + let mut labels_map = serde_json::Map::new(); + if let Some(hash) = &manifest.app.container.secret_env_hash { + labels_map.insert( + crate::manifest::SECRET_ENV_HASH_LABEL.to_string(), + serde_json::Value::String(hash.clone()), + ); + } + let mut body = serde_json::json!({ "name": name, "image": image_ref, "portmappings": port_mappings, "mounts": mounts, "env": env_map, + "secret_env": secret_env_map, + "labels": labels_map, "entrypoint": manifest.app.container.entrypoint.clone(), "command": manifest.app.container.custom_args.clone(), "hostadd": [ diff --git a/core/container/src/runtime.rs b/core/container/src/runtime.rs index cf0dd0f6..4ee040b3 100644 --- a/core/container/src/runtime.rs +++ b/core/container/src/runtime.rs @@ -2,7 +2,6 @@ use crate::manifest::{AppManifest, BuildConfig}; use crate::podman_client::{ContainerState, ContainerStatus, PodmanClient}; use anyhow::{Context, Result}; use async_trait::async_trait; -use std::process::Command; use std::time::Duration; use tokio::process::Command as TokioCommand; @@ -83,6 +82,18 @@ pub trait ContainerRuntime: Send + Sync { /// `create_container` / `image_exists` calls. Stdout/stderr are collected /// and included in the error on failure; on success they are discarded. async fn build_image(&self, config: &BuildConfig) -> Result<()>; + + /// Register the app's resolved secret-env entries in the runtime's + /// secret store (idempotent — skips entries whose content hash already + /// matches). Called before `create_container` for manifests with + /// `secret_env_refs`, so the create can reference secrets by name and + /// the values never appear in inspect output or unit files. Default is + /// a no-op for runtimes without a secret store (mocks, docker — the + /// docker fallback injects plain env in `create_container` instead). + async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> { + let _ = refs; + Ok(()) + } } pub struct PodmanRuntime { @@ -132,6 +143,68 @@ impl ContainerRuntime for PodmanRuntime { self.client.pull_image(image, signature).await } + async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> { + use crate::manifest::{secret_env_content_hash, SECRET_HASH_LABEL}; + use tokio::io::AsyncWriteExt; + + for r in refs { + let hash = secret_env_content_hash(&[(r.env_key.clone(), r.value.clone())]); + + // Skip the write when the stored secret already has this content + // (label round-trip beats rewriting the secret store every + // reconcile). Any inspect failure just falls through to create. + let fmt = format!("{{{{ index .Spec.Labels \"{SECRET_HASH_LABEL}\" }}}}"); + if let Ok(out) = self + .podman_cli(&["secret", "inspect", &r.secret_name, "--format", &fmt]) + .await + { + if out.status.success() + && String::from_utf8_lossy(&out.stdout).trim() == hash + { + continue; + } + } + + // Value goes via stdin — never argv, never a temp file. + let mut cmd = TokioCommand::new("podman"); + cmd.args([ + "secret", + "create", + "--replace", + "--label", + &format!("{SECRET_HASH_LABEL}={hash}"), + &r.secret_name, + "-", + ]); + cmd.stdin(std::process::Stdio::piped()); + cmd.stdout(std::process::Stdio::piped()); + cmd.stderr(std::process::Stdio::piped()); + cmd.kill_on_drop(true); + let mut child = cmd + .spawn() + .with_context(|| format!("spawning podman secret create {}", r.secret_name))?; + { + let mut stdin = child + .stdin + .take() + .context("podman secret create stdin unavailable")?; + stdin.write_all(r.value.as_bytes()).await?; + // drop closes the pipe so podman sees EOF + } + let out = tokio::time::timeout(PODMAN_CLI_DEFAULT_TIMEOUT, child.wait_with_output()) + .await + .with_context(|| format!("podman secret create {} timed out", r.secret_name))??; + if !out.status.success() { + anyhow::bail!( + "podman secret create {} failed: {}", + r.secret_name, + String::from_utf8_lossy(&out.stderr) + ); + } + } + Ok(()) + } + async fn create_container( &self, manifest: &AppManifest, @@ -547,7 +620,12 @@ impl DockerRuntime { #[async_trait] impl ContainerRuntime for DockerRuntime { - async fn pull_image(&self, image: &str, _signature: Option<&str>) -> Result<()> { + async fn pull_image(&self, image: &str, signature: Option<&str>) -> Result<()> { + // Same signature gate as the podman path — the docker fallback is + // dev-only, but a declared signature must never be skippable by + // switching runtimes. + crate::image_verify::enforce_signature_claim(image, signature, false).await?; + let mut cmd = self.docker_async(); cmd.arg("pull").arg(image); @@ -617,6 +695,12 @@ impl ContainerRuntime for DockerRuntime { for env in &manifest.app.environment { cmd.arg("-e").arg(env); } + // Dev-only fallback: docker has no rootless secret store, so secret + // env rides as plain env here. The podman path (production) passes + // these by secret reference instead — see ensure_env_secrets. + for r in &manifest.app.container.secret_env_refs { + cmd.arg("-e").arg(format!("{}={}", r.env_key, r.value)); + } // Resource limits if let Some(cpu) = manifest.app.resources.cpu_limit { @@ -853,11 +937,11 @@ pub struct AutoRuntime { impl AutoRuntime { pub async fn new(user: String) -> Result { // Try Podman first - if Self::check_podman_available() { + if Self::check_podman_available().await { Ok(Self { runtime: Box::new(PodmanRuntime::new(user)), }) - } else if Self::check_docker_available() { + } else if Self::check_docker_available().await { Ok(Self { runtime: Box::new(DockerRuntime::new(user)), }) @@ -866,12 +950,20 @@ impl AutoRuntime { } } - fn check_podman_available() -> bool { - Command::new("podman").arg("--version").output().is_ok() + async fn check_podman_available() -> bool { + TokioCommand::new("podman") + .arg("--version") + .output() + .await + .is_ok() } - fn check_docker_available() -> bool { - Command::new("docker").arg("--version").output().is_ok() + async fn check_docker_available() -> bool { + TokioCommand::new("docker") + .arg("--version") + .output() + .await + .is_ok() } } @@ -881,6 +973,10 @@ impl ContainerRuntime for AutoRuntime { self.runtime.pull_image(image, signature).await } + async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> { + self.runtime.ensure_env_secrets(refs).await + } + async fn create_container( &self, manifest: &AppManifest, diff --git a/core/security/src/image_verifier.rs b/core/security/src/image_verifier.rs deleted file mode 100644 index b6c6f2bf..00000000 --- a/core/security/src/image_verifier.rs +++ /dev/null @@ -1,111 +0,0 @@ -// Container image signature verification using Cosign -// Verifies that container images are signed and trusted - -use anyhow::{Context, Result}; -use std::process::Command; -use tracing::{info, warn}; - -pub struct ImageVerifier { - cosign_public_key: Option, - require_signatures: bool, -} - -impl ImageVerifier { - pub fn new(cosign_public_key: Option) -> Self { - Self { - cosign_public_key, - require_signatures: false, - } - } - - /// Create a verifier that requires all images to be signed. - pub fn new_strict(cosign_public_key: Option) -> Self { - Self { - cosign_public_key, - require_signatures: true, - } - } - - /// Verify a container image signature - pub async fn verify_image(&self, image: &str, signature: Option<&str>) -> Result { - if signature.is_none() && self.cosign_public_key.is_none() { - if self.require_signatures { - return Err(anyhow::anyhow!( - "Image '{}' has no signature and no cosign key is configured. \ - All container images must be signed for production use.", - image - )); - } - warn!("No signature provided for image: {}", image); - return Ok(false); - } - - // Check if cosign is available - let cosign_available = Command::new("cosign").arg("version").output().is_ok(); - - if !cosign_available { - if self.require_signatures { - return Err(anyhow::anyhow!( - "Cosign binary not found. Install cosign to verify container image signatures." - )); - } - warn!("Cosign not available, skipping signature verification"); - return Ok(false); - } - - // If public key is provided, use it for verification - if let Some(ref public_key) = self.cosign_public_key { - let output = Command::new("cosign") - .arg("verify") - .arg("--key") - .arg(public_key) - .arg(image) - .output() - .context("Failed to run cosign verify")?; - - if output.status.success() { - info!("Image signature verified: {}", image); - return Ok(true); - } else { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow::anyhow!("Signature verification failed: {}", stderr)); - } - } - - // If signature URL is provided, verify using that - if let Some(sig_url) = signature { - if sig_url.starts_with("cosign://") { - // Extract signature reference - let sig_ref = sig_url.strip_prefix("cosign://").unwrap(); - let output = Command::new("cosign") - .arg("verify") - .arg("--signature") - .arg(sig_ref) - .arg(image) - .output() - .context("Failed to run cosign verify")?; - - if output.status.success() { - info!("Image signature verified: {}", image); - return Ok(true); - } else { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow::anyhow!("Signature verification failed: {}", stderr)); - } - } - } - - Ok(false) - } - - /// Check if an image has a signature - pub async fn has_signature(&self, image: &str) -> bool { - // Try to find signature in registry - let output = Command::new("cosign") - .arg("triangulate") - .arg(image) - .output(); - - output.is_ok() && output.unwrap().status.success() - } -} diff --git a/core/security/src/lib.rs b/core/security/src/lib.rs index 48134224..a64872c1 100644 --- a/core/security/src/lib.rs +++ b/core/security/src/lib.rs @@ -1,7 +1,5 @@ pub mod container_policies; -pub mod image_verifier; pub mod secrets_manager; pub use container_policies::ContainerPolicyGenerator; -pub use image_verifier::ImageVerifier; pub use secrets_manager::SecretsManager; diff --git a/docs/1.8.0-RELEASE-HARDENING-PLAN.md b/docs/1.8.0-RELEASE-HARDENING-PLAN.md index 68bdc07f..02ac70ab 100644 --- a/docs/1.8.0-RELEASE-HARDENING-PLAN.md +++ b/docs/1.8.0-RELEASE-HARDENING-PLAN.md @@ -58,21 +58,32 @@ arbitrary app catalog to the entire fleet — fully unattended under `scripts/sign-manifest.sh` exists for re-signs. **Still open:** move the mirror to HTTPS + pinned cert (tracked with the next item); flip unsigned-manual-apply → hard-reject once the fleet is on a pinned-anchor binary. -- [ ] 🔴 **Implement container image signature verification (cosign).** - `container/src/podman_client.rs:255` — `pull_image(.., _signature)` silently discards - the signature that the manifest threads all the way down - (`prod_orchestrator.rs:1978/2435`). Wire `sigstore-rs`/`cosign verify` (or - `podman pull --signature-policy`); hard-fail when a declared signature doesn't verify. +- [x] 🔴 **Implement container image signature verification (cosign).** DONE 2026-07-04 + (code path; enforcement dormant until the ceremony): new `container::image_verify` + gates BOTH pull sites (`PodmanClient::pull_image` + the dev-only `DockerRuntime`). + Claims classify as None / the literal `cosign://...` placeholder (every fleet + manifest today → pull proceeds, logged) / Declared → `cosign verify --key + /etc/archipelago/cosign.pub --insecure-ignore-tlog=true` (+ both insecure-registry + flags for the HTTP mirror; flags verified against cosign docs), hard-fail on missing + key, missing cosign binary, timeout, or bad signature — a declared signature can + never be skipped, on either runtime. Key path overridable via + `ARCHIPELAGO_COSIGN_PUBKEY`. Deleted the caller-less, blocking, wrong-CLI + `security::ImageVerifier`. **Activation = ceremony work**: pin cosign.pub on nodes + + install cosign + publish real `image_signature` values (in that order); tracked with + the Workstream B signing ceremony item. - [ ] 🟠 **Move the image mirror to HTTPS; drop `--tls-verify=false`.** `podman_client.rs:641` `INSECURE_REGISTRY_HOSTS = ["146.59.87.168:3000"]` + `config.rs:104,124` allowlist pull images over unauthenticated HTTP. Remove the raw-IP entries; give the mirror a valid/pinned cert. (Same host also baked insecurely into the ISO — see §F.) -- [ ] 🟠 **Validate every image string at the pull site, not just the RPC boundary.** - `is_valid_docker_image` runs in `install.rs:224`/`runtime.rs:549` but - `prod_orchestrator::install_fresh` (1978) and `resolve_catalog_image` (944-971) pass - catalog/manifest images straight to `pull_image`. Call the validator right before - every pull. +- [x] 🟠 **Validate every image string at the pull site, not just the RPC boundary.** + DONE 2026-07-03: policy extracted to `container::image_policy` (single source of truth; + RPC-boundary check delegates to it) and BOTH orchestrator pull sites (`install_fresh` + + `ensure_resolved_source_available`) hard-bail on refs that fail it. Policy accepts + trusted-registry refs + registry-less Docker Hub shorthand (`grafana/grafana` — used by + 8 manifests, can't name an attacker host); rejects any explicit non-allowlisted + registry host, shell metachars, malformed refs. 4 new unit tests; container 159 / + package 46 green. --- @@ -83,11 +94,15 @@ atomic swap, single-depth backup). The gaps are **authenticity** (§A) and **verification depth** — plus the fact that the upgrade path has never run end-to-end on real hardware. -- [ ] 🔴 **Deepen the post-OTA health check.** `update.rs:456` (`probe_frontend_once`) - passes on any 2xx/3xx from `GET /`, and `verify_pending_update` (494-593) only rolls - back on that. A release with a broken RPC API, dead containers, or failed LND unlock - passes and never rolls back. Add `/rpc/v1 update.status` + container-list/required-stack - health assertions before clearing the pending-verify marker. +- [x] 🔴 **Deepen the post-OTA health check.** DONE 2026-07-03: `verify_pending_update` + now requires, in the same attempt, (1) frontend 2xx/3xx via nginx, (2) backend RPC + liveness — unauthenticated POST `/rpc/v1`; 401/403 = alive, 5xx/404/refused = dead, + so a 502-behind-static-files release now rolls back, (3) rootless `podman ps` + reachability; plus a pre-loop binary-version==marker assertion that catches a silent + or half swap (new frontend + old binary) deterministically. Per-app container + assertions deliberately EXCLUDED — the pre-Quadlet service restart legitimately kills + containers and the reconciler can need minutes (false-rollback risk); revisit after + the Phase-3 flip. LND-unlock-level checks remain out of scope for the 90s window. - [ ] 🟠 **Run one real upgrade-from-vN-1 soak on hardware before tagging.** No test installs the previous version, points it at a staged 1.8.0 manifest, applies, and asserts health + rollback. This is the top release risk for an OTA release. A @@ -114,25 +129,60 @@ modules; production request/boot paths are essentially panic-free. The real risk sweep (`scheduler.rs`), block-header cache (`mesh/mod.rs`), 7× peer-transport badge (`sync.rs` + `content.rs`). Federation tombstone/untombstone upgraded to hard errors (see §I). Install-log line write left fire-and-forget with an explanatory comment. -- [ ] 🟠 **Remove blocking `std::process::Command` from async handlers.** - `install.rs:2222` `published_host_port` (sync podman on the install path), - `dependencies.rs:316` (`df`), `system/handlers.rs:578` (`sudo`), `transport/fips.rs:50` - (`systemctl`) stall tokio workers under load. Convert to `tokio::process` or - `spawn_blocking`. Only 8 files use `std::process::Command` — bounded. -- [ ] 🟡 **Restrict Bitcoin RPC exposure.** `bootstrap.rs:409` writes - `rpcallowip=0.0.0.0/0`. Scope to the container subnet / `127.0.0.1`. -- [ ] 🟡 **Move generated secrets from env to file mounts.** `manifest.rs:1208-1226` - injects secrets as `-e KEY=value`, readable via `podman inspect` / `/proc//environ`. - Prefer bind-mounting the existing `0600` secret file or `podman --secret`. -- [ ] 🟡 **Harden rate-limit IP extraction.** `middleware.rs:120-128` trusts - client-spoofable `X-Real-IP`/`X-Forwarded-For` → per-request bucket rotation defeats the - login limiter. Trust forwarded headers only from a configured proxy; have nginx set them. -- [ ] 🟢 **Include `seq` in the mesh signed preimage.** `message_types.rs:245-288` signs - `(t,v,ts)` but sets the anti-replay `seq` after signing → a radio MITM can alter ordering - without breaking the signature. -- [ ] 🟢 **Guard the short-DID slice panic** (`mesh/listener/decode.rs:566`) and gate the - dev-mode `password123` bypass (`auth.rs:18`) behind `#[cfg]` before it can reach a - release build. +- [x] 🟠 **Remove blocking `std::process::Command` from async handlers.** DONE 2026-07-03: + converted to `tokio::process` — `published_host_port` (install), `detect_disk_gb` + (dependencies), factory-reset restart (system/handlers), `config.rs detect_host_ip`, + the orchestrator host-facts helpers (`detect_host_ip/mdns/disk_gb`, `bitcoin_host`, + `resolve_dynamic_env` now async through all 6 call sites), and `AutoRuntime::new` + probes. `transport/fips.rs is_available()` (sync trait method on the async route path) + now serves the cached value and refreshes via a background thread (stale-while- + revalidate) instead of blocking on systemctl. `image_verifier.rs` cosign sites have no + callers yet — handled with the §A cosign item. Tests: container 155 / transport 29 / + config 29 / package 46 all green. +- [ ] 🟡 **Restrict Bitcoin RPC exposure.** INVESTIGATED 2026-07-03 — the fix is NOT + `rpcallowip`: under rootless-podman NAT every forwarded connection reaches bitcoind + with an in-subnet source IP, so scoping `rpcallowip` blocks nothing (and container + consumers use archy-net DNS anyway). The actual exposure is the host-side publish + `8332:8332` (binds 0.0.0.0 → LAN can hit RPC, auth-only barrier; 4 write sites: + `bootstrap.rs:424`, `package/config.rs:694`, `package/install.rs:1333/2450`, plus + the knots manifest). Real fix = `127.0.0.1:8332:8332` host bind (P2P 8333 stays + public; zmq 28332/28333 should get the same look — unauthenticated). ⚠ May break + external wallets pointed straight at nodeIP:8332 — needs a user call + on-node gate + re-run, so NOT changed drive-by from the dev box. +- [x] 🟡 **Move secret env out of plaintext channels → podman secrets.** DONE 2026-07-05 + (code + unit tests; needs the on-node gate re-run before it counts as verified): + secret env no longer merges into `environment` — it would land in `podman inspect` + AND as plaintext `Environment=` lines in Quadlet unit files on disk (the worse leak). + New pipeline: `expand_and_partition_env` taints plain entries that interpolate + secrets (btcpay's `Password=${BTCPAY_DB_PASS}` connection strings travel as secrets + too), values register as podman secrets (stdin, `--replace`, content-hash label, + per-app cache so steady-state reconciles are podman-free), containers reference + them via `secret_env` (API) / `Secret=…,type=env` (Quadlet). Verified empirically + on fleet podman 5.4.2: value absent from inspect, runtime injection works. Rotation + drift via `io.archipelago.secret-env-hash` container label; pre-upgrade containers + lack the label → ONE-TIME recreate wave on first reconcile after deploy (by design — + scrubs plaintext secrets from existing container configs). Docker dev fallback keeps + plain env (no secret store). `/proc//environ` inside the container is unchanged + (env is the app-compat contract); the closed leaks are inspect output + unit files. +- [x] 🟡 **Harden rate-limit IP extraction.** DONE 2026-07-03: the accept loop injects the + TCP `PeerAddr` into request extensions; `extract_client_ip` honors + `X-Real-IP`/`X-Forwarded-For` ONLY when the connection is from loopback (our nginx, + which sets `X-Real-IP $remote_addr`) — direct connections (e.g. the FIPS peer + listener) bucket under their socket IP, so per-request header rotation no longer + defeats the login limiter. 3 unit tests. +- [x] 🟢 **Include `seq` in the mesh signed preimage.** DONE 2026-07-04 (receiver half): + `verify_signature` accepts a v2 preimage `(t,v,ts,seq)` alongside legacy v1 `(t,v,ts)`; + `signed_with_seq()` is the v2 sender path, deliberately NOT yet wired — receivers + hard-drop bad signatures, so senders stay on v1 until the whole fleet verifies v2. + The seq-tampering window closes only when the v1 arm is removed (track as a + post-fleet-rollout follow-up). Unit tests cover v2 verify, v2 seq-tamper rejection, + and v1 sign-then-set-seq compatibility. +- [x] 🟢 **Guard the short-DID slice panic** (`mesh/listener/decode.rs:566`) and gate the + dev-mode `password123` bypass (`auth.rs:18`) behind `#[cfg]`. DONE 2026-07-04: + advert_name uses `.get()` fallback (malformed radio-supplied DID can't panic the + listener); the pre-setup dev-password login + the constant itself are + `#[cfg(debug_assertions)]` — no release binary carries the bypass regardless of + runtime config. - [ ] 🟢 **Apply the seccomp/apparmor profile** — `security/src/container_policies.rs:71` is a TODO; the profile is defined but never applied to podman. @@ -159,9 +209,11 @@ The real issues are the app-bridge origin model and a bloated bundle. (precached by the service worker → blocks PWA install), plus ~18 MB of ~1 MB full-screen JPEGs. Convert backgrounds to WebP/AVIF at responsive sizes, lazy/stream the intro video, and exclude video/audio from the Workbox precache. Biggest, easiest perf win. -- [ ] 🟢 **DOMPurify the `Server.vue` QR SVG** (`:283/:295` render `v-html` unsanitized while - `TwoFactorSection.vue` sanitizes the analogous SVG); guard the unguarded `pollInterval` - (`Mesh.vue:391`); surface silent data-fetch failures (`curatedApps.ts:58/71`). +- [x] 🟢 **DOMPurify the `Server.vue` QR SVG / guard `Mesh.vue` pollInterval / surface + `curatedApps.ts` fetch failures.** DONE 2026-07-03: WireGuard peer QR now sanitized with + the same `USE_PROFILES: {svg}` call as TwoFactorSection; Mesh poll interval guarded + + nulled on unmount; catalog fetch failures log per-URL console.warn incl. the + all-sources-failed fallback. Bundle-verified. --- diff --git a/docs/UNIFIED-TASK-TRACKER.md b/docs/UNIFIED-TASK-TRACKER.md index f45731d5..5f8b9d87 100644 --- a/docs/UNIFIED-TASK-TRACKER.md +++ b/docs/UNIFIED-TASK-TRACKER.md @@ -113,6 +113,32 @@ those are marked ✅ below with the commit that did it, so we stop re-litigating manifest-driven apps, never stacks; fedimint/fedimint-gateway/fedimint-clientd are 3 separate single-container apps with manifest dependency edges, not a coordinated stack. Workstream A's stack-migration tail is fully closed. +- [ ] **Container thrashing/flapping + reconciler churn** (added 2026-07-04 — was + implicit across other tracks, now an explicit pre-tag concern). The root cause + of restart-storm flapping is pre-Quadlet architecture: restarting + `archipelago.service` SIGKILLs every container in its cgroup, then the + reconciler rebuilds the world over several minutes (the post-OTA health check + deliberately skips per-app container assertions because of exactly this). + Consolidated lever list, in order of impact: + - **Phase-3 Quadlet default-flip** (tracked above) — removes the SIGKILL-the-world + behavior entirely; the single biggest fix. + - **Workstream F lifecycle items** — immich/grafana uninstall hangs + ghost + containers, grafana reinstall stops, fedimint guardian sync + (`docs/PRODUCTION-MASTER-PLAN.md` workstream F). + - **Reconciler churn observability** — no metric/log today distinguishes "settling + after restart" from "flapping"; add a per-app restart counter + log line when an + app restarts >N times in M minutes so thrash is visible instead of anecdotal. + - **Failed-unit self-healing gap (observed live 2026-07-06 on .228)**: fedimint's + quadlet unit exited 255 at 21:21 and sat `failed` for 7+ hours — the reconciler + never revived it (it repairs missing/drifted containers but doesn't + `reset-failed`+start failed .services). Same for the indeedhub trio after the + gate run. The health monitor also can't help (container is gone when the unit + fails). Add a reconcile step: quadlet-backed app whose .service is `failed` and + not user-stopped → reset-failed + start, with backoff. + - Already landed, don't re-do: boot-reconciler circuit breaker (2026-07-01), + indeedhub crashloop fix (2026-07-01), async blocking-Command pass (`4c75bb3d`, + removes executor stalls that made the API janky under reconcile load). + - Perf polish riding along: 93 MB frontend dist shrink (hardening plan §D 🟡). - [ ] **Developer tooling CLI suite** (validate/render/local-install/lifecycle-test) — APP-PACKAGING-MIGRATION-PLAN.md step 5, needed before external devs can publish. - [x] ~~**Consolidated deploy 2026-07-01**: merged PR #67 (reticulum daemon diff --git a/neode-ui/src/views/Mesh.vue b/neode-ui/src/views/Mesh.vue index 64d910e1..259907a8 100644 --- a/neode-ui/src/views/Mesh.vue +++ b/neode-ui/src/views/Mesh.vue @@ -388,13 +388,15 @@ onMounted(async () => { if (!archPollInterval) { archPollInterval = setInterval(loadArchMessages, 15000) } - pollInterval = setInterval(() => { - mesh.fetchStatus() - mesh.fetchPeers() - mesh.fetchMessages() - mesh.fetchDeadmanStatus() - mesh.fetchBlockHeaders() - }, 5000) + if (!pollInterval) { + pollInterval = setInterval(() => { + mesh.fetchStatus() + mesh.fetchPeers() + mesh.fetchMessages() + mesh.fetchDeadmanStatus() + mesh.fetchBlockHeaders() + }, 5000) + } // Instant peer updates (#48): the backend nudges the data-model revision when // it discovers/updates a mesh peer, so refetch peers on the WS push rather @@ -414,7 +416,7 @@ onUnmounted(() => { window.visualViewport.removeEventListener('scroll', updateKeyboardInset) } document.documentElement.style.removeProperty('--keyboard-inset') - if (pollInterval) clearInterval(pollInterval) + if (pollInterval) { clearInterval(pollInterval); pollInterval = null } if (archPollInterval) { clearInterval(archPollInterval); archPollInterval = null } if (wsUnsub) { wsUnsub(); wsUnsub = null } }) diff --git a/neode-ui/src/views/Server.vue b/neode-ui/src/views/Server.vue index 2355d760..ace11b22 100644 --- a/neode-ui/src/views/Server.vue +++ b/neode-ui/src/views/Server.vue @@ -306,7 +306,7 @@
-
+

Scan with the WireGuard app

{{ peerQrData.peer_ip }}

@@ -318,7 +318,7 @@
-
+

Scan with the WireGuard app

{{ peerQrData.peer_ip }}

@@ -406,6 +406,7 @@