Merge branch 'main' into worktree-reticulum-tcp-interop

apps/immich/manifest.yml

@@ -30,7 +30,13 @@ app:
     disk_limit: 200Gi
 
   security:
-    capabilities: []
+    # Runs as container root over a data tree the legacy installer chowned
+    # to the subuid range (host 100000 = container uid 1). Without
+    # DAC_OVERRIDE the server EACCESes writing upload/encoded-video the
+    # moment the container is recreated against this manifest (latent until
+    # the 2026-07-05 secret-env migration recreated it). Same cap set as
+    # immich-postgres minus the setuid pair it doesn't use.
+    capabilities: [CHOWN, DAC_OVERRIDE, FOWNER]
     readonly_root: false
     network_policy: isolated
 

core/Cargo.lock

@@ -169,6 +169,7 @@ dependencies = [
  "async-trait",
  "chrono",
  "futures",
+ "hex",
  "hyper 0.14.32",
  "indexmap",
  "log",
@@ -176,6 +177,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
+ "sha2 0.10.9",
  "thiserror 1.0.69",
  "tokio",
  "tracing",

core/archipelago/src/api/rpc/auth.rs

@@ -1,4 +1,6 @@
-use super::{RpcHandler, DEV_DEFAULT_PASSWORD};
+use super::RpcHandler;
+#[cfg(debug_assertions)]
+use super::DEV_DEFAULT_PASSWORD;
 use anyhow::Result;
 
 impl RpcHandler {
@@ -14,7 +16,10 @@ impl RpcHandler {
 
         let is_setup = self.auth_manager.is_setup().await?;
         if !is_setup {
-            // Dev mode: allow default password so UI can log in without running setup
+            // Dev BUILDS only: allow the default password so the UI can log
+            // in without running setup. cfg-gated so no release binary can
+            // carry the bypass, whatever its runtime config says.
+            #[cfg(debug_assertions)]
             if self.config.dev_mode && password == DEV_DEFAULT_PASSWORD {
                 tracing::info!("[onboarding] login via dev default password");
                 return Ok(serde_json::Value::Null);

core/archipelago/src/api/rpc/middleware.rs

@@ -179,13 +179,91 @@ pub(super) fn extract_cookie(headers: &hyper::HeaderMap, name: &str) -> Option<S
     None
 }
 
-/// Extract the client IP from request headers (X-Real-IP or X-Forwarded-For).
-pub(super) fn extract_client_ip(headers: &hyper::HeaderMap) -> IpAddr {
+/// The TCP peer address of the connection a request arrived on, injected
+/// into request extensions by the server accept loop.
+#[derive(Debug, Clone, Copy)]
+pub struct PeerAddr(pub std::net::SocketAddr);
+
+/// Extract the client IP for rate limiting.
+///
+/// `X-Real-IP`/`X-Forwarded-For` are only honored when the connection
+/// itself comes from loopback — i.e. from our local nginx, which sets
+/// `X-Real-IP $remote_addr`. On a direct connection (the FIPS peer
+/// listener, or anything that isn't the local proxy) the headers are
+/// client-supplied, so trusting them let an attacker rotate per-request
+/// "IPs" and defeat the login rate limiter; there we use the socket
+/// address instead.
+pub(super) fn extract_client_ip(parts: &hyper::http::request::Parts) -> IpAddr {
+    let socket_ip = parts.extensions.get::<PeerAddr>().map(|p| p.0.ip());
+    match socket_ip {
+        Some(ip) if ip.is_loopback() => forwarded_client_ip(&parts.headers).unwrap_or(ip),
+        Some(ip) => ip,
+        // No socket info recorded (shouldn't happen in the server path);
+        // fall back to the pre-extension behavior.
+        None => forwarded_client_ip(&parts.headers)
+            .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)),
+    }
+}
+
+/// The proxy-reported client IP, if a forwarded header carries one.
+fn forwarded_client_ip(headers: &hyper::HeaderMap) -> Option<IpAddr> {
     headers
         .get("x-real-ip")
         .or_else(|| headers.get("x-forwarded-for"))
         .and_then(|v| v.to_str().ok())
         .and_then(|s| s.split(',').next())
         .and_then(|s| s.trim().parse::<IpAddr>().ok())
-        .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST))
+}
+
+#[cfg(test)]
+mod client_ip_tests {
+    use super::*;
+    use std::net::SocketAddr;
+
+    fn parts_with(
+        peer: Option<&str>,
+        real_ip: Option<&str>,
+    ) -> hyper::http::request::Parts {
+        let mut builder = hyper::Request::builder().uri("/rpc/v1");
+        if let Some(ip) = real_ip {
+            builder = builder.header("x-real-ip", ip);
+        }
+        let (mut parts, _) = builder.body(()).unwrap().into_parts();
+        if let Some(addr) = peer {
+            parts
+                .extensions
+                .insert(PeerAddr(addr.parse::<SocketAddr>().unwrap()));
+        }
+        parts
+    }
+
+    #[test]
+    fn loopback_connection_trusts_forwarded_header() {
+        // nginx on loopback forwards the real client IP — use it.
+        let parts = parts_with(Some("127.0.0.1:44412"), Some("192.168.1.50"));
+        assert_eq!(
+            extract_client_ip(&parts),
+            "192.168.1.50".parse::<IpAddr>().unwrap()
+        );
+    }
+
+    #[test]
+    fn direct_connection_ignores_spoofed_header() {
+        // A direct (non-proxy) client rotating X-Real-IP per request must
+        // still bucket under its socket address.
+        let parts = parts_with(Some("203.0.113.9:9999"), Some("10.0.0.1"));
+        assert_eq!(
+            extract_client_ip(&parts),
+            "203.0.113.9".parse::<IpAddr>().unwrap()
+        );
+    }
+
+    #[test]
+    fn loopback_connection_without_header_uses_socket_ip() {
+        let parts = parts_with(Some("127.0.0.1:5000"), None);
+        assert_eq!(
+            extract_client_ip(&parts),
+            "127.0.0.1".parse::<IpAddr>().unwrap()
+        );
+    }
 }

core/archipelago/src/api/rpc/mod.rs

@@ -58,9 +58,13 @@ use middleware::{
     derive_csrf_token, extract_client_ip, extract_cookie, sanitize_error_message,
     CACHEABLE_METHODS, UNAUTHENTICATED_METHODS,
 };
+pub use middleware::PeerAddr;
 use response::{cookie_header, json_response, ResponseCache, RpcError, RpcRequest, RpcResponse};
 
 /// Default dev password when no user is set up (matches mock-backend).
+/// Dev builds only — the pre-setup login bypass that reads this is
+/// cfg-gated out of release binaries.
+#[cfg(debug_assertions)]
 pub(crate) const DEV_DEFAULT_PASSWORD: &str = "password123";
 
 pub struct RpcHandler {
@@ -369,7 +373,7 @@ impl RpcHandler {
 
         // Rate limit login attempts
         if rpc_req.method == "auth.login" {
-            let client_ip = extract_client_ip(&parts.headers);
+            let client_ip = extract_client_ip(&parts);
             if !self.login_rate_limiter.check(client_ip).await {
                 return Ok(self.rate_limit_response());
             }
@@ -377,7 +381,7 @@ impl RpcHandler {
 
         // Rate limit sensitive endpoints
         {
-            let client_ip = extract_client_ip(&parts.headers);
+            let client_ip = extract_client_ip(&parts);
             if !self
                 .endpoint_rate_limiter
                 .check(&rpc_req.method, client_ip)
@@ -451,7 +455,7 @@ impl RpcHandler {
         let mut response = json_response(StatusCode::OK, &resp_body);
 
         // Post-dispatch: set cookies for auth-related methods
-        let client_ip = extract_client_ip(&parts.headers);
+        let client_ip = extract_client_ip(&parts);
         self.apply_auth_cookies(
             &rpc_req.method,
             &mut rpc_resp,

core/archipelago/src/api/rpc/package/config.rs

@@ -94,35 +94,11 @@ async fn dynamic_app_config(
     ))
 }
 
-/// Trusted Docker registries. Only images from these sources are allowed.
-#[allow(dead_code)]
-pub(super) const TRUSTED_REGISTRIES: &[&str] = &[
-    "docker.io/",
-    "ghcr.io/",
-    "localhost/",
-    "git.tx1138.com/",
-    "146.59.87.168:3000/",
-];
-
-/// Validate Docker image against trusted registry allowlist.
+/// Validate a Docker image reference. Delegates to the shared policy in
+/// `container::image_policy` — the same rules the orchestrator enforces at
+/// its pull sites, so the two layers can't drift apart.
 pub(super) fn is_valid_docker_image(image: &str) -> bool {
-    if image.is_empty() || image.len() > 256 {
-        return false;
-    }
-    // Reject shell metacharacters
-    let dangerous_chars = ['&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r'];
-    if image.chars().any(|c| dangerous_chars.contains(&c)) {
-        return false;
-    }
-    // Must come from a trusted registry — match the exact domain, not just prefix
-    let registry = match image.split('/').next() {
-        Some(r) => r,
-        None => return false,
-    };
-    matches!(
-        registry,
-        "docker.io" | "ghcr.io" | "localhost" | "git.tx1138.com" | "146.59.87.168:3000"
-    )
+    crate::container::image_policy::is_valid_docker_image(image)
 }
 
 /// Per-app Linux capabilities needed beyond the default cap-drop=ALL.

core/archipelago/src/api/rpc/package/dependencies.rs

@@ -472,7 +472,7 @@ pub(super) async fn check_bitcoin_pruning_compatibility(package_id: &str) -> Res
         tokio::time::sleep(std::time::Duration::from_secs(2)).await;
     }
 
-    if detect_disk_gb() < ARCHIVAL_BITCOIN_DISK_GB {
+    if detect_disk_gb().await < ARCHIVAL_BITCOIN_DISK_GB {
         anyhow::bail!(archival_bitcoin_required_message(package_id));
     }
 
@@ -497,10 +497,11 @@ fn check_blockchain_info_for_pruning(package_id: &str, json: &serde_json::Value)
     Ok(())
 }
 
-fn detect_disk_gb() -> u64 {
-    let output = std::process::Command::new("df")
+async fn detect_disk_gb() -> u64 {
+    let output = tokio::process::Command::new("df")
         .args(["-BG", "/var/lib/archipelago"])
-        .output();
+        .output()
+        .await;
     let Ok(output) = output else {
         return u64::MAX;
     };

core/archipelago/src/api/rpc/package/install.rs

@@ -2196,13 +2196,14 @@ async fn ensure_host_port_listener(
     container_name: &str,
     runtime_ports: &[String],
 ) -> Result<()> {
-    let Some(port) = runtime_ports
+    let mut port = runtime_ports
         .first()
         .and_then(|p| p.split(':').next())
-        .and_then(|p| p.parse::<u16>().ok())
-        .or_else(|| published_host_port(container_name))
-        .or_else(|| required_host_port(package_id))
-    else {
+        .and_then(|p| p.parse::<u16>().ok());
+    if port.is_none() {
+        port = published_host_port(container_name).await;
+    }
+    let Some(port) = port.or_else(|| required_host_port(package_id)) else {
         return Ok(());
     };
 
@@ -2248,10 +2249,11 @@ async fn ensure_host_port_listener(
     ))
 }
 
-fn published_host_port(container_name: &str) -> Option<u16> {
-    let output = std::process::Command::new("podman")
+async fn published_host_port(container_name: &str) -> Option<u16> {
+    let output = tokio::process::Command::new("podman")
         .args(["port", container_name])
         .output()
+        .await
         .ok()?;
     if !output.status.success() {
         return None;

core/archipelago/src/api/rpc/system/handlers.rs

@@ -575,7 +575,7 @@ impl RpcHandler {
         // Restart the service via systemd
         tokio::spawn(async {
             tokio::time::sleep(std::time::Duration::from_secs(2)).await;
-            let _ = std::process::Command::new("sudo")
+            let _ = tokio::process::Command::new("sudo")
                 .args(["systemctl", "restart", "archipelago"])
                 .spawn();
         });

core/archipelago/src/config.rs

@@ -81,10 +81,11 @@ pub struct Config {
 
 impl Config {
     /// Detect primary host IP (first non-loopback IPv4)
-    fn detect_host_ip() -> Result<String> {
-        let output = std::process::Command::new("hostname")
+    async fn detect_host_ip() -> Result<String> {
+        let output = tokio::process::Command::new("hostname")
             .args(["-I"])
             .output()
+            .await
             .context("Failed to run hostname -I")?;
         let s = String::from_utf8_lossy(&output.stdout);
         let ip = s
@@ -210,7 +211,9 @@ impl Config {
         if let Ok(ip) = std::env::var("ARCHIPELAGO_HOST_IP") {
             config.host_ip = ip;
         } else {
-            config.host_ip = Self::detect_host_ip().unwrap_or_else(|_| "127.0.0.1".to_string());
+            config.host_ip = Self::detect_host_ip()
+                .await
+                .unwrap_or_else(|_| "127.0.0.1".to_string());
         }
 
         // Ensure data directory exists

core/archipelago/src/container/image_policy.rs

@@ -0,0 +1,95 @@
+//! Trusted-registry policy for container image references — the single
+//! source of truth. The RPC boundary (`api::rpc::package::config`) and the
+//! orchestrator's pull sites both validate against this, so a catalog- or
+//! manifest-supplied ref can't reach `pull_image` unchecked (§A of the
+//! 1.8.0 hardening plan).
+
+/// Registries images may be pulled from with an explicit host part.
+pub const TRUSTED_REGISTRIES: &[&str] = &[
+    "docker.io",
+    "ghcr.io",
+    "localhost",
+    "git.tx1138.com",
+    "146.59.87.168:3000",
+];
+
+/// Validate a container image reference.
+///
+/// Accepts:
+///   * refs whose explicit registry host is on [`TRUSTED_REGISTRIES`]
+///     (`docker.io/grafana/grafana`, `146.59.87.168:3000/archy/x:1`), and
+///   * registry-less Docker Hub shorthand (`nginx`, `grafana/grafana`) —
+///     the first segment has no `.`/`:` so it cannot name an attacker host;
+///     resolution follows the host's registries.conf search order.
+///
+/// Rejects empty/oversized refs, shell metacharacters, and any ref whose
+/// explicit registry host is not on the allowlist.
+pub fn is_valid_docker_image(image: &str) -> bool {
+    if image.is_empty() || image.len() > 256 {
+        return false;
+    }
+    // Reject shell metacharacters
+    let dangerous_chars = [
+        '&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r', ' ', '\t',
+    ];
+    if image.chars().any(|c| dangerous_chars.contains(&c)) {
+        return false;
+    }
+    let first_segment = match image.split('/').next() {
+        Some(r) if !r.is_empty() => r,
+        _ => return false,
+    };
+    if TRUSTED_REGISTRIES.contains(&first_segment) {
+        return true;
+    }
+    // No dot/colon in the first segment ⇒ it's a Docker Hub namespace or a
+    // bare repo name, not a registry host — allowed. Anything that *looks*
+    // like a host (has a dot or port) but isn't allowlisted is rejected.
+    !first_segment.contains('.') && !first_segment.contains(':')
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn accepts_trusted_registries() {
+        for img in [
+            "docker.io/library/nginx:1.25",
+            "ghcr.io/owner/app:latest",
+            "localhost/archy-dev:1",
+            "git.tx1138.com/lfg2025/x:2",
+            "146.59.87.168:3000/archy/bitcoin-knots:28.1",
+        ] {
+            assert!(is_valid_docker_image(img), "{img} should be accepted");
+        }
+    }
+
+    #[test]
+    fn accepts_docker_hub_shorthand() {
+        for img in ["nginx", "grafana/grafana:11.2.0", "lightninglabs/lnd:v0.18"] {
+            assert!(is_valid_docker_image(img), "{img} should be accepted");
+        }
+    }
+
+    #[test]
+    fn rejects_untrusted_registry_hosts() {
+        for img in [
+            "evil.com/backdoor:latest",
+            "203.0.113.7:5000/x",
+            "registry.gitlab.com/x/y",
+            "quay.io/x/y",
+        ] {
+            assert!(!is_valid_docker_image(img), "{img} should be rejected");
+        }
+    }
+
+    #[test]
+    fn rejects_malformed_refs() {
+        assert!(!is_valid_docker_image(""));
+        assert!(!is_valid_docker_image(&"a".repeat(257)));
+        assert!(!is_valid_docker_image("docker.io/x; rm -rf /"));
+        assert!(!is_valid_docker_image("docker.io/$(curl evil)"));
+        assert!(!is_valid_docker_image("/leading-slash"));
+    }
+}

core/archipelago/src/container/mod.rs

@@ -7,6 +7,7 @@ pub mod dev_orchestrator;
 pub mod docker_packages;
 pub mod filebrowser;
 pub mod hooks;
+pub mod image_policy;
 pub mod image_versions;
 pub mod lnd;
 pub mod prod_orchestrator;

core/archipelago/src/container/prod_orchestrator.rs

@@ -32,7 +32,6 @@ use async_trait::async_trait;
 use std::collections::{HashMap, HashSet};
 use std::os::unix::fs::FileTypeExt;
 use std::path::{Path, PathBuf};
-use std::process::Command;
 use std::sync::Arc;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::sync::{Mutex, RwLock};
@@ -1065,6 +1064,11 @@ pub struct ProdContainerOrchestrator {
     /// false so the legacy path remains the production path until the
     /// 5× lifecycle harness goes green against the new path.
     use_quadlet_backends: bool,
+    /// app_id → last secret-env content hash pushed to the runtime's
+    /// secret store. Makes steady-state reconciles free of podman
+    /// secret calls; a rotation (hash change) falls through and
+    /// re-registers.
+    env_secret_cache: Mutex<HashMap<String, String>>,
     #[cfg(test)]
     test_disk_gb: Option<u64>,
     #[cfg(test)]
@@ -1127,6 +1131,7 @@ impl ProdContainerOrchestrator {
             lnd_paths: lnd::EnsurePaths::default(),
             secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"),
             use_quadlet_backends: config.use_quadlet_backends,
+            env_secret_cache: Mutex::new(HashMap::new()),
             #[cfg(test)]
             test_disk_gb: None,
             #[cfg(test)]
@@ -1148,6 +1153,7 @@ impl ProdContainerOrchestrator {
             lnd_paths: lnd::EnsurePaths::default(),
             secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"),
             use_quadlet_backends: false,
+            env_secret_cache: Mutex::new(HashMap::new()),
             test_disk_gb: None,
             test_bitcoin_host: None,
         }
@@ -1440,7 +1446,7 @@ impl ProdContainerOrchestrator {
                 .collect()
         };
         let mut report = ReconcileReport::default();
-        let disk_gb = self.disk_gb();
+        let disk_gb = self.disk_gb().await;
         // Register every candidate before the (sequential, possibly slow)
         // pass so the scanner overlays queued-but-down apps as Restarting
         // instead of Stopped. Each app is deregistered as its turn finishes,
@@ -1620,7 +1626,7 @@ impl ProdContainerOrchestrator {
         }
 
         let mut resolved_manifest = lm.manifest.clone();
-        self.resolve_dynamic_env(&mut resolved_manifest)?;
+        self.resolve_dynamic_env(&mut resolved_manifest).await?;
         let name = compute_container_name(&lm.manifest);
 
         // An explicitly user-stopped app MUST stay stopped. The reconcile filter
@@ -1970,7 +1976,7 @@ impl ProdContainerOrchestrator {
     async fn install_fresh(&self, lm: &LoadedManifest) -> Result<()> {
         self.ensure_app_secrets(&lm.manifest.app.id).await?;
         let mut resolved_manifest = lm.manifest.clone();
-        self.resolve_dynamic_env(&mut resolved_manifest)?;
+        self.resolve_dynamic_env(&mut resolved_manifest).await?;
         resolve_catalog_image(&mut resolved_manifest);
 
         let resolved = resolved_manifest.app.container.resolve().ok_or_else(|| {
@@ -1986,6 +1992,16 @@ impl ProdContainerOrchestrator {
                 image_signature,
                 ..
             } => {
+                // §A: validate at the pull site, not just the RPC boundary —
+                // catalog/manifest-supplied refs reach here without ever
+                // passing package::config's check.
+                if !crate::container::image_policy::is_valid_docker_image(&image) {
+                    anyhow::bail!(
+                        "refusing to pull image {:?} for {}: not from a trusted registry",
+                        image,
+                        lm.manifest.app.id
+                    );
+                }
                 self.runtime
                     .pull_image(&image, image_signature.as_deref())
                     .await
@@ -2263,7 +2279,7 @@ impl ProdContainerOrchestrator {
         // Re-render the manifest with dynamic env baked in, then go
         // through the same install path a fresh install would.
         let mut resolved = lm.manifest.clone();
-        self.resolve_dynamic_env(&mut resolved)?;
+        self.resolve_dynamic_env(&mut resolved).await?;
         self.install_via_quadlet(&resolved, name)
             .await
             .with_context(|| format!("Phase 3.3: re-install {name} via Quadlet"))?;
@@ -2329,7 +2345,7 @@ impl ProdContainerOrchestrator {
         let restart_required = quadlet::contains_stale_health_gate(&old_body);
 
         let mut resolved = lm.manifest.clone();
-        self.resolve_dynamic_env(&mut resolved)?;
+        self.resolve_dynamic_env(&mut resolved).await?;
         // Same catalog/pinned-version image resolution the installer applies, so
         // the drift re-render doesn't revert a pinned version back to the
         // manifest's shipped `:latest` tag on the next reconcile tick.
@@ -2404,7 +2420,7 @@ impl ProdContainerOrchestrator {
 
         for dep in dependencies {
             let mut resolved = dep.manifest.clone();
-            self.resolve_dynamic_env(&mut resolved)?;
+            self.resolve_dynamic_env(&mut resolved).await?;
             let name = compute_container_name(&dep.manifest);
             if self.runtime.get_container_status(&name).await.is_err() {
                 continue;
@@ -2431,6 +2447,14 @@ impl ProdContainerOrchestrator {
                 image_signature,
                 ..
             } => {
+                // Same trusted-registry gate as install_fresh (§A).
+                if !crate::container::image_policy::is_valid_docker_image(&image) {
+                    anyhow::bail!(
+                        "refusing to pull image {:?} for {}: not from a trusted registry",
+                        image,
+                        lm.manifest.app.id
+                    );
+                }
                 let exists = match self.runtime.image_exists(&image).await {
                     Ok(exists) => exists,
                     Err(err) => {
@@ -2588,7 +2612,7 @@ impl ProdContainerOrchestrator {
                 }
                 .read("bitcoin-rpc-password")
                 .context("lnd pre-start: read bitcoin RPC password")?;
-                let bitcoin_host = self.bitcoin_host();
+                let bitcoin_host = self.bitcoin_host().await;
                 let outcome = lnd::ensure_config(&self.lnd_paths, &rpc_pass, &bitcoin_host)
                     .await
                     .context("lnd pre-start: ensure lnd.conf")?;
@@ -2706,36 +2730,58 @@ impl ProdContainerOrchestrator {
         tokio::time::sleep(std::time::Duration::from_secs(1)).await;
     }
 
-    fn detect_host_facts(&self) -> HostFacts {
-        let host_ip = Self::detect_host_ip().unwrap_or_else(|| "127.0.0.1".to_string());
-        let host_mdns = Self::detect_host_mdns();
-        let disk_gb = self.disk_gb();
-        HostFacts {
-            host_ip,
-            host_mdns,
-            disk_gb,
-            // Cheap default; resolve_dynamic_env fills the real node name on
-            // demand (it costs a podman call) only for manifests that use
-            // {{BITCOIN_HOST}}, rather than every app on every reconcile.
-            bitcoin_host: "bitcoin-knots".to_string(),
+    async fn detect_host_facts(&self) -> HostFacts {
+        // Unit tests run under tokio's paused clock; awaiting a real
+        // subprocess there deadlocks against auto-advanced timers (the old
+        // BLOCKING detection only worked by never yielding). Canned facts.
+        #[cfg(test)]
+        {
+            return HostFacts {
+                host_ip: "127.0.0.1".to_string(),
+                host_mdns: "test.local".to_string(),
+                disk_gb: self.test_disk_gb.unwrap_or(1000),
+                bitcoin_host: "bitcoin-knots".to_string(),
+            };
+        }
+        #[allow(unreachable_code)]
+        {
+            let host_ip = Self::detect_host_ip()
+                .await
+                .unwrap_or_else(|| "127.0.0.1".to_string());
+            let host_mdns = Self::detect_host_mdns().await;
+            let disk_gb = self.disk_gb().await;
+            HostFacts {
+                host_ip,
+                host_mdns,
+                disk_gb,
+                // Cheap default; resolve_dynamic_env fills the real node name on
+                // demand (it costs a podman call) only for manifests that use
+                // {{BITCOIN_HOST}}, rather than every app on every reconcile.
+                bitcoin_host: "bitcoin-knots".to_string(),
+            }
         }
     }
 
     /// Container name of the running Bitcoin node (`bitcoin-knots` or
     /// `bitcoin-core`) for the `{{BITCOIN_HOST}}` derived-env placeholder.
-    /// Synchronous `podman ps` to match the surrounding host-fact detection;
-    /// defaults to `bitcoin-knots` when none is running (B12).
-    fn bitcoin_host(&self) -> String {
+    /// Defaults to `bitcoin-knots` when none is running (B12).
+    async fn bitcoin_host(&self) -> String {
+        // No real podman under the tests' paused clock (see detect_host_facts).
         #[cfg(test)]
-        if let Some(host) = &self.test_bitcoin_host {
-            return host.clone();
+        {
+            return self
+                .test_bitcoin_host
+                .clone()
+                .unwrap_or_else(|| "bitcoin-knots".to_string());
         }
+        #[allow(unreachable_code)]
         // Mirrors api::rpc::package::dependencies (the legacy install path);
         // both Bitcoin node variants are reachable on archy-net by name.
         const BITCOIN_NAMES: &[&str] = &["bitcoin-knots", "bitcoin-core", "bitcoin"];
-        let names = Command::new("podman")
+        let names = tokio::process::Command::new("podman")
             .args(["ps", "--format", "{{.Names}}"])
             .output()
+            .await
             .ok()
             .filter(|o| o.status.success())
             .map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
@@ -2753,8 +2799,12 @@ impl ProdContainerOrchestrator {
         self.test_bitcoin_host = Some(host.to_string());
     }
 
-    fn detect_host_ip() -> Option<String> {
-        let output = Command::new("hostname").arg("-I").output().ok()?;
+    async fn detect_host_ip() -> Option<String> {
+        let output = tokio::process::Command::new("hostname")
+            .arg("-I")
+            .output()
+            .await
+            .ok()?;
         if !output.status.success() {
             return None;
         }
@@ -2762,9 +2812,10 @@ impl ProdContainerOrchestrator {
         stdout.split_whitespace().next().map(|s| s.to_string())
     }
 
-    fn detect_host_mdns() -> String {
-        let hostname = Command::new("hostname")
+    async fn detect_host_mdns() -> String {
+        let hostname = tokio::process::Command::new("hostname")
             .output()
+            .await
             .ok()
             .and_then(|o| {
                 if o.status.success() {
@@ -2782,13 +2833,18 @@ impl ProdContainerOrchestrator {
         }
     }
 
-    fn detect_disk_gb() -> u64 {
+    async fn detect_disk_gb() -> u64 {
         let target = if Path::new("/var/lib/archipelago").exists() {
             "/var/lib/archipelago"
         } else {
             "/"
         };
-        let output = match Command::new("df").arg("-k").arg(target).output() {
+        let output = match tokio::process::Command::new("df")
+            .arg("-k")
+            .arg(target)
+            .output()
+            .await
+        {
             Ok(o) if o.status.success() => o,
             _ => return 0,
         };
@@ -2808,12 +2864,16 @@ impl ProdContainerOrchestrator {
         kb / 1_000_000
     }
 
-    fn disk_gb(&self) -> u64 {
+    async fn disk_gb(&self) -> u64 {
+        // No real df under the tests' paused clock (see detect_host_facts).
         #[cfg(test)]
-        if let Some(disk_gb) = self.test_disk_gb {
-            return disk_gb;
+        {
+            return self.test_disk_gb.unwrap_or(1000);
+        }
+        #[allow(unreachable_code)]
+        {
+            Self::detect_disk_gb().await
         }
-        Self::detect_disk_gb()
     }
 
     /// Ensure app-specific secrets exist *before* env resolution. The Bitcoin
@@ -2838,14 +2898,21 @@ impl ProdContainerOrchestrator {
         Ok(())
     }
 
-    fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
+    async fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
+        // Idempotency guard: partitioning already ran on this instance.
+        // Re-running would re-taint against an environment that no longer
+        // contains the composite entries and silently drop them. Callers
+        // always resolve a fresh clone, so this only trips on misuse.
+        if !manifest.app.container.secret_env_refs.is_empty() {
+            return Ok(());
+        }
         // Materialise any manifest-declared generated secrets before they're
         // read below. This is the single chokepoint every install/reconcile
         // path funnels through, so an app's secrets exist by the time its
         // `secret_env` resolves — no per-app code, no host provisioning.
         crate::container::secrets::ensure_generated_secrets(&self.secrets_dir, manifest)?;
 
-        let mut facts = self.detect_host_facts();
+        let mut facts = self.detect_host_facts().await;
         // Only pay the podman cost to detect Knots-vs-Core when this manifest
         // actually templates the Bitcoin node into its env (mempool — B12).
         if manifest
@@ -2855,18 +2922,23 @@ impl ProdContainerOrchestrator {
             .iter()
             .any(|e| e.template.contains("{{BITCOIN_HOST}}"))
         {
-            facts.bitcoin_host = self.bitcoin_host();
+            facts.bitcoin_host = self.bitcoin_host().await;
         }
         let mut env = manifest.app.environment.clone();
         env.extend(manifest.app.container.resolve_derived_env(&facts));
 
+        if manifest.app.id == "fedimint" || manifest.app.id == "fedimintd" {
+            env.retain(|entry| !entry.starts_with("FM_BITCOIND_URL="));
+            env.push("FM_BITCOIND_URL=http://bitcoin-knots:8332".to_string());
+        }
+
         let provider = FileSecretsProvider {
             root: self.secrets_dir.clone(),
         };
-        let secrets = manifest
+        let secret_pairs = manifest
             .app
             .container
-            .resolve_secret_env(&provider)
+            .resolve_secret_env_pairs(&provider)
             .map_err(|e| anyhow::anyhow!(e.to_string()))
             .with_context(|| {
                 format!(
@@ -2875,36 +2947,57 @@ impl ProdContainerOrchestrator {
                     self.secrets_dir.display()
                 )
             })?;
-        env.extend(secrets);
-        if manifest.app.id == "fedimint" || manifest.app.id == "fedimintd" {
-            env.retain(|entry| !entry.starts_with("FM_BITCOIND_URL="));
-            env.push("FM_BITCOIND_URL=http://bitcoin-knots:8332".to_string());
-        }
-        Self::expand_env_placeholders(&mut env);
-        manifest.app.environment = env;
-        Ok(())
-    }
 
-    fn expand_env_placeholders(env: &mut Vec<String>) {
-        let values: HashMap<String, String> = env
-            .iter()
-            .filter_map(|entry| {
-                let (key, value) = entry.split_once('=')?;
-                Some((key.to_string(), value.to_string()))
-            })
-            .collect();
+        // Secret values never merge into `environment` — they'd land in
+        // `podman inspect` output and Quadlet unit files on disk. Instead:
+        // expand ${KEY} placeholders (a plain entry that interpolates a
+        // secret is tainted and travels as a secret itself — btcpay's
+        // Password=${BTCPAY_DB_PASS} connection strings), keep the plain
+        // remainder as env, and hand the secret-bearing pairs to the
+        // runtime's secret store by reference.
+        let (plain, secret_bearing) =
+            archipelago_container::manifest::expand_and_partition_env(env, secret_pairs);
+        manifest.app.environment = plain;
+        if secret_bearing.is_empty() {
+            manifest.app.container.secret_env_refs = Vec::new();
+            manifest.app.container.secret_env_hash = None;
+        } else {
+            let hash =
+                archipelago_container::manifest::secret_env_content_hash(&secret_bearing);
+            let app_id = manifest.app.id.clone();
+            manifest.app.container.secret_env_refs = secret_bearing
+                .into_iter()
+                .map(|(key, value)| archipelago_container::manifest::SecretEnvRef {
+                    secret_name: format!(
+                        "archy-env-{}-{}",
+                        app_id,
+                        key.to_ascii_lowercase()
+                    ),
+                    env_key: key,
+                    value,
+                })
+                .collect();
+            manifest.app.container.secret_env_hash = Some(hash.clone());
 
-        for entry in env.iter_mut() {
-            let Some((key, value)) = entry.split_once('=') else {
-                continue;
-            };
-            let mut expanded = value.to_string();
-            for (placeholder_key, placeholder_value) in &values {
-                expanded =
-                    expanded.replace(&format!("${{{}}}", placeholder_key), placeholder_value);
+            // Register/refresh the podman secrets. A per-app hash cache makes
+            // the steady-state reconcile free: podman is only consulted when
+            // the resolved content actually changed (or on first touch after
+            // boot). Mock runtimes no-op via the trait default.
+            let cached = self
+                .env_secret_cache
+                .lock()
+                .await
+                .get(&app_id)
+                .cloned();
+            if cached.as_deref() != Some(hash.as_str()) {
+                self.runtime
+                    .ensure_env_secrets(&manifest.app.container.secret_env_refs)
+                    .await
+                    .with_context(|| format!("registering env secrets for {app_id}"))?;
+                self.env_secret_cache.lock().await.insert(app_id, hash);
             }
-            *entry = format!("{}={}", key, expanded);
         }
+        Ok(())
     }
 
     async fn container_env_drifted(&self, name: &str, manifest: &AppManifest) -> bool {
@@ -2940,12 +3033,40 @@ impl ProdContainerOrchestrator {
             })
             .collect();
 
-        manifest.app.environment.iter().any(|entry| {
+        if manifest.app.environment.iter().any(|entry| {
             let Some((key, expected)) = entry.split_once('=') else {
                 return false;
             };
             current.get(key).map_or(true, |actual| actual != expected)
-        })
+        }) {
+            return true;
+        }
+
+        // Secret-backed env never appears in Config.Env (that's the point) —
+        // rotation is detected via the content-hash label stamped at create
+        // time. A pre-upgrade container has no label, which reads as drift
+        // and triggers the one-time recreate that scrubs its plaintext
+        // secrets out of `podman inspect`.
+        if let Some(expected_hash) = &manifest.app.container.secret_env_hash {
+            let fmt = format!(
+                "{{{{ index .Config.Labels \"{}\" }}}}",
+                archipelago_container::manifest::SECRET_ENV_HASH_LABEL
+            );
+            let inspect = tokio::process::Command::new("podman")
+                .args(["inspect", name, "--format", &fmt])
+                .output()
+                .await;
+            let Ok(output) = inspect else {
+                return false;
+            };
+            if !output.status.success() {
+                return false;
+            }
+            if String::from_utf8_lossy(&output.stdout).trim() != expected_hash {
+                return true;
+            }
+        }
+        false
     }
 
     async fn container_command_drifted(&self, name: &str, manifest: &AppManifest) -> bool {
@@ -3208,7 +3329,7 @@ impl ProdContainerOrchestrator {
     ) -> Result<String> {
         let mut out = content.to_string();
         if out.contains("{{HOST_IP}}") || out.contains("{{HOST_MDNS}}") {
-            let facts = self.detect_host_facts();
+            let facts = self.detect_host_facts().await;
             out = out
                 .replace("{{HOST_IP}}", &facts.host_ip)
                 .replace("{{HOST_MDNS}}", &facts.host_mdns);
@@ -3297,7 +3418,7 @@ impl ProdContainerOrchestrator {
     /// however the box is reached locally. (Generalised from the old per-app
     /// netbird TLS helper, deleted in #20 ph4: rsa:2048, 10-year, no per-app Rust.)
     async fn ensure_manifest_certs(&self, manifest: &AppManifest) -> Result<()> {
-        let facts = self.detect_host_facts();
+        let facts = self.detect_host_facts().await;
         let render = |s: &str| {
             s.replace("{{HOST_IP}}", &facts.host_ip)
                 .replace("{{HOST_MDNS}}", &facts.host_mdns)
@@ -3594,7 +3715,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
         self.ensure_app_secrets(app_id).await?;
         let name = compute_container_name(&lm.manifest);
         let mut resolved_manifest = lm.manifest.clone();
-        self.resolve_dynamic_env(&mut resolved_manifest)?;
+        self.resolve_dynamic_env(&mut resolved_manifest).await?;
 
         let service = format!("{name}.service");
         if self.quadlet_unit_exists(&name).await? {
@@ -3830,6 +3951,9 @@ mod tests {
         images: StdMutex<HashMap<String, bool>>,
         /// container_name -> env that create_container received.
         created_env: StdMutex<HashMap<String, Vec<String>>>,
+        /// container_name -> secret env refs that create_container received.
+        created_secret_refs:
+            StdMutex<HashMap<String, Vec<archipelago_container::manifest::SecretEnvRef>>>,
         /// If set, the next `build_image` call fails with this message.
         fail_build: StdMutex<Option<String>>,
         /// If set, `image_exists` fails for this image reference.
@@ -3868,6 +3992,17 @@ mod tests {
                 .cloned()
                 .unwrap_or_default()
         }
+        fn created_secret_refs_for(
+            &self,
+            name: &str,
+        ) -> Vec<archipelago_container::manifest::SecretEnvRef> {
+            self.created_secret_refs
+                .lock()
+                .unwrap()
+                .get(name)
+                .cloned()
+                .unwrap_or_default()
+        }
     }
 
     #[async_trait]
@@ -3889,6 +4024,10 @@ mod tests {
                 .lock()
                 .unwrap()
                 .insert(name.to_string(), manifest.app.environment.clone());
+            self.created_secret_refs.lock().unwrap().insert(
+                name.to_string(),
+                manifest.app.container.secret_env_refs.clone(),
+            );
             Ok(name.to_string())
         }
         async fn start_container(&self, name: &str) -> Result<()> {
@@ -4328,7 +4467,7 @@ app:
             orch.set_bitcoin_host_for_test(node);
 
             let mut manifest = AppManifest::parse(yaml).unwrap();
-            orch.resolve_dynamic_env(&mut manifest).unwrap();
+            orch.resolve_dynamic_env(&mut manifest).await.unwrap();
 
             assert!(
                 manifest
@@ -4528,7 +4667,17 @@ app:
         assert!(env
             .iter()
             .any(|e| e.starts_with("FM_API_URL=ws://") && e.ends_with(":8174")));
-        assert!(env.iter().any(|e| e == "FM_BITCOIND_PASSWORD=secret-pass"));
+        // The secret must NOT ride in plain env (that's the podman-inspect /
+        // quadlet-unit-file leak this pipeline exists to close) — it travels
+        // as a secret ref with the value bound for the podman secret store.
+        assert!(!env.iter().any(|e| e.starts_with("FM_BITCOIND_PASSWORD=")));
+        let refs = rt.created_secret_refs_for("fedimint");
+        let r = refs
+            .iter()
+            .find(|r| r.env_key == "FM_BITCOIND_PASSWORD")
+            .expect("secret env must arrive as a ref");
+        assert_eq!(r.value, "secret-pass");
+        assert_eq!(r.secret_name, "archy-env-fedimint-fm_bitcoind_password");
     }
 
     #[tokio::test]

core/archipelago/src/container/quadlet.rs

@@ -142,6 +142,14 @@ pub struct QuadletUnit {
     // companion's rendered bytes are unchanged from before this PR.
     pub ports: Vec<(u16, u16, String)>,
     pub environment: Vec<String>,
+    /// Secret-backed env: (env_key, podman secret name). Rendered as
+    /// `Secret=<name>,type=env,target=<key>` so the VALUE never lands in
+    /// this unit file on disk — only a reference to the podman secret
+    /// store. The orchestrator registers the secrets before writing units.
+    pub secret_env: Vec<(String, String)>,
+    /// Container labels (`Label=k=v`). Carries the secret-env content hash
+    /// for rotation-drift detection.
+    pub labels: Vec<(String, String)>,
     pub devices: Vec<String>,
     pub add_hosts: Vec<(String, String)>,
     pub network_aliases: Vec<String>,
@@ -247,6 +255,12 @@ impl QuadletUnit {
             // accepts that form on a single Environment= line per pair.
             let _ = writeln!(s, "Environment={}", quote_environment(env));
         }
+        for (key, secret_name) in &self.secret_env {
+            let _ = writeln!(s, "Secret={secret_name},type=env,target={key}");
+        }
+        for (k, v) in &self.labels {
+            let _ = writeln!(s, "Label={k}={v}");
+        }
         for dev in &self.devices {
             let _ = writeln!(s, "AddDevice={dev}");
         }
@@ -415,6 +429,23 @@ impl QuadletUnit {
                 .map(|p| (p.host, p.container, p.protocol.clone()))
                 .collect(),
             environment: app.environment.clone(),
+            secret_env: app
+                .container
+                .secret_env_refs
+                .iter()
+                .map(|r| (r.env_key.clone(), r.secret_name.clone()))
+                .collect(),
+            labels: app
+                .container
+                .secret_env_hash
+                .iter()
+                .map(|h| {
+                    (
+                        archipelago_container::manifest::SECRET_ENV_HASH_LABEL.to_string(),
+                        h.clone(),
+                    )
+                })
+                .collect(),
             devices: app.devices.clone(),
             add_hosts: vec![("host.archipelago".into(), "10.89.0.1".into())],
             // Container always answers to its own name; manifest extras add the
@@ -847,6 +878,24 @@ mod tests {
     use super::*;
     use tempfile::tempdir;
 
+    #[test]
+    fn render_emits_secret_env_by_reference_never_value() {
+        let u = QuadletUnit {
+            name: "t".into(),
+            description: "t".into(),
+            image: "img".into(),
+            secret_env: vec![("DB_PASS".into(), "archy-env-app-db_pass".into())],
+            labels: vec![("io.archipelago.secret-env-hash".into(), "abc123".into())],
+            ..QuadletUnit::default()
+        };
+        let s = u.render();
+        assert!(s.contains("Secret=archy-env-app-db_pass,type=env,target=DB_PASS"));
+        assert!(s.contains("Label=io.archipelago.secret-env-hash=abc123"));
+        // the secret VALUE never had a path into this unit — but guard the
+        // env channel anyway: no Environment= line may mention the key
+        assert!(!s.contains("Environment=DB_PASS"));
+    }
+
     fn sample_unit() -> QuadletUnit {
         QuadletUnit {
             name: "archy-bitcoin-ui".into(),

core/archipelago/src/mesh/listener/decode.rs

@@ -563,7 +563,9 @@ pub(super) async fn handle_identity_received(
     // Update peer record
     let peer = MeshPeer {
         contact_id,
-        advert_name: format!("Archy-{}", &did[8..16.min(did.len())]),
+        // .get(): a malformed DID shorter than the "did:key:" prefix must
+        // not panic the listener on a radio-supplied string.
+        advert_name: format!("Archy-{}", did.get(8..16.min(did.len())).unwrap_or(did)),
         did: Some(did.to_string()),
         pubkey_hex: Some(ed_pubkey_hex.to_string()),
         // The advert signature was verified above, so this is an authenticated

core/archipelago/src/mesh/message_types.rs

@@ -258,7 +258,31 @@ impl TypedEnvelope {
         }
     }
 
-    /// Verify signature if present.
+    /// Signing preimage v2: binds the anti-replay `seq` so a radio MITM
+    /// can't reorder/replay a signed message under a different sequence
+    /// number. v1 (legacy) covers only (t, v, ts).
+    fn signing_preimage_v2(&self) -> Vec<u8> {
+        let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4 + 8);
+        sign_data.push(self.t);
+        sign_data.extend_from_slice(&self.v);
+        sign_data.extend_from_slice(&self.ts.to_le_bytes());
+        sign_data.extend_from_slice(&self.seq.to_le_bytes());
+        sign_data
+    }
+
+    fn signing_preimage_v1(&self) -> Vec<u8> {
+        let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4);
+        sign_data.push(self.t);
+        sign_data.extend_from_slice(&self.v);
+        sign_data.extend_from_slice(&self.ts.to_le_bytes());
+        sign_data
+    }
+
+    /// Verify signature if present. Accepts the seq-binding v2 preimage OR
+    /// the legacy (t, v, ts) preimage — senders still emit v1 until the
+    /// whole fleet verifies v2 (receivers hard-drop bad signatures, so
+    /// flipping the send side first would break mixed-fleet alerts). The
+    /// seq-tampering window closes only when the v1 arm is removed.
     pub fn verify_signature(&self, verifying_key: &ed25519_dalek::VerifyingKey) -> Result<bool> {
         let Some(sig_bytes) = &self.sig else {
             return Ok(false);
@@ -266,13 +290,14 @@ impl TypedEnvelope {
         let signature =
             ed25519_dalek::Signature::from_slice(sig_bytes).context("Invalid signature bytes")?;
 
-        let mut sign_data = Vec::with_capacity(1 + self.v.len() + 4);
-        sign_data.push(self.t);
-        sign_data.extend_from_slice(&self.v);
-        sign_data.extend_from_slice(&self.ts.to_le_bytes());
-
+        if verifying_key
+            .verify_strict(&self.signing_preimage_v2(), &signature)
+            .is_ok()
+        {
+            return Ok(true);
+        }
         verifying_key
-            .verify_strict(&sign_data, &signature)
+            .verify_strict(&self.signing_preimage_v1(), &signature)
             .context("Signature verification failed")?;
         Ok(true)
     }
@@ -284,12 +309,25 @@ impl TypedEnvelope {
 
     /// Set the outbound sequence number. Called by the send path after the
     /// target's counter has been incremented. Safe to call AFTER `new_signed`
-    /// because the signature covers `(t, v, ts)` — not `seq`.
+    /// because the v1 signature covers `(t, v, ts)` — not `seq`. Once the
+    /// fleet is on a build whose `verify_signature` accepts the v2 preimage,
+    /// flip senders to sign AFTER seq allocation via `signed_with_seq`.
     pub fn with_seq(mut self, seq: u64) -> Self {
         self.seq = seq;
         self
     }
 
+    /// v2 sender path (NOT yet wired — see `verify_signature` for the fleet
+    /// migration order): set seq first, then sign binding it.
+    #[allow(dead_code)]
+    pub fn signed_with_seq(mut self, seq: u64, signing_key: &ed25519_dalek::SigningKey) -> Self {
+        use ed25519_dalek::Signer;
+        self.seq = seq;
+        let signature = signing_key.sign(&self.signing_preimage_v2());
+        self.sig = Some(signature.to_bytes().to_vec());
+        self
+    }
+
     /// Encode to wire format: [0x02] [CBOR envelope].
     pub fn to_wire(&self) -> Result<Vec<u8>> {
         let mut buf = Vec::new();
@@ -816,6 +854,37 @@ mod tests {
         assert!(envelope.verify_signature(&key.verifying_key()).is_err());
     }
 
+    #[test]
+    fn test_v2_seq_bound_signature() {
+        use ed25519_dalek::SigningKey;
+        use rand::rngs::OsRng;
+
+        let key = SigningKey::generate(&mut OsRng);
+        let envelope = TypedEnvelope::new(MeshMessageType::Alert, b"test".to_vec())
+            .signed_with_seq(42, &key);
+        assert!(envelope.verify_signature(&key.verifying_key()).unwrap());
+
+        // v2 binds seq: replaying the signed envelope under a different
+        // sequence number must fail verification.
+        let mut replayed = envelope.clone();
+        replayed.seq = 43;
+        assert!(replayed.verify_signature(&key.verifying_key()).is_err());
+    }
+
+    #[test]
+    fn test_v1_signature_survives_seq_set_after_signing() {
+        use ed25519_dalek::SigningKey;
+        use rand::rngs::OsRng;
+
+        // Mixed-fleet compatibility: current senders sign first (v1
+        // preimage, no seq) and allocate seq afterwards; verify must still
+        // accept that.
+        let key = SigningKey::generate(&mut OsRng);
+        let envelope = TypedEnvelope::new_signed(MeshMessageType::Alert, b"test".to_vec(), &key)
+            .with_seq(7);
+        assert!(envelope.verify_signature(&key.verifying_key()).unwrap());
+    }
+
     #[test]
     fn test_invoice_payload_roundtrip() {
         let invoice = InvoicePayload {

core/archipelago/src/server.rs

@@ -1034,9 +1034,13 @@ async fn accept_loop(
                 let permit = active_connections.clone().acquire_owned().await;
                 tokio::spawn(async move {
                     let _permit = permit;
-                    let service = service_fn(move |req: hyper::Request<hyper::Body>| {
+                    let service = service_fn(move |mut req: hyper::Request<hyper::Body>| {
                         let handler = handler.clone();
                         async move {
+                            // Record the TCP peer so rate limiting only trusts
+                            // forwarded headers on loopback (nginx) connections.
+                            req.extensions_mut()
+                                .insert(crate::api::rpc::PeerAddr(peer_addr));
                             if peer_only && !is_peer_allowed_path(req.uri().path()) {
                                 let resp = hyper::Response::builder()
                                     .status(hyper::StatusCode::NOT_FOUND)

core/archipelago/src/transport/fips.rs

@@ -23,26 +23,36 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 /// TTL keeps the result responsive to daemon flaps without pounding DBus.
 const AVAILABILITY_CACHE_TTL: Duration = Duration::from_secs(10);
 
+/// Availability cache shared with the background probe thread, so the
+/// sync `is_available()` hot path never blocks on `systemctl`.
+struct AvailabilityCache {
+    available: AtomicBool,
+    probed_at_ms: AtomicU64,
+    probe_in_flight: AtomicBool,
+}
+
 pub struct FipsTransport {
     identity_dir: PathBuf,
-    available_cached: AtomicBool,
-    available_cached_at_ms: AtomicU64,
+    availability: std::sync::Arc<AvailabilityCache>,
 }
 
 impl FipsTransport {
     pub fn new(identity_dir: &Path) -> Self {
         Self {
             identity_dir: identity_dir.to_path_buf(),
-            available_cached: AtomicBool::new(false),
-            available_cached_at_ms: AtomicU64::new(0),
+            availability: std::sync::Arc::new(AvailabilityCache {
+                available: AtomicBool::new(false),
+                probed_at_ms: AtomicU64::new(0),
+                probe_in_flight: AtomicBool::new(false),
+            }),
         }
     }
 
     fn probe_daemon_active() -> bool {
-        // Cheap blocking probe: spawn `systemctl is-active` synchronously.
-        // Short-circuit if either the archipelago-managed unit or the
-        // upstream fips.service is active — legacy/dev nodes run only the
-        // upstream unit.
+        // Blocking probe — only ever run on a dedicated background thread
+        // (see is_available), never on a tokio worker. Short-circuit if
+        // either the archipelago-managed unit or the upstream fips.service
+        // is active — legacy/dev nodes run only the upstream unit.
         for unit in [
             crate::fips::SERVICE_UNIT,
             crate::fips::UPSTREAM_SERVICE_UNIT,
@@ -70,14 +80,30 @@ impl NodeTransport for FipsTransport {
             .duration_since(UNIX_EPOCH)
             .map(|d| d.as_millis() as u64)
             .unwrap_or(0);
-        let cached_at = self.available_cached_at_ms.load(Ordering::Relaxed);
+        let cached_at = self.availability.probed_at_ms.load(Ordering::Relaxed);
+        let cached = self.availability.available.load(Ordering::Relaxed);
         if now_ms.saturating_sub(cached_at) < AVAILABILITY_CACHE_TTL.as_millis() as u64 {
-            return self.available_cached.load(Ordering::Relaxed);
+            return cached;
         }
-        let val = Self::probe_daemon_active();
-        self.available_cached.store(val, Ordering::Relaxed);
-        self.available_cached_at_ms.store(now_ms, Ordering::Relaxed);
-        val
+        // Cache is stale. This sync trait method is called from async
+        // route(), so running the ~50ms systemctl probe inline stalls a
+        // tokio worker. Serve the stale value and refresh on a background
+        // thread instead — the transport supervisor's warm loop keeps this
+        // fresh in steady state, so staleness is bounded to one probe round.
+        let cache = std::sync::Arc::clone(&self.availability);
+        if !cache.probe_in_flight.swap(true, Ordering::Relaxed) {
+            std::thread::spawn(move || {
+                let val = Self::probe_daemon_active();
+                let probed_ms = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .map(|d| d.as_millis() as u64)
+                    .unwrap_or(0);
+                cache.available.store(val, Ordering::Relaxed);
+                cache.probed_at_ms.store(probed_ms, Ordering::Relaxed);
+                cache.probe_in_flight.store(false, Ordering::Relaxed);
+            });
+        }
+        cached
     }
 
     fn send<'a>(

core/archipelago/src/update.rs

@@ -513,8 +513,64 @@ async fn probe_frontend_once() -> Result<()> {
     anyhow::bail!("frontend probe returned HTTP {}", status);
 }
 
+/// Probe the backend RPC API through nginx. An unauthenticated call is
+/// EXPECTED to get 401/403 — any such response proves the Rust HTTP stack
+/// is alive behind nginx. 5xx (backend dead → nginx 502), 404 (proxy
+/// misroute), or connection errors mean the API is down even though the
+/// static frontend may still serve — exactly the failure mode the plain
+/// `GET /` probe waved through.
+async fn probe_backend_once() -> Result<()> {
+    let client = reqwest::Client::builder()
+        .danger_accept_invalid_certs(true)
+        .timeout(std::time::Duration::from_secs(5))
+        .build()
+        .context("build probe client")?;
+    let body = serde_json::json!({ "method": "update.status" });
+    let resp = match client
+        .post("https://127.0.0.1/rpc/v1")
+        .json(&body)
+        .send()
+        .await
+    {
+        Ok(resp) => resp,
+        Err(e) if e.is_connect() => client
+            .post("http://127.0.0.1/rpc/v1")
+            .json(&body)
+            .send()
+            .await
+            .context("probe POST http://127.0.0.1/rpc/v1 (https not bound on loopback)")?,
+        Err(e) => return Err(e).context("probe POST https://127.0.0.1/rpc/v1"),
+    };
+    let status = resp.status();
+    if status.is_server_error() || status == reqwest::StatusCode::NOT_FOUND {
+        anyhow::bail!("backend RPC probe returned HTTP {}", status);
+    }
+    Ok(())
+}
+
+/// Probe that the rootless container runtime is reachable from the new
+/// binary (uid mapping / podman socket intact after the swap). A healthy
+/// node answers `podman ps` in well under a second.
+async fn probe_container_runtime_once() -> Result<()> {
+    let out = tokio::process::Command::new("podman")
+        .args(["ps", "--format", "{{.Names}}"])
+        .output()
+        .await
+        .context("spawn podman ps")?;
+    if !out.status.success() {
+        anyhow::bail!(
+            "podman ps exited {}: {}",
+            out.status,
+            String::from_utf8_lossy(&out.stderr).trim()
+        );
+    }
+    Ok(())
+}
+
 /// Called from main.rs startup. If a pending-verification marker is
-/// present, probe the frontend; on failure, trigger rollback and
+/// present, verify the node actually works on the new version — binary
+/// version matches the marker, frontend serves, backend RPC answers,
+/// rootless podman is reachable. On failure, trigger rollback and
 /// restart the service so the OLD binary boots.
 ///
 /// This is the "post-OTA auto-rollback" guardrail. If ANY problem in
@@ -547,34 +603,59 @@ pub async fn verify_pending_update(data_dir: &Path) {
     info!(
         new_version = %marker.new_version,
         previous_version = %marker.previous_version,
-        "Post-OTA verification: probing frontend at https://127.0.0.1/"
+        "Post-OTA verification: probing frontend, backend RPC, and container runtime"
     );
 
-    // Give the new service time to bind its listeners + nginx to
-    // pick up any config changes. 15s matches what we observed on
-    // .116 during the v1.7.40 rollout recovery.
-    tokio::time::sleep(std::time::Duration::from_secs(15)).await;
-
-    let deadline =
-        std::time::Instant::now() + std::time::Duration::from_secs(PENDING_VERIFY_WINDOW_SECS);
+    // Binary identity check: if the running binary's version isn't the one
+    // the marker says we applied, the swap silently failed (or half-applied
+    // — new frontend with old binary). Deterministic, so no retry loop:
+    // fall through straight to rollback to restore a matched pair.
+    let running = env!("CARGO_PKG_VERSION");
     let mut attempt = 0u32;
     let mut last_err: Option<String> = None;
+    let version_ok = running == marker.new_version;
+    if !version_ok {
+        last_err = Some(format!(
+            "running binary is {} but marker says {} was applied — binary swap failed",
+            running, marker.new_version
+        ));
+    } else {
+        // Give the new service time to bind its listeners + nginx to
+        // pick up any config changes. 15s matches what we observed on
+        // .116 during the v1.7.40 rollout recovery.
+        tokio::time::sleep(std::time::Duration::from_secs(15)).await;
 
-    while std::time::Instant::now() < deadline {
-        attempt += 1;
-        match probe_frontend_once().await {
-            Ok(()) => {
-                info!(attempt, "Post-OTA verification succeeded — clearing marker");
-                clear_pending_verification(data_dir).await;
-                return;
-            }
-            Err(e) => {
-                let msg = e.to_string();
-                tracing::warn!(attempt, error = %msg, "Post-OTA probe failed, retrying");
-                last_err = Some(msg);
+        let deadline = std::time::Instant::now()
+            + std::time::Duration::from_secs(PENDING_VERIFY_WINDOW_SECS);
+
+        while std::time::Instant::now() < deadline {
+            attempt += 1;
+            // All three must pass in the same attempt: static frontend via
+            // nginx, backend RPC liveness, and rootless-podman reachability.
+            // (Individual app containers are NOT asserted — the pre-Quadlet
+            // service restart legitimately takes them down and the boot
+            // reconciler can need minutes to bring heavy apps back.)
+            let result = match probe_frontend_once().await {
+                Ok(()) => match probe_backend_once().await {
+                    Ok(()) => probe_container_runtime_once().await,
+                    Err(e) => Err(e),
+                },
+                Err(e) => Err(e),
+            };
+            match result {
+                Ok(()) => {
+                    info!(attempt, "Post-OTA verification succeeded — clearing marker");
+                    clear_pending_verification(data_dir).await;
+                    return;
+                }
+                Err(e) => {
+                    let msg = format!("{e:#}");
+                    tracing::warn!(attempt, error = %msg, "Post-OTA probe failed, retrying");
+                    last_err = Some(msg);
+                }
             }
+            tokio::time::sleep(std::time::Duration::from_secs(5)).await;
         }
-        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
     }
 
     tracing::error!(

core/container/Cargo.toml

@@ -19,6 +19,8 @@ chrono = { version = "0.4", features = ["serde"] }
 uuid = { version = "1.0", features = ["v4"] }
 log = "0.4"
 tracing = "0.1"
+sha2 = "0.10"
+hex = "0.4"
 
 [lib]
 name = "archipelago_container"

core/container/src/image_verify.rs

@@ -0,0 +1,207 @@
+//! Container image signature verification (cosign).
+//!
+//! The manifest/catalog `image_signature` field is a *claim* that the image
+//! is signed with the fleet's cosign key. Verification runs at the pull
+//! choke points (`PodmanClient::pull_image`, `DockerRuntime::pull_image`);
+//! a declared signature that cannot be verified hard-fails the pull.
+//!
+//! Every manifest has carried the literal placeholder `cosign://...` since
+//! the field was introduced — that means "not signed yet" and is treated as
+//! no claim, so enforcement stays dormant until the signing ceremony
+//! publishes real signatures AND nodes carry the pinned cosign public key.
+//! Ship order matters: key + cosign binary reach the fleet first, real
+//! signature values in the catalog come after.
+
+use anyhow::{bail, Context, Result};
+use std::path::PathBuf;
+
+/// The literal placeholder every pre-ceremony manifest carries.
+pub const SIGNATURE_PLACEHOLDER: &str = "cosign://...";
+
+/// Env override for the pinned cosign public key path (tests, staging).
+pub const COSIGN_PUBKEY_ENV: &str = "ARCHIPELAGO_COSIGN_PUBKEY";
+
+const DEFAULT_PUBKEY_PATH: &str = "/etc/archipelago/cosign.pub";
+
+const COSIGN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SignatureClaim {
+    /// No signature declared (field absent or empty).
+    None,
+    /// The literal `cosign://...` placeholder — manifest predates real signing.
+    Placeholder,
+    /// A real declared signature reference; MUST verify or the pull fails.
+    Declared(String),
+}
+
+pub fn classify_signature(signature: Option<&str>) -> SignatureClaim {
+    match signature.map(str::trim) {
+        None | Some("") => SignatureClaim::None,
+        Some(SIGNATURE_PLACEHOLDER) => SignatureClaim::Placeholder,
+        Some(s) => SignatureClaim::Declared(s.to_string()),
+    }
+}
+
+fn pinned_pubkey_path() -> PathBuf {
+    std::env::var(COSIGN_PUBKEY_ENV)
+        .map(PathBuf::from)
+        .unwrap_or_else(|_| PathBuf::from(DEFAULT_PUBKEY_PATH))
+}
+
+/// Verify a declared image signature with the fleet's pinned cosign key.
+/// Any failure — missing key, missing cosign binary, verification error —
+/// is a hard error: an image that CLAIMS to be signed must never be pulled
+/// on a node that can't prove the claim.
+pub async fn verify_declared_signature(
+    image: &str,
+    sig_ref: &str,
+    allow_insecure_registry: bool,
+) -> Result<()> {
+    verify_with_key_path(image, sig_ref, &pinned_pubkey_path(), allow_insecure_registry).await
+}
+
+async fn verify_with_key_path(
+    image: &str,
+    sig_ref: &str,
+    key_path: &std::path::Path,
+    allow_insecure_registry: bool,
+) -> Result<()> {
+    if !key_path.exists() {
+        bail!(
+            "Image '{image}' declares signature '{sig_ref}' but the pinned cosign \
+             public key is missing at {} (override with {COSIGN_PUBKEY_ENV}). \
+             Refusing to pull an image whose signature claim cannot be verified.",
+            key_path.display()
+        );
+    }
+
+    // Self-managed key => signatures aren't in the public Rekor transparency
+    // log, so tlog verification must be disabled explicitly (cosign v2
+    // defaults it on and would fail every private-key signature otherwise).
+    let mut cmd = tokio::process::Command::new("cosign");
+    cmd.arg("verify")
+        .arg("--key")
+        .arg(key_path)
+        .arg("--insecure-ignore-tlog=true");
+    if allow_insecure_registry {
+        // podman's --tls-verify=false covers both plain HTTP and bad TLS;
+        // cosign splits those into two flags — pass both to match.
+        cmd.arg("--allow-insecure-registry");
+        cmd.arg("--allow-http-registry");
+    }
+    cmd.arg(image);
+
+    let output = tokio::time::timeout(COSIGN_TIMEOUT, cmd.output())
+        .await
+        .map_err(|_| {
+            anyhow::anyhow!(
+                "cosign verify timed out after {}s for image '{image}'",
+                COSIGN_TIMEOUT.as_secs()
+            )
+        })?
+        .with_context(|| {
+            format!(
+                "Failed to run cosign for image '{image}' which declares signature \
+                 '{sig_ref}' — is cosign installed? A declared signature cannot be \
+                 skipped."
+            )
+        })?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        bail!(
+            "Signature verification FAILED for image '{image}' (declared: '{sig_ref}'): \
+             {stderr}"
+        );
+    }
+
+    tracing::info!("cosign signature verified for image {image}");
+    Ok(())
+}
+
+/// Shared pull-site gate: decide whether the pull may proceed.
+/// Returns Ok(()) for unsigned/placeholder claims (with a log line) and only
+/// after successful cosign verification for declared ones.
+pub async fn enforce_signature_claim(
+    image: &str,
+    signature: Option<&str>,
+    allow_insecure_registry: bool,
+) -> Result<()> {
+    match classify_signature(signature) {
+        SignatureClaim::None => {
+            tracing::debug!("image {image}: no signature declared, pulling unverified");
+            Ok(())
+        }
+        SignatureClaim::Placeholder => {
+            tracing::debug!(
+                "image {image}: signature is the pre-ceremony placeholder, pulling unverified"
+            );
+            Ok(())
+        }
+        SignatureClaim::Declared(sig_ref) => {
+            verify_declared_signature(image, &sig_ref, allow_insecure_registry).await
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn absent_and_empty_signatures_are_no_claim() {
+        assert_eq!(classify_signature(None), SignatureClaim::None);
+        assert_eq!(classify_signature(Some("")), SignatureClaim::None);
+        assert_eq!(classify_signature(Some("   ")), SignatureClaim::None);
+    }
+
+    #[test]
+    fn literal_placeholder_is_not_a_claim() {
+        assert_eq!(
+            classify_signature(Some("cosign://...")),
+            SignatureClaim::Placeholder
+        );
+        assert_eq!(
+            classify_signature(Some("  cosign://...  ")),
+            SignatureClaim::Placeholder
+        );
+    }
+
+    #[test]
+    fn real_values_are_declared_claims() {
+        assert_eq!(
+            classify_signature(Some("cosign://sha256-abc.sig")),
+            SignatureClaim::Declared("cosign://sha256-abc.sig".to_string())
+        );
+        // Unknown schemes still count as a claim — better to fail closed on
+        // a value we don't understand than to pull unverified.
+        assert_eq!(
+            classify_signature(Some("sigstore://whatever")),
+            SignatureClaim::Declared("sigstore://whatever".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn declared_signature_without_pinned_key_hard_fails() {
+        let err = verify_with_key_path(
+            "registry.example/app:1.0",
+            "cosign://sha256-abc.sig",
+            std::path::Path::new("/nonexistent/cosign.pub"),
+            false,
+        )
+        .await
+        .unwrap_err();
+        assert!(err.to_string().contains("pinned cosign public key is missing"));
+    }
+
+    #[tokio::test]
+    async fn unsigned_and_placeholder_claims_pass_the_gate() {
+        enforce_signature_claim("registry.example/app:1.0", None, false)
+            .await
+            .unwrap();
+        enforce_signature_claim("registry.example/app:1.0", Some("cosign://..."), false)
+            .await
+            .unwrap();
+    }
+}

core/container/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod bitcoin_simulator;
 pub mod health_monitor;
+pub mod image_verify;
 pub mod manifest;
 pub mod podman_client;
 pub mod port_manager;

core/container/src/manifest.rs

@@ -243,6 +243,22 @@ pub struct ContainerConfig {
     /// Example: `"100070:100070"` for Postgres' mapped subuid.
     #[serde(default)]
     pub data_uid: Option<String>,
+
+    /// Runtime-resolved secret env entries (never serialized, never in a
+    /// manifest file). Populated by the orchestrator's env-resolution
+    /// chokepoint; the backends turn each ref into a podman secret
+    /// reference (`secret_env` in the API spec / `Secret=…,type=env` in
+    /// Quadlet) so the VALUE never lands in `podman inspect` output or a
+    /// unit file on disk. The dev-only docker fallback injects `value` as
+    /// plain env — docker has no rootless secret store.
+    #[serde(skip)]
+    pub secret_env_refs: Vec<SecretEnvRef>,
+
+    /// sha256 over all resolved secret env pairs, stamped onto the
+    /// container as a label so rotation is detectable as drift without
+    /// exposing values. None when the app has no secret env.
+    #[serde(skip)]
+    pub secret_env_hash: Option<String>,
 }
 
 /// Derived-env entry. The template is rendered against `HostFacts` at
@@ -265,6 +281,75 @@ pub struct SecretEnv {
     pub secret_file: String,
 }
 
+/// A fully resolved secret env entry, produced at apply time. `value` lives
+/// only in memory on its way to the podman secret store (or, on the dev
+/// docker fallback, straight into the container env).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SecretEnvRef {
+    pub env_key: String,
+    /// Podman secret object name: `archy-env-<app>-<KEY>`.
+    pub secret_name: String,
+    pub value: String,
+}
+
+/// Container label carrying the combined secret-env content hash, used by
+/// the reconciler to detect secret rotation as env drift.
+pub const SECRET_ENV_HASH_LABEL: &str = "io.archipelago.secret-env-hash";
+
+/// Podman secret label carrying the individual secret's content hash, used
+/// to skip re-registration when nothing changed.
+pub const SECRET_HASH_LABEL: &str = "io.archipelago.hash";
+
+/// Expand `${KEY}` placeholders across plain + secret values, then split
+/// the result into (plain env, secret-bearing pairs). Any plain entry whose
+/// value interpolates a secret key is *tainted* — it embeds the secret and
+/// must travel as a secret itself (btcpay's
+/// `BTCPAY_POSTGRES=…Password=${BTCPAY_DB_PASS};…` pattern). Secret values
+/// themselves are taken verbatim: a generated secret that happens to
+/// contain `${` must not be mangled by expansion.
+pub fn expand_and_partition_env(
+    plain: Vec<String>,
+    secrets: Vec<(String, String)>,
+) -> (Vec<String>, Vec<(String, String)>) {
+    let plain_values: std::collections::HashMap<String, String> = plain
+        .iter()
+        .filter_map(|entry| {
+            let (key, value) = entry.split_once('=')?;
+            Some((key.to_string(), value.to_string()))
+        })
+        .collect();
+
+    let mut out_plain = Vec::with_capacity(plain.len());
+    let mut out_secret: Vec<(String, String)> = Vec::with_capacity(secrets.len());
+
+    for entry in plain {
+        let Some((key, value)) = entry.split_once('=') else {
+            out_plain.push(entry);
+            continue;
+        };
+        let mut expanded = value.to_string();
+        let mut tainted = false;
+        for (k, v) in &plain_values {
+            expanded = expanded.replace(&format!("${{{k}}}"), v);
+        }
+        for (k, v) in &secrets {
+            let placeholder = format!("${{{k}}}");
+            if expanded.contains(&placeholder) {
+                expanded = expanded.replace(&placeholder, v);
+                tainted = true;
+            }
+        }
+        if tainted {
+            out_secret.push((key.to_string(), expanded));
+        } else {
+            out_plain.push(format!("{key}={expanded}"));
+        }
+    }
+
+    out_secret.extend(secrets);
+    (out_plain, out_secret)
+}
+
 /// How a [`GeneratedSecret`] is produced. Each kind is deterministic in shape
 /// (so the orchestrator knows which files to expect) but random in value.
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
@@ -1209,6 +1294,19 @@ impl ContainerConfig {
         &self,
         provider: &dyn SecretsProvider,
     ) -> Result<Vec<String>, ManifestError> {
+        Ok(self
+            .resolve_secret_env_pairs(provider)?
+            .into_iter()
+            .map(|(k, v)| format!("{k}={v}"))
+            .collect())
+    }
+
+    /// Like `resolve_secret_env` but returns (key, value) pairs — the shape
+    /// the podman-secret pipeline needs.
+    pub fn resolve_secret_env_pairs(
+        &self,
+        provider: &dyn SecretsProvider,
+    ) -> Result<Vec<(String, String)>, ManifestError> {
         let mut out = Vec::with_capacity(self.secret_env.len());
         for e in &self.secret_env {
             let v = provider.read(&e.secret_file)?;
@@ -1220,12 +1318,28 @@ impl ContainerConfig {
                     e.key, e.secret_file
                 )));
             }
-            out.push(format!("{}={}", e.key, v));
+            out.push((e.key.clone(), v));
         }
         Ok(out)
     }
 }
 
+/// Deterministic content hash over resolved secret env pairs (sorted by
+/// key), used for the container drift label and per-secret labels.
+pub fn secret_env_content_hash(pairs: &[(String, String)]) -> String {
+    use sha2::{Digest, Sha256};
+    let mut sorted: Vec<&(String, String)> = pairs.iter().collect();
+    sorted.sort_by(|a, b| a.0.cmp(&b.0));
+    let mut hasher = Sha256::new();
+    for (k, v) in sorted {
+        hasher.update(k.as_bytes());
+        hasher.update([0u8]);
+        hasher.update(v.as_bytes());
+        hasher.update([0u8]);
+    }
+    hex::encode(hasher.finalize())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1233,6 +1347,53 @@ mod tests {
     use std::fs;
     use std::path::{Path, PathBuf};
 
+    #[test]
+    fn partition_taints_plain_entries_that_interpolate_secrets() {
+        let plain = vec![
+            "PLAIN=1".to_string(),
+            "COMPOSED=${BASE}/x".to_string(),
+            "DB_URL=postgres://u:${DB_PASS}@db/x".to_string(),
+            "BASE=/srv".to_string(),
+            "UNKNOWN=${NOT_DEFINED}".to_string(),
+        ];
+        let secrets = vec![("DB_PASS".to_string(), "s3cret".to_string())];
+        let (p, s) = expand_and_partition_env(plain, secrets);
+
+        // plain-from-plain expansion stays plain; unknown placeholders stay
+        // literal (legacy expander parity)
+        assert!(p.contains(&"PLAIN=1".to_string()));
+        assert!(p.contains(&"COMPOSED=/srv/x".to_string()));
+        assert!(p.contains(&"UNKNOWN=${NOT_DEFINED}".to_string()));
+        // the tainted entry moved out of plain, fully expanded
+        assert!(!p.iter().any(|e| e.starts_with("DB_URL=")));
+        assert!(s.contains(&("DB_URL".to_string(), "postgres://u:s3cret@db/x".to_string())));
+        // the original secret rides along verbatim
+        assert!(s.contains(&("DB_PASS".to_string(), "s3cret".to_string())));
+    }
+
+    #[test]
+    fn secret_values_are_never_expanded() {
+        // A generated secret containing `${` must pass through untouched.
+        let secrets = vec![("WEIRD".to_string(), "pa${PLAIN}ss".to_string())];
+        let (_, s) = expand_and_partition_env(vec!["PLAIN=1".to_string()], secrets);
+        assert!(s.contains(&("WEIRD".to_string(), "pa${PLAIN}ss".to_string())));
+    }
+
+    #[test]
+    fn secret_env_hash_is_order_independent() {
+        let a = vec![
+            ("K1".to_string(), "v1".to_string()),
+            ("K2".to_string(), "v2".to_string()),
+        ];
+        let b = vec![
+            ("K2".to_string(), "v2".to_string()),
+            ("K1".to_string(), "v1".to_string()),
+        ];
+        assert_eq!(secret_env_content_hash(&a), secret_env_content_hash(&b));
+        let c = vec![("K1".to_string(), "CHANGED".to_string())];
+        assert_ne!(secret_env_content_hash(&a), secret_env_content_hash(&c));
+    }
+
     #[test]
     fn hooks_parse_and_validate() {
         let yaml = r#"
@@ -1765,6 +1926,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let facts = HostFacts {
             host_ip: "192.168.1.116".to_string(),
@@ -1817,6 +1980,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let p = MapSecretsProvider {
             data: HashMap::from([
@@ -1855,6 +2020,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let p = MapSecretsProvider {
             data: HashMap::from([("bitcoin-rpc-password".to_string(), "   \n".to_string())]),

core/container/src/podman_client.rs

@@ -252,7 +252,17 @@ impl PodmanClient {
 
     // ─── Container Operations ────────────────────────────────────
 
-    pub async fn pull_image(&self, image: &str, _signature: Option<&str>) -> Result<()> {
+    pub async fn pull_image(&self, image: &str, signature: Option<&str>) -> Result<()> {
+        // A declared (non-placeholder) signature must verify before we fetch
+        // anything; placeholder/absent claims pull unverified until the
+        // signing ceremony ships real signatures (see image_verify).
+        crate::image_verify::enforce_signature_claim(
+            image,
+            signature,
+            image_uses_insecure_registry(image),
+        )
+        .await?;
+
         // Image pull uses CLI — it's a streaming operation that the API handles differently
         let mut cmd = tokio::process::Command::new("podman");
         cmd.arg("pull");
@@ -372,12 +382,32 @@ impl PodmanClient {
             manifest.app.security.network_policy.as_str(),
         );
 
+        // Secret env travels by reference (podman injects the value at
+        // start), so it never shows up in `podman inspect` output. The
+        // combined content hash rides as a label for rotation-drift checks.
+        let mut secret_env_map = serde_json::Map::new();
+        for r in &manifest.app.container.secret_env_refs {
+            secret_env_map.insert(
+                r.env_key.clone(),
+                serde_json::Value::String(r.secret_name.clone()),
+            );
+        }
+        let mut labels_map = serde_json::Map::new();
+        if let Some(hash) = &manifest.app.container.secret_env_hash {
+            labels_map.insert(
+                crate::manifest::SECRET_ENV_HASH_LABEL.to_string(),
+                serde_json::Value::String(hash.clone()),
+            );
+        }
+
         let mut body = serde_json::json!({
             "name": name,
             "image": image_ref,
             "portmappings": port_mappings,
             "mounts": mounts,
             "env": env_map,
+            "secret_env": secret_env_map,
+            "labels": labels_map,
             "entrypoint": manifest.app.container.entrypoint.clone(),
             "command": manifest.app.container.custom_args.clone(),
             "hostadd": [

core/container/src/runtime.rs

@@ -2,7 +2,6 @@ use crate::manifest::{AppManifest, BuildConfig};
 use crate::podman_client::{ContainerState, ContainerStatus, PodmanClient};
 use anyhow::{Context, Result};
 use async_trait::async_trait;
-use std::process::Command;
 use std::time::Duration;
 use tokio::process::Command as TokioCommand;
 
@@ -83,6 +82,18 @@ pub trait ContainerRuntime: Send + Sync {
     /// `create_container` / `image_exists` calls. Stdout/stderr are collected
     /// and included in the error on failure; on success they are discarded.
     async fn build_image(&self, config: &BuildConfig) -> Result<()>;
+
+    /// Register the app's resolved secret-env entries in the runtime's
+    /// secret store (idempotent — skips entries whose content hash already
+    /// matches). Called before `create_container` for manifests with
+    /// `secret_env_refs`, so the create can reference secrets by name and
+    /// the values never appear in inspect output or unit files. Default is
+    /// a no-op for runtimes without a secret store (mocks, docker — the
+    /// docker fallback injects plain env in `create_container` instead).
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        let _ = refs;
+        Ok(())
+    }
 }
 
 pub struct PodmanRuntime {
@@ -132,6 +143,68 @@ impl ContainerRuntime for PodmanRuntime {
         self.client.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        use crate::manifest::{secret_env_content_hash, SECRET_HASH_LABEL};
+        use tokio::io::AsyncWriteExt;
+
+        for r in refs {
+            let hash = secret_env_content_hash(&[(r.env_key.clone(), r.value.clone())]);
+
+            // Skip the write when the stored secret already has this content
+            // (label round-trip beats rewriting the secret store every
+            // reconcile). Any inspect failure just falls through to create.
+            let fmt = format!("{{{{ index .Spec.Labels \"{SECRET_HASH_LABEL}\" }}}}");
+            if let Ok(out) = self
+                .podman_cli(&["secret", "inspect", &r.secret_name, "--format", &fmt])
+                .await
+            {
+                if out.status.success()
+                    && String::from_utf8_lossy(&out.stdout).trim() == hash
+                {
+                    continue;
+                }
+            }
+
+            // Value goes via stdin — never argv, never a temp file.
+            let mut cmd = TokioCommand::new("podman");
+            cmd.args([
+                "secret",
+                "create",
+                "--replace",
+                "--label",
+                &format!("{SECRET_HASH_LABEL}={hash}"),
+                &r.secret_name,
+                "-",
+            ]);
+            cmd.stdin(std::process::Stdio::piped());
+            cmd.stdout(std::process::Stdio::piped());
+            cmd.stderr(std::process::Stdio::piped());
+            cmd.kill_on_drop(true);
+            let mut child = cmd
+                .spawn()
+                .with_context(|| format!("spawning podman secret create {}", r.secret_name))?;
+            {
+                let mut stdin = child
+                    .stdin
+                    .take()
+                    .context("podman secret create stdin unavailable")?;
+                stdin.write_all(r.value.as_bytes()).await?;
+                // drop closes the pipe so podman sees EOF
+            }
+            let out = tokio::time::timeout(PODMAN_CLI_DEFAULT_TIMEOUT, child.wait_with_output())
+                .await
+                .with_context(|| format!("podman secret create {} timed out", r.secret_name))??;
+            if !out.status.success() {
+                anyhow::bail!(
+                    "podman secret create {} failed: {}",
+                    r.secret_name,
+                    String::from_utf8_lossy(&out.stderr)
+                );
+            }
+        }
+        Ok(())
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,
@@ -547,7 +620,12 @@ impl DockerRuntime {
 
 #[async_trait]
 impl ContainerRuntime for DockerRuntime {
-    async fn pull_image(&self, image: &str, _signature: Option<&str>) -> Result<()> {
+    async fn pull_image(&self, image: &str, signature: Option<&str>) -> Result<()> {
+        // Same signature gate as the podman path — the docker fallback is
+        // dev-only, but a declared signature must never be skippable by
+        // switching runtimes.
+        crate::image_verify::enforce_signature_claim(image, signature, false).await?;
+
         let mut cmd = self.docker_async();
         cmd.arg("pull").arg(image);
 
@@ -617,6 +695,12 @@ impl ContainerRuntime for DockerRuntime {
         for env in &manifest.app.environment {
             cmd.arg("-e").arg(env);
         }
+        // Dev-only fallback: docker has no rootless secret store, so secret
+        // env rides as plain env here. The podman path (production) passes
+        // these by secret reference instead — see ensure_env_secrets.
+        for r in &manifest.app.container.secret_env_refs {
+            cmd.arg("-e").arg(format!("{}={}", r.env_key, r.value));
+        }
 
         // Resource limits
         if let Some(cpu) = manifest.app.resources.cpu_limit {
@@ -853,11 +937,11 @@ pub struct AutoRuntime {
 impl AutoRuntime {
     pub async fn new(user: String) -> Result<Self> {
         // Try Podman first
-        if Self::check_podman_available() {
+        if Self::check_podman_available().await {
             Ok(Self {
                 runtime: Box::new(PodmanRuntime::new(user)),
             })
-        } else if Self::check_docker_available() {
+        } else if Self::check_docker_available().await {
             Ok(Self {
                 runtime: Box::new(DockerRuntime::new(user)),
             })
@@ -866,12 +950,20 @@ impl AutoRuntime {
         }
     }
 
-    fn check_podman_available() -> bool {
-        Command::new("podman").arg("--version").output().is_ok()
+    async fn check_podman_available() -> bool {
+        TokioCommand::new("podman")
+            .arg("--version")
+            .output()
+            .await
+            .is_ok()
     }
 
-    fn check_docker_available() -> bool {
-        Command::new("docker").arg("--version").output().is_ok()
+    async fn check_docker_available() -> bool {
+        TokioCommand::new("docker")
+            .arg("--version")
+            .output()
+            .await
+            .is_ok()
     }
 }
 
@@ -881,6 +973,10 @@ impl ContainerRuntime for AutoRuntime {
         self.runtime.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        self.runtime.ensure_env_secrets(refs).await
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,

core/security/src/image_verifier.rs

@@ -1,111 +0,0 @@
-// Container image signature verification using Cosign
-// Verifies that container images are signed and trusted
-
-use anyhow::{Context, Result};
-use std::process::Command;
-use tracing::{info, warn};
-
-pub struct ImageVerifier {
-    cosign_public_key: Option<String>,
-    require_signatures: bool,
-}
-
-impl ImageVerifier {
-    pub fn new(cosign_public_key: Option<String>) -> Self {
-        Self {
-            cosign_public_key,
-            require_signatures: false,
-        }
-    }
-
-    /// Create a verifier that requires all images to be signed.
-    pub fn new_strict(cosign_public_key: Option<String>) -> Self {
-        Self {
-            cosign_public_key,
-            require_signatures: true,
-        }
-    }
-
-    /// Verify a container image signature
-    pub async fn verify_image(&self, image: &str, signature: Option<&str>) -> Result<bool> {
-        if signature.is_none() && self.cosign_public_key.is_none() {
-            if self.require_signatures {
-                return Err(anyhow::anyhow!(
-                    "Image '{}' has no signature and no cosign key is configured. \
-                     All container images must be signed for production use.",
-                    image
-                ));
-            }
-            warn!("No signature provided for image: {}", image);
-            return Ok(false);
-        }
-
-        // Check if cosign is available
-        let cosign_available = Command::new("cosign").arg("version").output().is_ok();
-
-        if !cosign_available {
-            if self.require_signatures {
-                return Err(anyhow::anyhow!(
-                    "Cosign binary not found. Install cosign to verify container image signatures."
-                ));
-            }
-            warn!("Cosign not available, skipping signature verification");
-            return Ok(false);
-        }
-
-        // If public key is provided, use it for verification
-        if let Some(ref public_key) = self.cosign_public_key {
-            let output = Command::new("cosign")
-                .arg("verify")
-                .arg("--key")
-                .arg(public_key)
-                .arg(image)
-                .output()
-                .context("Failed to run cosign verify")?;
-
-            if output.status.success() {
-                info!("Image signature verified: {}", image);
-                return Ok(true);
-            } else {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                return Err(anyhow::anyhow!("Signature verification failed: {}", stderr));
-            }
-        }
-
-        // If signature URL is provided, verify using that
-        if let Some(sig_url) = signature {
-            if sig_url.starts_with("cosign://") {
-                // Extract signature reference
-                let sig_ref = sig_url.strip_prefix("cosign://").unwrap();
-                let output = Command::new("cosign")
-                    .arg("verify")
-                    .arg("--signature")
-                    .arg(sig_ref)
-                    .arg(image)
-                    .output()
-                    .context("Failed to run cosign verify")?;
-
-                if output.status.success() {
-                    info!("Image signature verified: {}", image);
-                    return Ok(true);
-                } else {
-                    let stderr = String::from_utf8_lossy(&output.stderr);
-                    return Err(anyhow::anyhow!("Signature verification failed: {}", stderr));
-                }
-            }
-        }
-
-        Ok(false)
-    }
-
-    /// Check if an image has a signature
-    pub async fn has_signature(&self, image: &str) -> bool {
-        // Try to find signature in registry
-        let output = Command::new("cosign")
-            .arg("triangulate")
-            .arg(image)
-            .output();
-
-        output.is_ok() && output.unwrap().status.success()
-    }
-}

core/security/src/lib.rs

@@ -1,7 +1,5 @@
 pub mod container_policies;
-pub mod image_verifier;
 pub mod secrets_manager;
 
 pub use container_policies::ContainerPolicyGenerator;
-pub use image_verifier::ImageVerifier;
 pub use secrets_manager::SecretsManager;

docs/1.8.0-RELEASE-HARDENING-PLAN.md

@@ -58,21 +58,32 @@ arbitrary app catalog to the entire fleet — fully unattended under
       `scripts/sign-manifest.sh` exists for re-signs. **Still open:** move the mirror
       to HTTPS + pinned cert (tracked with the next item); flip unsigned-manual-apply →
       hard-reject once the fleet is on a pinned-anchor binary.
-- [ ] 🔴 **Implement container image signature verification (cosign).**
-      `container/src/podman_client.rs:255` — `pull_image(.., _signature)` silently discards
-      the signature that the manifest threads all the way down
-      (`prod_orchestrator.rs:1978/2435`). Wire `sigstore-rs`/`cosign verify` (or
-      `podman pull --signature-policy`); hard-fail when a declared signature doesn't verify.
+- [x] 🔴 **Implement container image signature verification (cosign).** DONE 2026-07-04
+      (code path; enforcement dormant until the ceremony): new `container::image_verify`
+      gates BOTH pull sites (`PodmanClient::pull_image` + the dev-only `DockerRuntime`).
+      Claims classify as None / the literal `cosign://...` placeholder (every fleet
+      manifest today → pull proceeds, logged) / Declared → `cosign verify --key
+      /etc/archipelago/cosign.pub --insecure-ignore-tlog=true` (+ both insecure-registry
+      flags for the HTTP mirror; flags verified against cosign docs), hard-fail on missing
+      key, missing cosign binary, timeout, or bad signature — a declared signature can
+      never be skipped, on either runtime. Key path overridable via
+      `ARCHIPELAGO_COSIGN_PUBKEY`. Deleted the caller-less, blocking, wrong-CLI
+      `security::ImageVerifier`. **Activation = ceremony work**: pin cosign.pub on nodes +
+      install cosign + publish real `image_signature` values (in that order); tracked with
+      the Workstream B signing ceremony item.
 - [ ] 🟠 **Move the image mirror to HTTPS; drop `--tls-verify=false`.**
       `podman_client.rs:641` `INSECURE_REGISTRY_HOSTS = ["146.59.87.168:3000"]` +
       `config.rs:104,124` allowlist pull images over unauthenticated HTTP. Remove the raw-IP
       entries; give the mirror a valid/pinned cert. (Same host also baked insecurely into
       the ISO — see §F.)
-- [ ] 🟠 **Validate every image string at the pull site, not just the RPC boundary.**
-      `is_valid_docker_image` runs in `install.rs:224`/`runtime.rs:549` but
-      `prod_orchestrator::install_fresh` (1978) and `resolve_catalog_image` (944-971) pass
-      catalog/manifest images straight to `pull_image`. Call the validator right before
-      every pull.
+- [x] 🟠 **Validate every image string at the pull site, not just the RPC boundary.**
+      DONE 2026-07-03: policy extracted to `container::image_policy` (single source of truth;
+      RPC-boundary check delegates to it) and BOTH orchestrator pull sites (`install_fresh` +
+      `ensure_resolved_source_available`) hard-bail on refs that fail it. Policy accepts
+      trusted-registry refs + registry-less Docker Hub shorthand (`grafana/grafana` — used by
+      8 manifests, can't name an attacker host); rejects any explicit non-allowlisted
+      registry host, shell metachars, malformed refs. 4 new unit tests; container 159 /
+      package 46 green.
 
 ---
 
@@ -83,11 +94,15 @@ atomic swap, single-depth backup). The gaps are **authenticity** (§A) and
 **verification depth** — plus the fact that the upgrade path has never run
 end-to-end on real hardware.
 
-- [ ] 🔴 **Deepen the post-OTA health check.** `update.rs:456` (`probe_frontend_once`)
-      passes on any 2xx/3xx from `GET /`, and `verify_pending_update` (494-593) only rolls
-      back on that. A release with a broken RPC API, dead containers, or failed LND unlock
-      passes and never rolls back. Add `/rpc/v1 update.status` + container-list/required-stack
-      health assertions before clearing the pending-verify marker.
+- [x] 🔴 **Deepen the post-OTA health check.** DONE 2026-07-03: `verify_pending_update`
+      now requires, in the same attempt, (1) frontend 2xx/3xx via nginx, (2) backend RPC
+      liveness — unauthenticated POST `/rpc/v1`; 401/403 = alive, 5xx/404/refused = dead,
+      so a 502-behind-static-files release now rolls back, (3) rootless `podman ps`
+      reachability; plus a pre-loop binary-version==marker assertion that catches a silent
+      or half swap (new frontend + old binary) deterministically. Per-app container
+      assertions deliberately EXCLUDED — the pre-Quadlet service restart legitimately kills
+      containers and the reconciler can need minutes (false-rollback risk); revisit after
+      the Phase-3 flip. LND-unlock-level checks remain out of scope for the 90s window.
 - [ ] 🟠 **Run one real upgrade-from-vN-1 soak on hardware before tagging.**
       No test installs the previous version, points it at a staged 1.8.0 manifest, applies,
       and asserts health + rollback. This is the top release risk for an OTA release. A
@@ -114,25 +129,60 @@ modules; production request/boot paths are essentially panic-free. The real risk
       sweep (`scheduler.rs`), block-header cache (`mesh/mod.rs`), 7× peer-transport badge
       (`sync.rs` + `content.rs`). Federation tombstone/untombstone upgraded to hard errors
       (see §I). Install-log line write left fire-and-forget with an explanatory comment.
-- [ ] 🟠 **Remove blocking `std::process::Command` from async handlers.**
-      `install.rs:2222` `published_host_port` (sync podman on the install path),
-      `dependencies.rs:316` (`df`), `system/handlers.rs:578` (`sudo`), `transport/fips.rs:50`
-      (`systemctl`) stall tokio workers under load. Convert to `tokio::process` or
-      `spawn_blocking`. Only 8 files use `std::process::Command` — bounded.
-- [ ] 🟡 **Restrict Bitcoin RPC exposure.** `bootstrap.rs:409` writes
-      `rpcallowip=0.0.0.0/0`. Scope to the container subnet / `127.0.0.1`.
-- [ ] 🟡 **Move generated secrets from env to file mounts.** `manifest.rs:1208-1226`
-      injects secrets as `-e KEY=value`, readable via `podman inspect` / `/proc/<pid>/environ`.
-      Prefer bind-mounting the existing `0600` secret file or `podman --secret`.
-- [ ] 🟡 **Harden rate-limit IP extraction.** `middleware.rs:120-128` trusts
-      client-spoofable `X-Real-IP`/`X-Forwarded-For` → per-request bucket rotation defeats the
-      login limiter. Trust forwarded headers only from a configured proxy; have nginx set them.
-- [ ] 🟢 **Include `seq` in the mesh signed preimage.** `message_types.rs:245-288` signs
-      `(t,v,ts)` but sets the anti-replay `seq` after signing → a radio MITM can alter ordering
-      without breaking the signature.
-- [ ] 🟢 **Guard the short-DID slice panic** (`mesh/listener/decode.rs:566`) and gate the
-      dev-mode `password123` bypass (`auth.rs:18`) behind `#[cfg]` before it can reach a
-      release build.
+- [x] 🟠 **Remove blocking `std::process::Command` from async handlers.** DONE 2026-07-03:
+      converted to `tokio::process` — `published_host_port` (install), `detect_disk_gb`
+      (dependencies), factory-reset restart (system/handlers), `config.rs detect_host_ip`,
+      the orchestrator host-facts helpers (`detect_host_ip/mdns/disk_gb`, `bitcoin_host`,
+      `resolve_dynamic_env` now async through all 6 call sites), and `AutoRuntime::new`
+      probes. `transport/fips.rs is_available()` (sync trait method on the async route path)
+      now serves the cached value and refreshes via a background thread (stale-while-
+      revalidate) instead of blocking on systemctl. `image_verifier.rs` cosign sites have no
+      callers yet — handled with the §A cosign item. Tests: container 155 / transport 29 /
+      config 29 / package 46 all green.
+- [ ] 🟡 **Restrict Bitcoin RPC exposure.** INVESTIGATED 2026-07-03 — the fix is NOT
+      `rpcallowip`: under rootless-podman NAT every forwarded connection reaches bitcoind
+      with an in-subnet source IP, so scoping `rpcallowip` blocks nothing (and container
+      consumers use archy-net DNS anyway). The actual exposure is the host-side publish
+      `8332:8332` (binds 0.0.0.0 → LAN can hit RPC, auth-only barrier; 4 write sites:
+      `bootstrap.rs:424`, `package/config.rs:694`, `package/install.rs:1333/2450`, plus
+      the knots manifest). Real fix = `127.0.0.1:8332:8332` host bind (P2P 8333 stays
+      public; zmq 28332/28333 should get the same look — unauthenticated). ⚠ May break
+      external wallets pointed straight at nodeIP:8332 — needs a user call + on-node gate
+      re-run, so NOT changed drive-by from the dev box.
+- [x] 🟡 **Move secret env out of plaintext channels → podman secrets.** DONE 2026-07-05
+      (code + unit tests; needs the on-node gate re-run before it counts as verified):
+      secret env no longer merges into `environment` — it would land in `podman inspect`
+      AND as plaintext `Environment=` lines in Quadlet unit files on disk (the worse leak).
+      New pipeline: `expand_and_partition_env` taints plain entries that interpolate
+      secrets (btcpay's `Password=${BTCPAY_DB_PASS}` connection strings travel as secrets
+      too), values register as podman secrets (stdin, `--replace`, content-hash label,
+      per-app cache so steady-state reconciles are podman-free), containers reference
+      them via `secret_env` (API) / `Secret=…,type=env` (Quadlet). Verified empirically
+      on fleet podman 5.4.2: value absent from inspect, runtime injection works. Rotation
+      drift via `io.archipelago.secret-env-hash` container label; pre-upgrade containers
+      lack the label → ONE-TIME recreate wave on first reconcile after deploy (by design —
+      scrubs plaintext secrets from existing container configs). Docker dev fallback keeps
+      plain env (no secret store). `/proc/<pid>/environ` inside the container is unchanged
+      (env is the app-compat contract); the closed leaks are inspect output + unit files.
+- [x] 🟡 **Harden rate-limit IP extraction.** DONE 2026-07-03: the accept loop injects the
+      TCP `PeerAddr` into request extensions; `extract_client_ip` honors
+      `X-Real-IP`/`X-Forwarded-For` ONLY when the connection is from loopback (our nginx,
+      which sets `X-Real-IP $remote_addr`) — direct connections (e.g. the FIPS peer
+      listener) bucket under their socket IP, so per-request header rotation no longer
+      defeats the login limiter. 3 unit tests.
+- [x] 🟢 **Include `seq` in the mesh signed preimage.** DONE 2026-07-04 (receiver half):
+      `verify_signature` accepts a v2 preimage `(t,v,ts,seq)` alongside legacy v1 `(t,v,ts)`;
+      `signed_with_seq()` is the v2 sender path, deliberately NOT yet wired — receivers
+      hard-drop bad signatures, so senders stay on v1 until the whole fleet verifies v2.
+      The seq-tampering window closes only when the v1 arm is removed (track as a
+      post-fleet-rollout follow-up). Unit tests cover v2 verify, v2 seq-tamper rejection,
+      and v1 sign-then-set-seq compatibility.
+- [x] 🟢 **Guard the short-DID slice panic** (`mesh/listener/decode.rs:566`) and gate the
+      dev-mode `password123` bypass (`auth.rs:18`) behind `#[cfg]`. DONE 2026-07-04:
+      advert_name uses `.get()` fallback (malformed radio-supplied DID can't panic the
+      listener); the pre-setup dev-password login + the constant itself are
+      `#[cfg(debug_assertions)]` — no release binary carries the bypass regardless of
+      runtime config.
 - [ ] 🟢 **Apply the seccomp/apparmor profile** — `security/src/container_policies.rs:71` is a
       TODO; the profile is defined but never applied to podman.
 
@@ -159,9 +209,11 @@ The real issues are the app-bridge origin model and a bloated bundle.
       (precached by the service worker → blocks PWA install), plus ~18 MB of ~1 MB full-screen
       JPEGs. Convert backgrounds to WebP/AVIF at responsive sizes, lazy/stream the intro video,
       and exclude video/audio from the Workbox precache. Biggest, easiest perf win.
-- [ ] 🟢 **DOMPurify the `Server.vue` QR SVG** (`:283/:295` render `v-html` unsanitized while
-      `TwoFactorSection.vue` sanitizes the analogous SVG); guard the unguarded `pollInterval`
-      (`Mesh.vue:391`); surface silent data-fetch failures (`curatedApps.ts:58/71`).
+- [x] 🟢 **DOMPurify the `Server.vue` QR SVG / guard `Mesh.vue` pollInterval / surface
+      `curatedApps.ts` fetch failures.** DONE 2026-07-03: WireGuard peer QR now sanitized with
+      the same `USE_PROFILES: {svg}` call as TwoFactorSection; Mesh poll interval guarded +
+      nulled on unmount; catalog fetch failures log per-URL console.warn incl. the
+      all-sources-failed fallback. Bundle-verified.
 
 ---
 

docs/UNIFIED-TASK-TRACKER.md

@@ -113,6 +113,32 @@ those are marked ✅ below with the commit that did it, so we stop re-litigating
       manifest-driven apps, never stacks; fedimint/fedimint-gateway/fedimint-clientd
       are 3 separate single-container apps with manifest dependency edges, not a
       coordinated stack. Workstream A's stack-migration tail is fully closed.
+- [ ] **Container thrashing/flapping + reconciler churn** (added 2026-07-04 — was
+      implicit across other tracks, now an explicit pre-tag concern). The root cause
+      of restart-storm flapping is pre-Quadlet architecture: restarting
+      `archipelago.service` SIGKILLs every container in its cgroup, then the
+      reconciler rebuilds the world over several minutes (the post-OTA health check
+      deliberately skips per-app container assertions because of exactly this).
+      Consolidated lever list, in order of impact:
+  - **Phase-3 Quadlet default-flip** (tracked above) — removes the SIGKILL-the-world
+    behavior entirely; the single biggest fix.
+  - **Workstream F lifecycle items** — immich/grafana uninstall hangs + ghost
+    containers, grafana reinstall stops, fedimint guardian sync
+    (`docs/PRODUCTION-MASTER-PLAN.md` workstream F).
+  - **Reconciler churn observability** — no metric/log today distinguishes "settling
+    after restart" from "flapping"; add a per-app restart counter + log line when an
+    app restarts >N times in M minutes so thrash is visible instead of anecdotal.
+  - **Failed-unit self-healing gap (observed live 2026-07-06 on .228)**: fedimint's
+    quadlet unit exited 255 at 21:21 and sat `failed` for 7+ hours — the reconciler
+    never revived it (it repairs missing/drifted containers but doesn't
+    `reset-failed`+start failed .services). Same for the indeedhub trio after the
+    gate run. The health monitor also can't help (container is gone when the unit
+    fails). Add a reconcile step: quadlet-backed app whose .service is `failed` and
+    not user-stopped → reset-failed + start, with backoff.
+  - Already landed, don't re-do: boot-reconciler circuit breaker (2026-07-01),
+    indeedhub crashloop fix (2026-07-01), async blocking-Command pass (`4c75bb3d`,
+    removes executor stalls that made the API janky under reconcile load).
+  - Perf polish riding along: 93 MB frontend dist shrink (hardening plan §D 🟡).
 - [ ] **Developer tooling CLI suite** (validate/render/local-install/lifecycle-test) —
       APP-PACKAGING-MIGRATION-PLAN.md step 5, needed before external devs can publish.
 - [x] ~~**Consolidated deploy 2026-07-01**: merged PR #67 (reticulum daemon

neode-ui/src/views/Mesh.vue

@@ -388,13 +388,15 @@ onMounted(async () => {
   if (!archPollInterval) {
     archPollInterval = setInterval(loadArchMessages, 15000)
   }
-  pollInterval = setInterval(() => {
-    mesh.fetchStatus()
-    mesh.fetchPeers()
-    mesh.fetchMessages()
-    mesh.fetchDeadmanStatus()
-    mesh.fetchBlockHeaders()
-  }, 5000)
+  if (!pollInterval) {
+    pollInterval = setInterval(() => {
+      mesh.fetchStatus()
+      mesh.fetchPeers()
+      mesh.fetchMessages()
+      mesh.fetchDeadmanStatus()
+      mesh.fetchBlockHeaders()
+    }, 5000)
+  }
 
   // Instant peer updates (#48): the backend nudges the data-model revision when
   // it discovers/updates a mesh peer, so refetch peers on the WS push rather
@@ -414,7 +416,7 @@ onUnmounted(() => {
     window.visualViewport.removeEventListener('scroll', updateKeyboardInset)
   }
   document.documentElement.style.removeProperty('--keyboard-inset')
-  if (pollInterval) clearInterval(pollInterval)
+  if (pollInterval) { clearInterval(pollInterval); pollInterval = null }
   if (archPollInterval) { clearInterval(archPollInterval); archPollInterval = null }
   if (wsUnsub) { wsUnsub(); wsUnsub = null }
 })

neode-ui/src/views/Server.vue

@@ -306,7 +306,7 @@
             </div>
             <!-- Existing peer QR view -->
             <div v-else-if="peerQrData && !showingNewDevice" class="text-center">
-              <div class="bg-white rounded-xl p-4 mb-4 inline-block" v-html="peerQrData.qr_svg"></div>
+              <div class="bg-white rounded-xl p-4 mb-4 inline-block" v-html="sanitizedPeerQrSvg"></div>
               <p class="text-sm text-white/70 mb-2">Scan with the <strong>WireGuard</strong> app</p>
               <p class="text-xs text-white/40 font-mono mb-4">{{ peerQrData.peer_ip }}</p>
               <div class="flex gap-2">
@@ -318,7 +318,7 @@
             <div v-else>
               <div v-if="peerQrData">
                 <div class="text-center">
-                  <div class="bg-white rounded-xl p-4 mb-4 inline-block" v-html="peerQrData.qr_svg"></div>
+                  <div class="bg-white rounded-xl p-4 mb-4 inline-block" v-html="sanitizedPeerQrSvg"></div>
                   <p class="text-sm text-white/70 mb-2">Scan with the <strong>WireGuard</strong> app</p>
                   <p class="text-xs text-white/40 font-mono mb-4">{{ peerQrData.peer_ip }}</p>
                   <div class="flex gap-2">
@@ -406,6 +406,7 @@
 
 <script setup lang="ts">
 import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
+import DOMPurify from 'dompurify'
 import { rpcClient } from '@/api/rpc-client'
 import { useAppStore } from '@/stores/app'
 import QuickActionsCard from './server/QuickActionsCard.vue'
@@ -499,6 +500,11 @@ const showAddDeviceModal = ref(false)
 const newPeerName = ref('')
 const creatingPeer = ref(false)
 const peerQrData = ref<{ qr_svg: string; config: string; peer_ip: string } | null>(null)
+// Sanitize like TwoFactorSection's TOTP QR — the SVG is backend-generated,
+// but v-html without a sanitizer is one compromised RPC away from XSS.
+const sanitizedPeerQrSvg = computed(() =>
+  DOMPurify.sanitize(peerQrData.value?.qr_svg ?? '', { USE_PROFILES: { svg: true } }),
+)
 const peerError = ref('')
 const copiedConfig = ref(false)
 const vpnPeers = ref<{ name: string; ip: string; type?: string; npub?: string }[]>([])

neode-ui/src/views/discover/curatedApps.ts

@@ -57,7 +57,10 @@ export async function fetchAppCatalog(): Promise<AppCatalog | null> {
       // Cache in localStorage for offline fallback
       try { localStorage.setItem('archy_catalog', JSON.stringify(data)) } catch {}
       return data
-    } catch { continue }
+    } catch (e) {
+      console.warn(`[catalog] fetch failed for ${url}:`, e)
+      continue
+    }
   }
 
   // Try localStorage cache as final fallback
@@ -68,8 +71,11 @@ export async function fetchAppCatalog(): Promise<AppCatalog | null> {
       catalogFetchedAt = Date.now() - CATALOG_TTL + 5 * 60 * 1000 // re-check in 5 min
       return cachedCatalog
     }
-  } catch {}
+  } catch (e) {
+    console.warn('[catalog] localStorage fallback unreadable:', e)
+  }
 
+  console.warn('[catalog] all sources failed — using hardcoded app list')
   return null
 }