Archipelago — open-source initial import

core/archipelago/src/container/app_catalog.rs

@@ -0,0 +1,644 @@
+//! Remote app version catalog — DECOUPLES per-app updates from the binary OTA.
+//!
+//! Background: `image_versions.rs` reads the pinned image tags from
+//! `image-versions.sh`, which is deployed *with the archipelago binary*. That
+//! coupled every app update to a full node release. This module adds a remote
+//! catalog (`app-catalog.json`) fetched over HTTP from the same origin as the
+//! OTA manifest, refreshed periodically and on demand. Bumping an app's version
+//! is then a JSON edit + push — no binary release.
+//!
+//! Resolution order (origin-always-wins, matching the DHT design's posture):
+//!   1. Remote catalog (this module) — the live source of "available update".
+//!   2. `image-versions.sh` pin — offline/baseline fallback when the catalog is
+//!      missing or doesn't cover the app.
+//!
+//! ## Forward-compatibility with the DHT distribution plan
+//! (`docs/dht-distribution-design.md`)
+//! This catalog IS the "discovery / authenticity" layer of that plan. The schema
+//! is deliberately extensible so the later phases bolt on WITHOUT a breaking
+//! change:
+//!   - `signature` / `signed_by` (top level) — Phase 0 seed-derived release-root
+//!     signature over the canonical JSON. Absent today; verified when present.
+//!   - per-image `digest` / `size` — BLAKE3/SHA-256 content address + length, so
+//!     the iroh swarm can fetch images by hash with the registry as origin.
+//! Unknown fields are ignored (no `deny_unknown_fields`), so adding fields on the
+//! publisher side never breaks older nodes.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+use std::time::SystemTime;
+use tracing::{debug, info, warn};
+
+/// Filename for both the published catalog and the on-node cache.
+pub const APP_CATALOG_FILE: &str = "app-catalog.json";
+
+/// Cache of the parsed catalog, invalidated when the cache file mtime changes.
+static CACHE: Mutex<Option<CacheEntry>> = Mutex::new(None);
+
+struct CacheEntry {
+    mtime: SystemTime,
+    catalog: AppCatalog,
+}
+
+/// Top-level catalog document.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct AppCatalog {
+    /// Schema version. 1 = current. Bump only on incompatible changes.
+    #[serde(default)]
+    pub schema: u32,
+    /// Publish date (RFC 3339 or YYYY-MM-DD). Informational.
+    #[serde(default)]
+    pub updated: String,
+    /// app_id -> entry.
+    #[serde(default)]
+    pub apps: HashMap<String, AppCatalogEntry>,
+    /// DHT-plan forward-compat: detached signature over the canonical JSON,
+    /// produced by the seed-derived release-root key. Absent today.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub signature: Option<String>,
+    /// DHT-plan forward-compat: publisher identity (did:key / npub).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub signed_by: Option<String>,
+}
+
+/// Per-app catalog entry.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct AppCatalogEntry {
+    /// User-facing version string (drives the "Update available" badge text).
+    pub version: String,
+    /// Primary single-container image reference (`registry/repo:tag`). For stack
+    /// apps this is the primary container's image (the one whose version the
+    /// badge tracks — e.g. the IndeeHub frontend).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+    /// Stack apps only: container_name -> image reference. Components omitted here
+    /// fall back to the `image-versions.sh` pin during an update.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub images: Option<HashMap<String, String>>,
+    /// DHT-plan forward-compat: content address of the primary image (unused now).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub digest: Option<String>,
+    /// DHT-plan forward-compat: size in bytes of the primary image (unused now).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub size: Option<u64>,
+    /// Optional human-readable changelog lines for this version.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub changelog: Vec<String>,
+    /// Multi-version support (`docs/bitcoin-multi-version-design.md`): the bounded
+    /// set of versions a user may install or switch to for this app. Empty for
+    /// single-version apps; `version`/`image` above remain the default/latest for
+    /// back-compat. Old nodes ignore this field (no `deny_unknown_fields`).
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub versions: Vec<CatalogVersion>,
+    /// Full app manifest, embedded so the app installs from the registry alone —
+    /// no OTA-shipped `apps/<id>/manifest.yml`. Carried as the raw value the
+    /// publisher signed (so it stays part of the verified preimage) and
+    /// deserialized into an `AppManifest` by the orchestrator at load time, where
+    /// it overrides the disk manifest (origin-wins). Absent during the migration
+    /// window => the node falls back to the disk manifest. See
+    /// `docs/registry-manifest-design.md`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub manifest: Option<serde_json::Value>,
+}
+
+/// One selectable version in an app's `versions[]` list. The catalog carries a
+/// curated, bounded set (current + a few majors back); see
+/// `docs/bitcoin-multi-version-design.md` §3 Phase 1.
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct CatalogVersion {
+    /// User-facing + tag-matching version string (e.g. `31.0`,
+    /// `29.3.knots20260508`). Treated as the image tag.
+    pub version: String,
+    /// Concrete image reference for this version. When omitted the orchestrator
+    /// falls back to composing `<default-repo>:<version>` from the entry image.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+    /// Marks the default / latest version pre-selected in the install modal.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub default: bool,
+    /// Deprecated versions are still installable but badged in the UI.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub deprecated: bool,
+    /// Optional end-of-life date (YYYY-MM-DD), surfaced in the UI.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub eol: Option<String>,
+}
+
+/// Read-side cache file search order. Mirrors `image_versions.rs`: the running
+/// daemon's data dir first (via env for dev), then the canonical runtime path.
+fn cache_paths() -> Vec<PathBuf> {
+    let mut paths = Vec::new();
+    if let Ok(dir) = std::env::var("ARCHIPELAGO_DATA_DIR") {
+        paths.push(Path::new(&dir).join(APP_CATALOG_FILE));
+    }
+    paths.push(Path::new("/var/lib/archipelago").join(APP_CATALOG_FILE));
+    paths
+}
+
+fn find_cache_file() -> Option<(PathBuf, SystemTime)> {
+    for p in cache_paths() {
+        if let Ok(meta) = p.metadata() {
+            if let Ok(mtime) = meta.modified() {
+                return Some((p, mtime));
+            }
+        }
+    }
+    None
+}
+
+/// Load and cache the on-node catalog. Returns an empty catalog when absent —
+/// callers then fall back to `image-versions.sh`.
+fn load_catalog() -> AppCatalog {
+    let (path, mtime) = match find_cache_file() {
+        Some(v) => v,
+        None => return AppCatalog::default(),
+    };
+
+    {
+        let cache = CACHE.lock().unwrap();
+        if let Some(ref entry) = *cache {
+            if entry.mtime == mtime {
+                return entry.catalog.clone();
+            }
+        }
+    }
+
+    let content = match std::fs::read_to_string(&path) {
+        Ok(c) => c,
+        Err(e) => {
+            debug!("app-catalog: failed to read {}: {}", path.display(), e);
+            return AppCatalog::default();
+        }
+    };
+    let catalog: AppCatalog = match serde_json::from_str(&content) {
+        Ok(c) => c,
+        Err(e) => {
+            warn!("app-catalog: invalid JSON at {}: {}", path.display(), e);
+            return AppCatalog::default();
+        }
+    };
+
+    {
+        let mut cache = CACHE.lock().unwrap();
+        *cache = Some(CacheEntry {
+            mtime,
+            catalog: catalog.clone(),
+        });
+    }
+    catalog
+}
+
+fn entry_for(app_id: &str) -> Option<AppCatalogEntry> {
+    load_catalog().apps.get(app_id).cloned()
+}
+
+/// Primary image for an app per the remote catalog, if covered.
+pub fn catalog_primary_image(app_id: &str) -> Option<String> {
+    entry_for(app_id).and_then(|e| e.image)
+}
+
+/// Per-container stack image overrides from the catalog (container_name -> image).
+pub fn catalog_stack_images(app_id: &str) -> HashMap<String, String> {
+    entry_for(app_id).and_then(|e| e.images).unwrap_or_default()
+}
+
+/// All `(app_id, manifest-value)` pairs the registry catalog carries. The
+/// orchestrator deserializes + validates each into an `AppManifest` and prefers
+/// it over the disk manifest (origin-wins); disk remains the migration fallback.
+/// Empty when the catalog is absent or no entry embeds a manifest.
+pub fn catalog_manifest_values() -> Vec<(String, serde_json::Value)> {
+    load_catalog()
+        .apps
+        .into_iter()
+        .filter_map(|(id, e)| e.manifest.map(|m| (id, m)))
+        .collect()
+}
+
+/// A catalog-embedded manifest as the node actually applies it: parsed,
+/// id-checked, validated, and image-only (build-source manifests defer to
+/// disk). `None` = the caller must fall back to the disk manifest.
+///
+/// Shared between the orchestrator's load overlay and the app gate's port
+/// classification so both answer "which manifest governs this app?" from the
+/// same origin. They diverged once — the orchestrator published containers
+/// from the catalog while the gate classified from stale disk manifests, and
+/// the gate externally bound a port the catalog had declared `auth: local`
+/// (nbxplorer 32838, a test node 2026-08-04).
+pub fn catalog_manifest_overlay(
+    app_id: &str,
+    value: serde_json::Value,
+) -> Option<archipelago_container::manifest::AppManifest> {
+    let m: archipelago_container::manifest::AppManifest = match serde_json::from_value(value) {
+        Ok(m) => m,
+        Err(e) => {
+            tracing::warn!(app = %app_id, error = %e,
+                "skipping unparseable catalog manifest; using disk fallback");
+            return None;
+        }
+    };
+    if m.app.id != app_id {
+        tracing::warn!(catalog_id = %app_id, manifest_id = %m.app.id,
+            "skipping catalog manifest: embedded app id mismatches catalog key");
+        return None;
+    }
+    if let Err(e) = m.validate() {
+        tracing::warn!(app = %app_id, error = %e,
+            "skipping invalid catalog manifest; using disk fallback");
+        return None;
+    }
+    if m.app.container.build.is_some() {
+        tracing::debug!(app = %app_id,
+            "catalog manifest has a build source; deferring to disk (phase 1 = image-only)");
+        return None;
+    }
+    Some(m)
+}
+
+/// Like [`catalog_manifest_overlay`] but WITHOUT the build-source refusal —
+/// for PORT CLASSIFICATION only, never for install/orchestration.
+///
+/// The on-node-built companion UIs (lnd-ui, bitcoin-ui, electrs-ui, fips-ui)
+/// are exactly the apps whose port policy (auth/bind/session_passthrough)
+/// must reach the gate reliably, yet their build sources made the overlay
+/// defer to DISK manifests — whose only delivery paths (frontend runtime
+/// payload, per-node repo copies) proved stale or absent across the fleet in
+/// the v1.7.125 rollout: nodes served ungated UIs or 401-dead panels until
+/// hand-fixed. The signed catalog is fresher and operator-signed; and the
+/// gate's address binds fail safely on conflict with a container that
+/// publishes differently (logged as CANNOT PROTECT), so classifying from the
+/// catalog cannot open anything the running container hasn't already opened.
+pub fn catalog_manifest_ports_overlay(
+    app_id: &str,
+    value: serde_json::Value,
+) -> Option<archipelago_container::manifest::AppManifest> {
+    let m: archipelago_container::manifest::AppManifest = serde_json::from_value(value).ok()?;
+    if m.app.id != app_id {
+        return None;
+    }
+    if m.validate().is_err() {
+        return None;
+    }
+    Some(m)
+}
+
+/// The catalog's default/latest version string for an app (the top-level
+/// `version` field), if covered. Used to decide whether an install-time
+/// selection should pin (older) or track-latest (default).
+pub fn catalog_default_version(app_id: &str) -> Option<String> {
+    entry_for(app_id)
+        .map(|e| e.version)
+        .filter(|v| !v.is_empty())
+}
+
+/// Curated, selectable versions for an app per the remote catalog. Empty when
+/// the catalog is absent or the app is single-version. The default entry (if
+/// any) sorts first so callers can pre-select it.
+pub fn catalog_versions(app_id: &str) -> Vec<CatalogVersion> {
+    let mut versions = entry_for(app_id).map(|e| e.versions).unwrap_or_default();
+    versions.sort_by_key(|v| !v.default); // default first, stable otherwise
+    versions
+}
+
+/// Resolve the image for a specific selectable `version` of `app_id`, validated
+/// same-repo against `manifest_image` (the same guard `catalog_image_override`
+/// applies). The version's explicit `image` is used when present; otherwise the
+/// repo of `manifest_image` is retagged with `version`. Returns `None` when the
+/// version is unknown or would point at a different repository — the caller then
+/// keeps the default resolution and the switch is refused upstream.
+pub fn catalog_image_for_version(
+    app_id: &str,
+    version: &str,
+    manifest_image: &str,
+) -> Option<String> {
+    let entry = catalog_versions(app_id)
+        .into_iter()
+        .find(|v| v.version == version)?;
+    let manifest_repo =
+        crate::container::image_versions::image_without_registry_or_tag(manifest_image);
+    let candidate = match entry.image {
+        Some(img) => img,
+        None => {
+            // Retag the manifest's full registry/repo with the requested version.
+            let repo = manifest_image
+                .rsplit_once(':')
+                // keep registry:port colons intact: only strip a tag after the last '/'
+                .filter(|(left, _)| left.contains('/'))
+                .map(|(left, _)| left)
+                .unwrap_or(manifest_image);
+            format!("{repo}:{version}")
+        }
+    };
+    let same_repo = crate::container::image_versions::image_without_registry_or_tag(&candidate)
+        == manifest_repo;
+    if same_repo {
+        Some(candidate)
+    } else {
+        warn!(
+            "app-catalog: ignoring version {} for {} — repo mismatch (candidate={}, manifest={})",
+            version, app_id, candidate, manifest_image
+        );
+        None
+    }
+}
+
+/// Image override for the orchestrator's install/upgrade path. Returns the
+/// catalog's primary image for `app_id` ONLY when it refers to the same
+/// repository as the manifest's current image — a guard so a catalog typo can
+/// never redirect an app to an unrelated image. `None` means "use the manifest
+/// image as-is" (catalog absent, app uncovered, or repo mismatch).
+pub fn catalog_image_override(app_id: &str, manifest_image: &str) -> Option<String> {
+    let candidate = catalog_primary_image(app_id)?;
+    let same_repo = crate::container::image_versions::image_without_registry_or_tag(&candidate)
+        == crate::container::image_versions::image_without_registry_or_tag(manifest_image);
+    if same_repo {
+        Some(candidate)
+    } else {
+        warn!(
+            "app-catalog: ignoring image for {} — repo mismatch (catalog={}, manifest={})",
+            app_id, candidate, manifest_image
+        );
+        None
+    }
+}
+
+/// Decoupled "available update" check for ALL apps.
+///
+/// Prefers the remote catalog; when the catalog covers the app, its verdict is
+/// authoritative (so we never advertise a stale `image-versions.sh` pin over a
+/// newer catalog, nor vice-versa). Falls back to the deployed pin only when the
+/// catalog is missing or doesn't cover the app.
+pub fn available_update_for_app(app_id: &str, running_image: &str) -> Option<String> {
+    // A runner-pinned version is an explicit "stay here" choice — never advertise
+    // an update over it (design §3 Phase 3). Auto-update, when enabled, ignores
+    // the pin and is driven by the catalog tick, not this badge.
+    if crate::container::version_config::pinned_version(app_id).is_some() {
+        return None;
+    }
+    if let Some(catalog_image) = catalog_primary_image(app_id) {
+        // Catalog covers this app with a concrete image -> authoritative.
+        return crate::container::image_versions::available_update_for_images(
+            &catalog_image,
+            running_image,
+        );
+    }
+    // Not covered by the catalog -> baseline pin from image-versions.sh.
+    crate::container::image_versions::available_update_for_app(app_id, running_image)
+}
+
+/// Derive candidate catalog URLs from the OTA mirror list by swapping the
+/// manifest filename for the catalog filename. Falls back to the default
+/// manifest origin when no mirrors are configured.
+fn catalog_urls_from_mirrors(mirrors: &[crate::update::UpdateMirror]) -> Vec<String> {
+    let mut urls: Vec<String> = mirrors
+        .iter()
+        .filter_map(|m| {
+            // mirror.url ends with ".../releases/manifest.json"
+            if m.url.ends_with("manifest.json") {
+                Some(m.url.replace("manifest.json", APP_CATALOG_FILE))
+            } else {
+                None
+            }
+        })
+        .collect();
+    urls.dedup();
+    urls
+}
+
+/// Outcome of [`refresh_catalog`]: the app count of the fetched catalog and
+/// whether the cached bytes actually changed. `changed` drives the manifest-
+/// overlay reload — catalog manifests only take effect once `load_manifests`
+/// re-runs, and reloading on every unchanged hourly poll would be pure churn.
+#[derive(Debug, Clone, Copy)]
+pub struct CatalogRefresh {
+    pub apps: usize,
+    pub changed: bool,
+}
+
+/// Fetch the catalog from the first reachable mirror and atomically write it to
+/// `<data_dir>/app-catalog.json`. Returns the app count and whether the cache
+/// changed. Best-effort: a fetch failure leaves the existing cache untouched
+/// (origin-always-wins; updates simply aren't refreshed this cycle).
+pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<CatalogRefresh> {
+    let mirrors = crate::update::load_mirrors(data_dir)
+        .await
+        .unwrap_or_default();
+    let urls = catalog_urls_from_mirrors(&mirrors);
+    if urls.is_empty() {
+        debug!("app-catalog: no mirror-derived URLs to fetch from");
+        return Ok(CatalogRefresh {
+            apps: 0,
+            changed: false,
+        });
+    }
+
+    let client = reqwest::Client::builder()
+        .timeout(std::time::Duration::from_secs(20))
+        .build()?;
+
+    let mut last_err: Option<anyhow::Error> = None;
+    for url in &urls {
+        match fetch_one(&client, url).await {
+            Ok((catalog, body)) => {
+                let count = catalog.apps.len();
+                let changed = write_cache(data_dir, &body)?;
+                if changed {
+                    // Invalidate the in-process cache so the next read re-parses.
+                    *CACHE.lock().unwrap() = None;
+                }
+                info!(
+                    "app-catalog: refreshed from {} ({} apps{})",
+                    url,
+                    count,
+                    if changed { ", changed" } else { ", unchanged" }
+                );
+                return Ok(CatalogRefresh {
+                    apps: count,
+                    changed,
+                });
+            }
+            Err(e) => {
+                debug!("app-catalog: fetch {} failed: {}", url, e);
+                last_err = Some(e);
+            }
+        }
+    }
+    Err(last_err.unwrap_or_else(|| anyhow::anyhow!("no catalog mirrors reachable")))
+}
+
+async fn fetch_one(client: &reqwest::Client, url: &str) -> anyhow::Result<(AppCatalog, String)> {
+    let resp = client.get(url).send().await?;
+    if !resp.status().is_success() {
+        anyhow::bail!("HTTP {}", resp.status());
+    }
+    let body = resp.text().await?;
+    let catalog: AppCatalog = serde_json::from_str(&body)?;
+
+    // DHT Phase 0 authenticity: verify the release-root signature when present.
+    // We verify against the raw JSON (the exact bytes the publisher signed),
+    // not a re-serialization of the typed struct, so unknown forward-compat
+    // fields stay part of the signed preimage. Unsigned catalogs are still
+    // accepted during the migration window — same trust level as today's
+    // manifest — but a *present* signature that fails is a hard reject so a
+    // tampering mirror cannot pass off altered bytes.
+    let raw: serde_json::Value = serde_json::from_str(&body)?;
+    match crate::trust::verify_detached(&raw)? {
+        crate::trust::SignatureStatus::Unsigned => {
+            debug!("app-catalog: unsigned (accepted during migration window)");
+        }
+        crate::trust::SignatureStatus::Verified {
+            signer_did,
+            anchored,
+        } => {
+            if anchored {
+                info!(
+                    "app-catalog: release-root signature verified ({})",
+                    signer_did
+                );
+            } else {
+                warn!(
+                    "app-catalog: signature self-consistent but release-root anchor \
+                     not pinned ({}); cannot confirm signer identity",
+                    signer_did
+                );
+            }
+        }
+    }
+
+    Ok((catalog, body))
+}
+
+/// Atomically write the catalog cache. Caches the RAW fetched bytes — not a
+/// re-serialization of the typed struct — for two reasons: the struct's
+/// `apps` HashMap serializes in nondeterministic order (a re-serialized
+/// comparison would report "changed" on every poll), and the raw bytes are
+/// the signed preimage, so the cache stays signature-verifiable. Returns
+/// `false` (skipping the write) when the bytes are identical to what's
+/// already cached, so callers can tell a genuine catalog change from an
+/// unchanged poll.
+fn write_cache(data_dir: &Path, body: &str) -> anyhow::Result<bool> {
+    let dest = data_dir.join(APP_CATALOG_FILE);
+    if std::fs::read_to_string(&dest)
+        .map(|current| current == body)
+        .unwrap_or(false)
+    {
+        return Ok(false);
+    }
+    let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE));
+    std::fs::write(&tmp, body)?;
+    std::fs::rename(&tmp, &dest)?;
+    Ok(true)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_and_ignores_unknown_fields() {
+        let json = r#"{
+            "schema": 1,
+            "updated": "2026-06-16",
+            "future_field": "ignored",
+            "signature": "sig123",
+            "signed_by": "did:key:zABC",
+            "apps": {
+                "indeedhub": {
+                    "version": "1.0.1",
+                    "image": "source.archipelago-foundation.org/lfg2025/indeedhub:1.0.1",
+                    "digest": "blake3:deadbeef",
+                    "size": 12345,
+                    "another_future_field": true
+                }
+            }
+        }"#;
+        let cat: AppCatalog = serde_json::from_str(json).unwrap();
+        assert_eq!(cat.schema, 1);
+        assert_eq!(cat.signature.as_deref(), Some("sig123"));
+        let e = cat.apps.get("indeedhub").unwrap();
+        assert_eq!(e.version, "1.0.1");
+        assert_eq!(
+            e.image.as_deref(),
+            Some("source.archipelago-foundation.org/lfg2025/indeedhub:1.0.1")
+        );
+        assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef"));
+    }
+
+    #[test]
+    fn write_cache_reports_changed_only_on_new_bytes() {
+        // The changed flag gates the runtime manifest-overlay reload: an
+        // unchanged hourly poll must NOT report changed (or every tick would
+        // rebuild the manifest map), while a genuinely new catalog must. Raw
+        // fetched bytes are compared — a re-serialized comparison would flap
+        // on the apps HashMap's nondeterministic key order (seen live on .228).
+        let dir = tempfile::tempdir().unwrap();
+        let body = r#"{"schema":1,"apps":{"demo":{"version":"1.0.0"}}}"#;
+        assert!(
+            write_cache(dir.path(), body).unwrap(),
+            "first write is a change"
+        );
+        assert!(
+            !write_cache(dir.path(), body).unwrap(),
+            "identical rewrite is not a change"
+        );
+        let body2 = r#"{"schema":1,"apps":{"demo":{"version":"1.0.1"}}}"#;
+        assert!(
+            write_cache(dir.path(), body2).unwrap(),
+            "new catalog bytes are a change"
+        );
+        assert_eq!(
+            std::fs::read_to_string(dir.path().join(APP_CATALOG_FILE)).unwrap(),
+            body2,
+            "cache holds the raw signed preimage bytes"
+        );
+    }
+
+    #[test]
+    fn entry_carries_embedded_manifest() {
+        let json = r#"{
+            "schema": 1,
+            "apps": {
+                "demo": {
+                    "version": "1.0.0",
+                    "manifest": {
+                        "app": {
+                            "id": "demo",
+                            "name": "Demo",
+                            "version": "1.0.0",
+                            "container": { "image": "registry/demo:1.0.0" }
+                        }
+                    }
+                }
+            }
+        }"#;
+        let cat: AppCatalog = serde_json::from_str(json).unwrap();
+        let e = cat.apps.get("demo").unwrap();
+        let m = e.manifest.as_ref().expect("manifest present");
+        assert_eq!(m["app"]["id"], "demo");
+    }
+
+    #[test]
+    fn empty_catalog_when_absent_is_default() {
+        let cat = AppCatalog::default();
+        assert!(cat.apps.is_empty());
+        assert!(cat.signature.is_none());
+    }
+
+    #[test]
+    fn catalog_url_derived_from_mirror() {
+        let mirrors = vec![crate::update::UpdateMirror {
+            url: "https://source.archipelago-foundation.org/lfg2025/archy/raw/branch/main/releases/manifest.json"
+                .to_string(),
+            label: "Server 1".to_string(),
+        }];
+        let urls = catalog_urls_from_mirrors(&mirrors);
+        assert_eq!(
+            urls,
+            vec![
+                "https://source.archipelago-foundation.org/lfg2025/archy/raw/branch/main/releases/app-catalog.json"
+                    .to_string()
+            ]
+        );
+    }
+}

core/archipelago/src/container/bitcoin_ui.rs

@@ -0,0 +1,298 @@
+//! bitcoin-ui nginx.conf renderer.
+//!
+//! Step 7 of the rust-orchestrator migration. Replaces the old
+//! `sed -i __BITCOIN_RPC_AUTH__` approach from `first-boot-containers.sh`
+//! (which destructively overwrote its own template, broke on rotation,
+//! and had no story for dual Knots/Core UIs) with a binary-embedded
+//! template rendered at install/reconcile time and atomic-written to
+//! disk.
+//!
+//! The manifest bind-mounts the rendered file read-only into the
+//! container at `/etc/nginx/conf.d/default.conf`. On every reconcile
+//! pass we re-render and compare — if the rendered bytes would differ
+//! from what's on disk (password rotated, template changed via OTA),
+//! we rewrite atomically and the reconciler restarts the container.
+//!
+//! Source of truth:
+//!   * RPC user: hardcoded `archipelago` (matches the image's `bitcoin.conf`).
+//!   * RPC password: `/var/lib/archipelago/secrets/bitcoin-rpc-password`,
+//!     plaintext, written by the seed-derived credential setup.
+//!
+//! Both Knots and Core back-ends expose RPC on 127.0.0.1:8332 with the
+//! same auth shape, so one template serves both.
+
+use anyhow::{Context, Result};
+use base64::Engine;
+use sha2::{Digest, Sha256};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::fs;
+
+/// The nginx.conf template. Embedded at compile time so it can never
+/// drift from the code that renders it, and ships atomically with OTA.
+///
+/// `{{BITCOIN_RPC_AUTH}}` is the only placeholder — replaced with a
+/// `base64(user:password)` blob at render time.
+pub(crate) const TEMPLATE: &str = include_str!("bitcoin_ui_nginx.conf.template");
+
+/// The single placeholder in `TEMPLATE`.
+const PLACEHOLDER: &str = "{{BITCOIN_RPC_AUTH}}";
+
+/// Hardcoded RPC user. Matches the user written into `bitcoin.conf` by
+/// the bitcoin-core/bitcoin-knots bootstrap, and the legacy
+/// `BITCOIN_RPC_USER="archipelago"` from `first-boot-containers.sh`.
+const RPC_USER: &str = "archipelago";
+
+/// Default path to the plaintext RPC password secret.
+///
+/// Written by the seed-derived credential flow; same file the bash
+/// scripts read today at `first-boot-containers.sh:277` and `:1225`.
+pub const DEFAULT_SECRET_PATH: &str = "/var/lib/archipelago/secrets/bitcoin-rpc-password";
+
+/// Default output path for the rendered nginx.conf.
+///
+/// The manifest bind-mounts this file read-only into the bitcoin-ui
+/// container at `/etc/nginx/conf.d/default.conf`.
+pub const DEFAULT_RENDERED_PATH: &str = "/var/lib/archipelago/bitcoin-ui/nginx.conf";
+
+/// Parameters for rendering. Injectable so tests can hit a tmpdir
+/// instead of `/var/lib/archipelago`.
+#[derive(Debug, Clone)]
+pub struct RenderPaths {
+    /// Path to read the plaintext RPC password from.
+    pub secret_path: PathBuf,
+    /// Path to write the rendered nginx.conf to.
+    pub rendered_path: PathBuf,
+}
+
+impl Default for RenderPaths {
+    fn default() -> Self {
+        Self {
+            secret_path: PathBuf::from(DEFAULT_SECRET_PATH),
+            rendered_path: PathBuf::from(DEFAULT_RENDERED_PATH),
+        }
+    }
+}
+
+/// Outcome of a render pass. `Written` if the rendered bytes differed
+/// from the current on-disk contents and we rewrote; `Unchanged` if
+/// they matched and we left the file alone.
+///
+/// The caller (reconciler / install path) decides whether to restart
+/// the bitcoin-ui container based on this.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RenderOutcome {
+    Written,
+    Unchanged,
+}
+
+/// Render the bitcoin-ui nginx.conf and atomic-write it to disk if it
+/// differs from what's already there.
+///
+/// Idempotent: safe to call on every reconcile pass. Does a byte
+/// comparison before writing so an unchanged password + template is a
+/// no-op (no inode churn, no container restart cascade).
+///
+/// Errors if the secret file is missing or empty. Upstream callers
+/// treat that as "bitcoin-ui isn't installable yet" rather than fatal
+/// — the RPC password comes into being during bitcoin-core's own
+/// bootstrap, which may not have happened yet on a fresh node.
+pub async fn render(paths: &RenderPaths) -> Result<RenderOutcome> {
+    let password = read_password(&paths.secret_path).await?;
+    let auth_b64 = encode_basic_auth(RPC_USER, &password);
+    let rendered = TEMPLATE.replace(PLACEHOLDER, &auth_b64);
+
+    // Compare against existing. read-to-string fails on ENOENT (first
+    // install) — treat as "different".
+    let existing = fs::read_to_string(&paths.rendered_path).await.ok();
+    if existing.as_deref() == Some(rendered.as_str()) {
+        return Ok(RenderOutcome::Unchanged);
+    }
+
+    // Atomic write: write to sibling tmp + rename. Keeps the bind-
+    // mounted file pointing at a fully-formed config at all times.
+    let parent = paths
+        .rendered_path
+        .parent()
+        .ok_or_else(|| anyhow::anyhow!("rendered_path has no parent directory"))?;
+    fs::create_dir_all(parent)
+        .await
+        .with_context(|| format!("creating {}", parent.display()))?;
+
+    let tmp = unique_tmp_path(&paths.rendered_path);
+    fs::write(&tmp, &rendered)
+        .await
+        .with_context(|| format!("writing tmp {}", tmp.display()))?;
+    fs::rename(&tmp, &paths.rendered_path)
+        .await
+        .with_context(|| {
+            format!(
+                "renaming {} -> {}",
+                tmp.display(),
+                paths.rendered_path.display()
+            )
+        })?;
+
+    tracing::info!(
+        path = %paths.rendered_path.display(),
+        auth_hash = %short_hash(&auth_b64),
+        "bitcoin-ui nginx.conf rendered"
+    );
+
+    Ok(RenderOutcome::Written)
+}
+
+fn unique_tmp_path(dest: &Path) -> PathBuf {
+    static COUNTER: AtomicU64 = AtomicU64::new(0);
+    let n = COUNTER.fetch_add(1, Ordering::Relaxed);
+    let ts = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_nanos())
+        .unwrap_or(0);
+    dest.with_extension(format!("tmp.{ts}.{n}"))
+}
+
+/// Read the plaintext RPC password from disk. Trims trailing newlines
+/// (common from `echo "$PASS" > file`) but rejects an empty result.
+async fn read_password(path: &Path) -> Result<String> {
+    let raw = fs::read_to_string(path)
+        .await
+        .with_context(|| format!("reading bitcoin RPC password from {}", path.display()))?;
+    let trimmed = raw.trim().to_string();
+    if trimmed.is_empty() {
+        anyhow::bail!(
+            "bitcoin RPC password file {} is empty — bitcoin-core bootstrap hasn't written it yet",
+            path.display()
+        );
+    }
+    Ok(trimmed)
+}
+
+/// `base64("user:password")` — the value nginx puts after `Basic ` in
+/// the upstream `Authorization` header.
+fn encode_basic_auth(user: &str, password: &str) -> String {
+    let raw = format!("{user}:{password}");
+    base64::engine::general_purpose::STANDARD.encode(raw.as_bytes())
+}
+
+/// Short hash of the auth value for logging — we never want the
+/// plaintext or full base64 in logs (it's a credential), but a stable
+/// fingerprint helps correlate rotations.
+fn short_hash(s: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(s.as_bytes());
+    let digest = hasher.finalize();
+    hex::encode(&digest[..4])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    fn paths_in(dir: &Path, password: &str) -> RenderPaths {
+        let secret = dir.join("bitcoin-rpc-password");
+        std::fs::write(&secret, password).unwrap();
+        RenderPaths {
+            secret_path: secret,
+            rendered_path: dir.join("nginx.conf"),
+        }
+    }
+
+    #[tokio::test]
+    async fn render_writes_file_with_substitution() {
+        let tmp = TempDir::new().unwrap();
+        let paths = paths_in(tmp.path(), "hunter2");
+        let outcome = render(&paths).await.unwrap();
+        assert_eq!(outcome, RenderOutcome::Written);
+        let contents = std::fs::read_to_string(&paths.rendered_path).unwrap();
+        // archipelago:hunter2 -> "YXJjaGlwZWxhZ286aHVudGVyMg=="
+        assert!(
+            contents.contains("YXJjaGlwZWxhZ286aHVudGVyMg=="),
+            "base64 auth not found in rendered config:\n{contents}"
+        );
+        assert!(
+            !contents.contains(PLACEHOLDER),
+            "placeholder left in output"
+        );
+    }
+
+    #[tokio::test]
+    async fn render_is_idempotent_when_password_unchanged() {
+        let tmp = TempDir::new().unwrap();
+        let paths = paths_in(tmp.path(), "hunter2");
+        let first = render(&paths).await.unwrap();
+        assert_eq!(first, RenderOutcome::Written);
+        let second = render(&paths).await.unwrap();
+        assert_eq!(second, RenderOutcome::Unchanged);
+    }
+
+    #[tokio::test]
+    async fn render_rewrites_on_password_rotation() {
+        let tmp = TempDir::new().unwrap();
+        let paths = paths_in(tmp.path(), "old-pass");
+        render(&paths).await.unwrap();
+        // Rotate.
+        std::fs::write(&paths.secret_path, "new-pass").unwrap();
+        let outcome = render(&paths).await.unwrap();
+        assert_eq!(outcome, RenderOutcome::Written);
+        let contents = std::fs::read_to_string(&paths.rendered_path).unwrap();
+        // archipelago:new-pass -> "YXJjaGlwZWxhZ286bmV3LXBhc3M="
+        assert!(contents.contains("YXJjaGlwZWxhZ286bmV3LXBhc3M="));
+    }
+
+    #[tokio::test]
+    async fn render_trims_trailing_newline_from_secret() {
+        // Matches `echo "$PASS" > file` behaviour.
+        let tmp = TempDir::new().unwrap();
+        let paths = paths_in(tmp.path(), "hunter2\n");
+        render(&paths).await.unwrap();
+        let contents = std::fs::read_to_string(&paths.rendered_path).unwrap();
+        assert!(
+            contents.contains("YXJjaGlwZWxhZ286aHVudGVyMg=="),
+            "trailing newline should be stripped before encoding"
+        );
+    }
+
+    #[tokio::test]
+    async fn render_errors_on_empty_password() {
+        let tmp = TempDir::new().unwrap();
+        let paths = paths_in(tmp.path(), "");
+        let err = render(&paths).await.unwrap_err();
+        let msg = format!("{err}");
+        assert!(msg.contains("empty"), "unexpected error: {msg}");
+    }
+
+    #[tokio::test]
+    async fn render_errors_when_secret_missing() {
+        let tmp = TempDir::new().unwrap();
+        let paths = RenderPaths {
+            secret_path: tmp.path().join("does-not-exist"),
+            rendered_path: tmp.path().join("nginx.conf"),
+        };
+        let err = render(&paths).await.unwrap_err();
+        let msg = format!("{err}");
+        assert!(
+            msg.contains("reading bitcoin RPC password"),
+            "unexpected error: {msg}"
+        );
+    }
+
+    #[test]
+    fn template_contains_exactly_one_placeholder() {
+        // Safety net: if someone adds a second placeholder to the
+        // template without updating the renderer, we want a test to
+        // fail loudly rather than ship a half-substituted config.
+        let count = TEMPLATE.matches(PLACEHOLDER).count();
+        assert_eq!(count, 1, "template must contain exactly one {PLACEHOLDER}");
+    }
+
+    #[test]
+    fn template_proxies_bitcoin_rpc_on_8332() {
+        // Lock in the core shape so a bad template edit doesn't ship.
+        assert!(TEMPLATE.contains("proxy_pass http://127.0.0.1:8332/"));
+        assert!(TEMPLATE.contains("location /bitcoin-rpc/"));
+        assert!(TEMPLATE.contains("listen 127.0.0.1:8334"));
+    }
+}

core/archipelago/src/container/bitcoin_ui_nginx.conf.template

@@ -0,0 +1,83 @@
+server {
+    # Loopback ONLY. This container is host-networked, so this nginx binds the
+    # HOST's address directly — `listen 8334;` meant every interface, and the
+    # app gate could never stand in front of it (there is no podman publish to
+    # pin, and the manifest declared no port, so the gate neither protected it
+    # nor reported it — it served this page to anyone who asked, on LAN,
+    # Tailscale and the mesh alike). Binding loopback lets the daemon claim the
+    # external addresses and authenticate them; see appgate::listener.
+    listen 127.0.0.1:8334;
+    server_name _;
+    root /usr/share/nginx/html;
+    index index.html;
+    # Session gate for the credential-injecting proxy below. Internal: it can
+    # only be reached by nginx's own auth_request subrequest, never by a client.
+    location = /_session_check {
+        internal;
+        proxy_pass http://127.0.0.1:5678/auth/session-check;
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+        proxy_set_header Host $host;
+        proxy_set_header Cookie $http_cookie;
+        proxy_set_header X-CSRF-Token $http_x_csrf_token;
+    }
+
+    # Bitcoin Core RPC. This proxy ADDS Bitcoin Core's Basic auth that the
+    # caller never supplied, so reaching this location at all is equivalent to
+    # holding the node's RPC credentials — which is why it must be gated here
+    # rather than anywhere upstream.
+    #
+    # It previously had no gate at all. This port (8334) binds 0.0.0.0 AND sits
+    # on the fips0 mesh allowlist (fips/app_ports.rs), so any mesh peer, LAN
+    # host or Tailscale peer could POST authenticated Bitcoin Core RPC —
+    # including wallet methods, with a wallet loaded. Verified live on
+    # a test node 2026-08-02.
+    #
+    # `Access-Control-Allow-Origin *` is also removed: paired with a proxy that
+    # injects credentials it let any web page a user visited drive this RPC.
+    location /bitcoin-rpc/ {
+        # Preflight carries no cookies by design — answer it before the gate,
+        # otherwise the browser reports an opaque CORS failure instead of a 401.
+        if ($request_method = OPTIONS) { return 204; }
+        auth_request /_session_check;
+        proxy_pass http://127.0.0.1:8332/;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Authorization "Basic {{BITCOIN_RPC_AUTH}}";
+        add_header Access-Control-Allow-Origin $scheme://$http_host always;
+        add_header Access-Control-Allow-Credentials "true" always;
+        add_header Vary "Origin" always;
+        add_header Access-Control-Allow-Methods "POST, GET, OPTIONS" always;
+        add_header Access-Control-Allow-Headers "Content-Type, Authorization" always;
+    }
+    location /bitcoin-status {
+        proxy_pass http://127.0.0.1:5678/bitcoin-status;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        add_header Cache-Control "no-store";
+    }
+    location /rpc/v1 {
+        proxy_pass http://127.0.0.1:5678/rpc/v1;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header Cookie $http_cookie;
+        proxy_set_header X-CSRF-Token $http_x_csrf_token;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        add_header Cache-Control "no-store";
+    }
+    # no-cache so a rebuilt image is actually seen. Without this nginx sends
+    # only ETag/Last-Modified for index.html, and browsers apply heuristic
+    # caching to it — so an OTA that ships a new bitcoin-ui kept rendering the
+    # previous UI until the user hard-refreshed. docker/lnd-ui/nginx.conf has
+    # carried the same header for this reason. "no-cache" (revalidate), not
+    # "no-store", so the ETag still saves the transfer when nothing changed.
+    location / {
+        add_header Cache-Control "no-cache";
+        try_files $uri $uri/ /index.html;
+    }
+}

core/archipelago/src/container/boot_reconciler.rs

@@ -0,0 +1,488 @@
+//! BootReconciler — the long-running task that keeps the prod orchestrator's
+//! desired-state view in lockstep with what podman actually has.
+//!
+//! Step 5 of the rust-orchestrator migration. Spawned once from `main.rs`
+//! (Step 6) after the initial `adopt_existing()` pass. Every `interval` it
+//! calls `ProdContainerOrchestrator::reconcile_existing()`, which repairs
+//! containers that already exist without installing every catalog manifest.
+//!
+//! Per answered design Q3, `interval` defaults to 30 seconds.
+//!
+//! Shutdown is signalled via `Arc<Notify>`. The reconciler finishes its
+//! current `reconcile_all` call before exiting — we don't interrupt an
+//! in-flight pull or build.
+//!
+//! See `docs/rust-orchestrator-migration.md` §269-352.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use tokio::sync::Notify;
+use tokio::time::{self, Instant};
+
+use crate::container::prod_orchestrator::{ProdContainerOrchestrator, ReconcileReport};
+
+/// Default reconciler cadence (answered design Q3).
+pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(30);
+
+pub struct BootReconciler {
+    orchestrator: Arc<ProdContainerOrchestrator>,
+    interval: Duration,
+    shutdown: Arc<Notify>,
+    /// Run the companion-unit repair stage each tick. Default true.
+    /// Tests disable this — companion reconcile shells out to
+    /// `systemctl --user` and `podman`, which both block real time
+    /// and would race the paused-clock test fixtures.
+    companion_stage: bool,
+    wait_for_recovery: bool,
+}
+
+impl BootReconciler {
+    pub fn new(
+        orchestrator: Arc<ProdContainerOrchestrator>,
+        interval: Duration,
+        shutdown: Arc<Notify>,
+    ) -> Self {
+        Self {
+            orchestrator,
+            interval,
+            shutdown,
+            companion_stage: true,
+            wait_for_recovery: true,
+        }
+    }
+
+    /// Disable the companion-unit reconcile stage. Used by unit tests
+    /// that exercise loop cadence without the real systemd / podman
+    /// surface. Production must not call this.
+    #[cfg(test)]
+    pub fn without_companion_stage(mut self) -> Self {
+        self.companion_stage = false;
+        self.wait_for_recovery = false;
+        self
+    }
+
+    /// Run the reconcile loop until `shutdown` is notified.
+    ///
+    /// Does one reconcile immediately, then sleeps `interval` between
+    /// subsequent passes. A `shutdown.notify_one()` call unblocks the sleep
+    /// and the task returns after the *next* pass completes.
+    ///
+    /// Each pass is two stages:
+    /// 1. App reconcile: `reconcile_all()` keeps every loaded manifest's
+    ///    container running.
+    /// 2. Companion reconcile: any expected Quadlet companion unit that
+    ///    is missing or inactive is repaired (writes the unit, daemon-
+    ///    reloads, starts the service). This is the safety net for the
+    ///    "someone deleted my unit file" / "systemd lost the service"
+    ///    failure modes.
+    ///
+    /// Never panics: per-app failures are absorbed into `ReconcileReport`
+    /// by the orchestrator, and companion failures are logged but never
+    /// propagated.
+    pub async fn run_forever(self) {
+        let wait_start = Instant::now();
+        while self.wait_for_recovery && !crate::crash_recovery::is_recovery_complete() {
+            if wait_start.elapsed() > Duration::from_secs(1800) {
+                tracing::warn!("boot reconciler: boot recovery did not complete within 30 minutes, starting anyway");
+                break;
+            }
+            tokio::select! {
+                _ = time::sleep(Duration::from_secs(5)) => {}
+                _ = self.shutdown.notified() => {
+                    tracing::info!("boot reconciler: shutdown requested before recovery completed");
+                    return;
+                }
+            }
+        }
+
+        // Companion self-heal runs on its OWN cadence, decoupled from the
+        // per-app reconcile pass. On a heavily loaded node `reconcile_existing`
+        // over dozens of apps can take well over a minute, which would delay a
+        // companion-unit repair (deleted/lost unit file) past any reasonable
+        // safety window. Detecting + rewriting a companion unit is cheap, so it
+        // gets a dedicated `interval` loop. The handle is aborted when the main
+        // loop exits (shutdown uses `notify_one`, so we must NOT add a second
+        // waiter on `self.shutdown` — it would steal the single wake permit).
+        let companion_handle = if self.companion_stage {
+            let orchestrator = self.orchestrator.clone();
+            let interval = self.interval;
+            let data_dir = orchestrator.data_dir().to_path_buf();
+            Some(tokio::spawn(async move {
+                let mut failure_rounds: u32 = 0;
+                loop {
+                    // `installed_app_ids`, NOT `manifest_ids`: a manifest exists
+                    // on disk for every *available* app, so driving companion
+                    // provisioning from it stood up a UI for apps nobody had
+                    // installed and self-healed it forever (archi-dev-box ran
+                    // archy-fedimint-ui and archy-lnd-ui with no fedimint and no
+                    // lnd container present — the Guardian UI served its wait
+                    // page with nothing behind it, reported as "fedimint
+                    // installs but does not work"). `None` means the container
+                    // listing failed: skip the whole stage rather than reap
+                    // every companion on a transient probe error.
+                    let Some(installed) = orchestrator.installed_app_ids().await else {
+                        tracing::warn!(
+                            "companion reconcile: cannot determine installed apps, skipping this pass"
+                        );
+                        time::sleep(interval).await;
+                        continue;
+                    };
+                    let failures = crate::container::companion::reconcile(&installed).await;
+                    // Reaper, RE-WIRED 2026-08-10 — driven by the DURABLE
+                    // installed-apps registry, never by runtime inference.
+                    //
+                    // History: this call was unwired on 2026-08-08 after it
+                    // removed archy-bitcoin-ui and archy-lnd-ui for apps that
+                    // WERE installed. Not a logic error — the inputs lied:
+                    // `installed_app_ids` infers installation from runtime
+                    // state (containers present + running-containers.json),
+                    // and the clean-exit vanishing bug falsified both signals
+                    // at once. The unwire commit set the re-wire bar: a
+                    // durable record of "this app is installed".
+                    //
+                    // That record now exists — installed-apps.json, written on
+                    // install, cleared on deliberate uninstall, backfilled at
+                    // boot from demonstrably-present containers, and immune to
+                    // container absence by construction (89b03c47 holds
+                    // entries while a container is gone). A vanished backend
+                    // no longer looks uninstalled, so the failure mode that
+                    // burned archi-dev-box cannot recur through this path.
+                    //
+                    // `None` = the registry could not be read (missing or
+                    // corrupt) — which is "I could not look", NOT "nothing is
+                    // installed". The reaper stays idle in that case; the
+                    // runtime-derived `installed` set above is deliberately
+                    // NOT used as a fallback (it is exactly the input class
+                    // that caused the 2026-08-08 incident). ORPHAN_GRACE still
+                    // applies on top: a companion must be orphaned for the
+                    // full grace period before it is touched.
+                    if let Some(durable) =
+                        crate::crash_recovery::load_installed_apps_if_recorded(&data_dir).await
+                    {
+                        let durable: Vec<String> = durable.into_iter().collect();
+                        for (companion, err) in
+                            crate::container::companion::reap_orphans(&durable).await
+                        {
+                            tracing::warn!(
+                                companion = %companion,
+                                error = %err,
+                                "companion reap failed"
+                            );
+                        }
+                    }
+                    for (companion, err) in &failures {
+                        tracing::warn!(
+                            companion = %companion,
+                            error = %err,
+                            "companion reconcile failed"
+                        );
+                    }
+                    // A failed repair can involve registry pulls and full
+                    // image builds; retrying every 30s hammered unreachable
+                    // registries ~174×/image/day on an offline node
+                    // (a test node log sweep, 2026-07-22). Back off
+                    // exponentially while rounds keep failing — 30s doubling
+                    // to a 1h cap — and reset the moment a round is clean.
+                    failure_rounds = if failures.is_empty() {
+                        0
+                    } else {
+                        failure_rounds.saturating_add(1)
+                    };
+                    let backoff = interval
+                        .saturating_mul(2u32.saturating_pow(failure_rounds.min(7)))
+                        .min(Duration::from_secs(3600));
+                    time::sleep(backoff).await;
+                }
+            }))
+        } else {
+            None
+        };
+
+        // Initial pass: no delay.
+        self.tick().await;
+
+        loop {
+            let deadline = Instant::now() + self.interval;
+            tokio::select! {
+                _ = time::sleep_until(deadline) => {
+                    self.tick().await;
+                }
+                _ = self.shutdown.notified() => {
+                    tracing::info!("boot reconciler: shutdown requested, exiting loop");
+                    break;
+                }
+            }
+        }
+
+        if let Some(handle) = companion_handle {
+            handle.abort();
+        }
+    }
+
+    async fn tick(&self) {
+        let report = self.orchestrator.reconcile_existing().await;
+        Self::log_report(&report);
+    }
+
+    fn log_report(report: &ReconcileReport) {
+        for (app_id, action) in &report.actions {
+            tracing::debug!(app_id = %app_id, action = ?action, "reconcile action");
+        }
+        for (app_id, err) in &report.failures {
+            tracing::warn!(app_id = %app_id, error = %err, "reconcile failure");
+        }
+        if report.failures.is_empty() {
+            tracing::debug!(count = report.actions.len(), "reconcile pass complete");
+        } else {
+            tracing::warn!(
+                ok = report.actions.len(),
+                failed = report.failures.len(),
+                "reconcile pass completed with failures"
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::container::prod_orchestrator::ProdContainerOrchestrator;
+    use anyhow::Result;
+    use archipelago_container::{
+        AppManifest, BuildConfig, ContainerRuntime as ContainerRuntimeTrait, ContainerState,
+        ContainerStatus,
+    };
+    use async_trait::async_trait;
+    use std::collections::HashMap;
+    use std::path::PathBuf;
+    use std::sync::Mutex as StdMutex;
+
+    /// Instrumented runtime that counts reconcile-loop side effects so tests
+    /// can tell exactly how many passes have fired. All containers are
+    /// reported as already Running so `reconcile_all` will NoOp — we are only
+    /// measuring loop cadence, not install behavior.
+    #[derive(Default)]
+    struct CountingRuntime {
+        /// Number of times get_container_status has been called. Each
+        /// reconcile_all pass hits this once per manifest, so with one
+        /// manifest this equals the number of reconcile passes.
+        status_calls: StdMutex<u32>,
+        running: StdMutex<HashMap<String, ContainerState>>,
+    }
+
+    impl CountingRuntime {
+        fn new_with(names: &[&str]) -> Self {
+            let me = Self::default();
+            let mut m = me.running.lock().unwrap();
+            for n in names {
+                m.insert((*n).to_string(), ContainerState::Running);
+            }
+            drop(m);
+            me
+        }
+        fn status_call_count(&self) -> u32 {
+            *self.status_calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait]
+    impl ContainerRuntimeTrait for CountingRuntime {
+        async fn pull_image(&self, _: &str, _: Option<&str>) -> Result<()> {
+            Ok(())
+        }
+        async fn create_container(&self, _: &AppManifest, name: &str, _: u16) -> Result<String> {
+            Ok(name.to_string())
+        }
+        async fn start_container(&self, _: &str) -> Result<()> {
+            Ok(())
+        }
+        async fn stop_container(&self, _: &str) -> Result<()> {
+            Ok(())
+        }
+        async fn remove_container(&self, _: &str) -> Result<()> {
+            Ok(())
+        }
+        async fn get_container_status(&self, name: &str) -> Result<ContainerStatus> {
+            *self.status_calls.lock().unwrap() += 1;
+            let state = self
+                .running
+                .lock()
+                .unwrap()
+                .get(name)
+                .cloned()
+                .ok_or_else(|| anyhow::anyhow!("not found: {name}"))?;
+            Ok(ContainerStatus {
+                id: format!("id-{name}"),
+                name: name.to_string(),
+                state,
+                health: None,
+                exit_code: None,
+                started_at: None,
+                image: "test".into(),
+                created: "now".into(),
+                ports: vec![],
+                lan_address: None,
+            })
+        }
+        async fn get_container_logs(&self, _: &str, _: u32) -> Result<Vec<String>> {
+            Ok(vec![])
+        }
+        async fn list_containers(&self) -> Result<Vec<ContainerStatus>> {
+            Ok(vec![])
+        }
+        async fn image_exists(&self, _: &str) -> Result<bool> {
+            Ok(true)
+        }
+        async fn build_image(&self, _: &BuildConfig) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    fn pull_manifest(id: &str, image: &str) -> AppManifest {
+        let yaml = format!(
+            "app:\n  id: {id}\n  name: {id}\n  version: 1.0.0\n  container:\n    image: {image}\n"
+        );
+        AppManifest::parse(&yaml).unwrap()
+    }
+
+    async fn orch_with_one_running_manifest(
+        rt: Arc<CountingRuntime>,
+    ) -> Arc<ProdContainerOrchestrator> {
+        let mut orch =
+            ProdContainerOrchestrator::with_runtime(rt, PathBuf::from("/nonexistent-for-tests"));
+        let tmp = tempfile::tempdir().unwrap().keep();
+        orch.set_data_dir(tmp);
+        orch.set_disk_gb_for_test(2_000);
+        let orch = Arc::new(orch);
+        orch.insert_manifest_for_test(
+            pull_manifest("test-app", "docker.io/example/test-app:1"),
+            PathBuf::from("/tmp/test-app"),
+        )
+        .await;
+        orch
+    }
+
+    async fn wait_for_status_calls(rt: &CountingRuntime, expected: u32) -> u32 {
+        for _ in 0..1000 {
+            let count = rt.status_call_count();
+            if count >= expected {
+                return count;
+            }
+            tokio::task::yield_now().await;
+            tokio::time::sleep(Duration::from_millis(1)).await;
+        }
+        rt.status_call_count()
+    }
+
+    #[tokio::test]
+    async fn initial_pass_fires_immediately() {
+        let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
+        let orch = orch_with_one_running_manifest(rt.clone()).await;
+        let shutdown = Arc::new(Notify::new());
+        let reconciler =
+            BootReconciler::new(orch.clone(), Duration::from_millis(50), shutdown.clone())
+                .without_companion_stage();
+        let handle = tokio::spawn(reconciler.run_forever());
+
+        // We expect exactly one reconcile pass to have run by now (the initial),
+        // NOT a second one (the 30s sleep hasn't elapsed in paused time).
+        assert_eq!(
+            wait_for_status_calls(&rt, 1).await,
+            1,
+            "initial pass should fire once"
+        );
+
+        shutdown.notify_one();
+        tokio::task::yield_now().await;
+        let _ = tokio::time::timeout(Duration::from_secs(1), handle).await;
+    }
+
+    #[tokio::test]
+    async fn second_pass_fires_after_interval() {
+        let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
+        let orch = orch_with_one_running_manifest(rt.clone()).await;
+        let shutdown = Arc::new(Notify::new());
+        let reconciler =
+            BootReconciler::new(orch.clone(), Duration::from_millis(10), shutdown.clone())
+                .without_companion_stage();
+        let handle = tokio::spawn(reconciler.run_forever());
+
+        assert_eq!(wait_for_status_calls(&rt, 1).await, 1);
+
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        let count = wait_for_status_calls(&rt, 2).await;
+
+        assert!(
+            count >= 2,
+            "a second reconcile pass should fire after one interval"
+        );
+
+        shutdown.notify_one();
+        tokio::task::yield_now().await;
+        let _ = tokio::time::timeout(Duration::from_secs(1), handle).await;
+    }
+
+    #[tokio::test]
+    async fn shutdown_terminates_loop() {
+        let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
+        let orch = orch_with_one_running_manifest(rt.clone()).await;
+        let shutdown = Arc::new(Notify::new());
+        let reconciler =
+            BootReconciler::new(orch.clone(), Duration::from_millis(50), shutdown.clone())
+                .without_companion_stage();
+        let handle = tokio::spawn(reconciler.run_forever());
+        wait_for_status_calls(&rt, 1).await;
+
+        shutdown.notify_one();
+        let result = tokio::time::timeout(Duration::from_secs(5), handle).await;
+        assert!(result.is_ok(), "reconciler did not exit after shutdown");
+    }
+
+    #[tokio::test]
+    async fn failure_in_one_pass_does_not_stop_loop() {
+        // Manifest references a container the runtime does not have AND
+        // cannot create (no install path — install_fresh will also fail to
+        // pull, since CountingRuntime::pull_image returns Ok but the
+        // manifest's referenced container stays uncreated). In practice
+        // reconcile_all will observe the missing container, install_fresh
+        // will run, and the next pass will see a new state. We care about
+        // "loop keeps ticking even when the report has actions".
+        let rt = Arc::new(CountingRuntime::default());
+        let mut orch = ProdContainerOrchestrator::with_runtime(
+            rt.clone(),
+            PathBuf::from("/nonexistent-for-tests"),
+        );
+        let tmp = tempfile::tempdir().unwrap().keep();
+        orch.set_data_dir(tmp);
+        orch.set_disk_gb_for_test(2_000);
+        let orch = Arc::new(orch);
+        orch.insert_manifest_for_test(
+            pull_manifest("test-app", "docker.io/example/test-app:1"),
+            PathBuf::from("/tmp/test-app"),
+        )
+        .await;
+        let shutdown = Arc::new(Notify::new());
+        let reconciler =
+            BootReconciler::new(orch.clone(), Duration::from_millis(10), shutdown.clone())
+                .without_companion_stage();
+        let handle = tokio::spawn(reconciler.run_forever());
+
+        let first = wait_for_status_calls(&rt, 1).await;
+        assert!(first >= 1, "initial pass should have touched the runtime");
+
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        let second = wait_for_status_calls(&rt, first + 1).await;
+        assert!(
+            second > first,
+            "loop should have fired a second pass after the interval"
+        );
+
+        shutdown.notify_one();
+        let _ = tokio::time::timeout(Duration::from_secs(5), handle).await;
+    }
+}

core/archipelago/src/container/data_manager.rs

@@ -0,0 +1,96 @@
+use anyhow::{Context, Result};
+use std::path::PathBuf;
+use tokio::fs;
+
+pub struct DevDataManager {
+    dev_data_dir: PathBuf,
+}
+
+impl DevDataManager {
+    pub fn new(dev_data_dir: PathBuf) -> Self {
+        Self { dev_data_dir }
+    }
+
+    /// Get the dev data directory for an app
+    pub fn get_app_data_dir(&self, app_id: &str) -> PathBuf {
+        self.dev_data_dir.join(app_id)
+    }
+
+    /// Create data directory for an app
+    pub async fn create_app_data_dir(&self, app_id: &str) -> Result<PathBuf> {
+        let app_dir = self.get_app_data_dir(app_id);
+        fs::create_dir_all(&app_dir)
+            .await
+            .with_context(|| format!("Failed to create app data directory: {:?}", app_dir))?;
+        Ok(app_dir)
+    }
+
+    /// Map a volume source path to dev path
+    pub fn map_volume_path(&self, app_id: &str, volume_source: &str) -> PathBuf {
+        // If volume source is already in dev_data_dir, use it as-is
+        if volume_source.starts_with(self.dev_data_dir.to_str().unwrap_or("")) {
+            PathBuf::from(volume_source)
+        } else {
+            // Map production path to dev path
+            // e.g., /var/lib/archipelago/bitcoin -> /tmp/archipelago-dev/bitcoin
+            let app_dir = self.get_app_data_dir(app_id);
+
+            // Extract the relative path from the production path
+            if let Some(relative) = volume_source.strip_prefix("/var/lib/archipelago/") {
+                app_dir.join(relative)
+            } else if let Some(relative) = volume_source.strip_prefix("/var/lib/archipelago") {
+                app_dir.join(relative)
+            } else {
+                // If it doesn't match expected pattern, use app_id as base
+                app_dir.join(volume_source.trim_start_matches('/'))
+            }
+        }
+    }
+
+    /// Clean up app data directory
+    pub async fn cleanup_app_data(&self, app_id: &str) -> Result<()> {
+        let app_dir = self.get_app_data_dir(app_id);
+        if app_dir.exists() {
+            fs::remove_dir_all(&app_dir)
+                .await
+                .with_context(|| format!("Failed to remove app data directory: {:?}", app_dir))?;
+        }
+        Ok(())
+    }
+
+    /// Preserve app data (no-op for cleanup, used when removing container)
+    pub async fn preserve_app_data(&self, _app_id: &str) -> Result<()> {
+        // In dev mode, we might want to preserve data between container removals
+        // This is a no-op by default, but can be extended
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_map_volume_path() {
+        let temp_dir = std::env::temp_dir().join("test-archipelago");
+        let manager = DevDataManager::new(temp_dir.clone());
+
+        let dev_path = manager.map_volume_path("bitcoin-core", "/var/lib/archipelago/bitcoin");
+        assert!(dev_path.to_string_lossy().contains("bitcoin-core"));
+
+        // Cleanup
+        let _ = tokio::fs::remove_dir_all(&temp_dir).await;
+    }
+
+    #[tokio::test]
+    async fn test_create_app_data_dir() {
+        let temp_dir = std::env::temp_dir().join("test-archipelago-2");
+        let manager = DevDataManager::new(temp_dir.clone());
+
+        let app_dir = manager.create_app_data_dir("test-app").await.unwrap();
+        assert!(app_dir.exists());
+
+        // Cleanup
+        let _ = tokio::fs::remove_dir_all(&temp_dir).await;
+    }
+}

core/archipelago/src/container/dev_orchestrator.rs

@@ -0,0 +1,411 @@
+use anyhow::{Context, Result};
+use archipelago_container::{
+    AppManifest, BitcoinSimulationMode, BitcoinSimulator,
+    ContainerRuntime as ContainerRuntimeTrait, ContainerStatus, PortManager, ResolvedSource,
+};
+use async_trait::async_trait;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use crate::config::{BitcoinSimulation, Config, ContainerRuntime};
+use crate::container::data_manager::DevDataManager;
+use crate::container::traits::ContainerOrchestrator;
+
+pub struct DevContainerOrchestrator {
+    runtime: Arc<dyn ContainerRuntimeTrait>,
+    port_manager: Arc<PortManager>,
+    bitcoin_simulator: Arc<BitcoinSimulator>,
+    data_manager: Arc<DevDataManager>,
+    config: Config,
+}
+
+impl DevContainerOrchestrator {
+    pub async fn new(config: Config) -> Result<Self> {
+        let user = std::env::var("USER").unwrap_or_else(|_| "archipelago".to_string());
+
+        // Create runtime based on config
+        let runtime: Arc<dyn ContainerRuntimeTrait> = match &config.container_runtime {
+            ContainerRuntime::Podman => {
+                Arc::new(archipelago_container::PodmanRuntime::new(user.clone()))
+            }
+            ContainerRuntime::Docker => {
+                Arc::new(archipelago_container::DockerRuntime::new(user.clone()))
+            }
+            ContainerRuntime::Auto => Arc::new(
+                archipelago_container::AutoRuntime::new(user.clone())
+                    .await
+                    .context("Failed to create auto runtime")?,
+            ),
+        };
+
+        let port_manager = Arc::new(PortManager::new(config.port_offset));
+        let bitcoin_simulator = Arc::new(BitcoinSimulator::new(BitcoinSimulationMode::from(
+            match &config.bitcoin_simulation {
+                BitcoinSimulation::Mock => "mock",
+                BitcoinSimulation::Testnet => "testnet",
+                BitcoinSimulation::Mainnet => "mainnet",
+                BitcoinSimulation::None => "none",
+            },
+        )));
+        let data_manager = Arc::new(DevDataManager::new(config.dev_data_dir.clone()));
+
+        Ok(Self {
+            runtime,
+            port_manager,
+            bitcoin_simulator,
+            data_manager,
+            config,
+        })
+    }
+
+    /// Install a container from a manifest
+    pub async fn install_container(
+        &self,
+        manifest: &AppManifest,
+        _manifest_path: &str,
+    ) -> Result<String> {
+        let app_id = &manifest.app.id;
+        let container_name = format!("archipelago-{}-dev", app_id);
+
+        // Check dependencies
+        if self.config.dev_mode {
+            // In dev mode, check if Bitcoin dependency can be satisfied
+            for dep in &manifest.app.dependencies {
+                if let archipelago_container::Dependency::App {
+                    app_id: dep_id,
+                    version: _,
+                } = dep
+                {
+                    if dep_id == "bitcoin-core" && !self.bitcoin_simulator.is_bitcoin_available() {
+                        return Err(anyhow::anyhow!(
+                            "Bitcoin Core dependency not satisfied (simulation: {:?})",
+                            self.bitcoin_simulator.mode()
+                        ));
+                    }
+                }
+            }
+        }
+
+        // Allocate ports
+        let base_ports: Vec<u16> = manifest.app.ports.iter().map(|p| p.host).collect();
+        let _dev_ports = self
+            .port_manager
+            .allocate_ports(app_id, &base_ports)
+            .context("Failed to allocate ports")?;
+
+        // Create app data directory
+        self.data_manager
+            .create_app_data_dir(app_id)
+            .await
+            .context("Failed to create app data directory")?;
+
+        // Map volumes to dev paths
+        let mut dev_manifest = manifest.clone();
+        for volume in &mut dev_manifest.app.volumes {
+            let dev_path = self.data_manager.map_volume_path(app_id, &volume.source);
+            volume.source = dev_path.to_string_lossy().to_string();
+        }
+
+        // Resolve pull-or-build. Dev orchestrator currently only supports pull;
+        // Build support lands in Step 2 of the rust-orchestrator migration.
+        match manifest.app.container.resolve().ok_or_else(|| {
+            anyhow::anyhow!("manifest container config invalid (neither image nor build)")
+        })? {
+            ResolvedSource::Pull {
+                image,
+                image_signature,
+                ..
+            } => {
+                self.runtime
+                    .pull_image(&image, image_signature.as_deref())
+                    .await
+                    .context("Failed to pull image")?;
+            }
+            ResolvedSource::Build(_) => {
+                anyhow::bail!(
+                    "dev orchestrator does not yet support local image builds (see rust-orchestrator-migration.md Step 2)"
+                );
+            }
+        }
+
+        // Create container with port offset
+        let port_offset = if self.config.dev_mode {
+            self.config.port_offset
+        } else {
+            0
+        };
+
+        self.runtime
+            .create_container(&dev_manifest, &container_name, port_offset)
+            .await
+            .context("Failed to create container")?;
+
+        Ok(container_name)
+    }
+
+    /// Start a container
+    pub async fn start_container(&self, app_id: &str) -> Result<()> {
+        let container_name = format!("archipelago-{}-dev", app_id);
+        self.runtime
+            .start_container(&container_name)
+            .await
+            .context("Failed to start container")?;
+        Ok(())
+    }
+
+    /// Stop a container
+    pub async fn stop_container(&self, app_id: &str) -> Result<()> {
+        let container_name = format!("archipelago-{}-dev", app_id);
+        self.runtime
+            .stop_container(&container_name)
+            .await
+            .context("Failed to stop container")?;
+        Ok(())
+    }
+
+    /// Remove a container
+    pub async fn remove_container(&self, app_id: &str, preserve_data: bool) -> Result<()> {
+        let container_name = format!("archipelago-{}-dev", app_id);
+
+        // Stop container first
+        let _ = self.runtime.stop_container(&container_name).await;
+
+        // Remove container
+        self.runtime
+            .remove_container(&container_name)
+            .await
+            .context("Failed to remove container")?;
+
+        // Release ports
+        let _ = self.port_manager.release_ports(app_id);
+
+        // Clean up or preserve data
+        if preserve_data {
+            self.data_manager.preserve_app_data(app_id).await?;
+        } else {
+            self.data_manager.cleanup_app_data(app_id).await?;
+        }
+
+        Ok(())
+    }
+
+    /// Get container status with dev port info
+    pub async fn get_container_status(&self, app_id: &str) -> Result<ContainerStatus> {
+        let container_name = format!("archipelago-{}-dev", app_id);
+        let mut status = self
+            .runtime
+            .get_container_status(&container_name)
+            .await
+            .context("Failed to get container status")?;
+
+        // Add dev port information
+        if let Ok(Some(ports)) = self.port_manager.get_port_mapping(app_id) {
+            status.ports = ports.iter().map(|p| p.to_string()).collect();
+        }
+
+        Ok(status)
+    }
+
+    /// List all containers with dev info
+    pub async fn list_containers(&self) -> Result<Vec<ContainerStatus>> {
+        let containers = self
+            .runtime
+            .list_containers()
+            .await
+            .context("Failed to list containers")?;
+
+        // Filter to only archipelago containers and add port info
+        let mut result = Vec::new();
+        for container in containers {
+            if container.name.contains("archipelago-") {
+                // Extract app_id from container name
+                if let Some(app_id) = container.name.strip_prefix("archipelago-") {
+                    if let Some(app_id) = app_id.strip_suffix("-dev") {
+                        if let Ok(Some(ports)) = self.port_manager.get_port_mapping(app_id) {
+                            let mut container_with_ports = container.clone();
+                            container_with_ports.ports =
+                                ports.iter().map(|p| p.to_string()).collect();
+                            result.push(container_with_ports);
+                        } else {
+                            result.push(container);
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(result)
+    }
+
+    /// Get container logs
+    pub async fn get_container_logs(&self, app_id: &str, lines: u32) -> Result<Vec<String>> {
+        let container_name = format!("archipelago-{}-dev", app_id);
+        self.runtime
+            .get_container_logs(&container_name, lines)
+            .await
+            .context("Failed to get container logs")
+    }
+
+    /// Get health status
+    pub async fn get_health_status(&self, app_id: &str) -> Result<String> {
+        let status = self.get_container_status(app_id).await?;
+        match status.state {
+            archipelago_container::ContainerState::Running => Ok("healthy".to_string()),
+            archipelago_container::ContainerState::Stopped
+            | archipelago_container::ContainerState::Exited
+            | archipelago_container::ContainerState::Stopping => Ok("unhealthy".to_string()),
+            archipelago_container::ContainerState::Created => Ok("starting".to_string()),
+            archipelago_container::ContainerState::Paused => Ok("paused".to_string()),
+            archipelago_container::ContainerState::Unknown(_) => Ok("unknown".to_string()),
+        }
+    }
+
+    /// Load a manifest for `app_id` from the dev-mode apps directory.
+    ///
+    /// Used by the trait-level `install(app_id)` entry point.
+    ///
+    /// Search order intentionally mirrors production/operator reality:
+    /// 1) `$ARCHIPELAGO_APPS_DIR` (explicit override)
+    /// 2) `/opt/archipelago/apps` (image-recipe canonical path)
+    /// 3) `/home/archipelago/Projects/archy/apps` (repo-local fallback on dev nodes)
+    /// 4) `<data_dir>/apps` (legacy dev layout)
+    async fn load_manifest_for(&self, app_id: &str) -> Result<AppManifest> {
+        let candidates = candidate_manifest_paths(app_id, &self.config.data_dir);
+        let mut last_err: Option<anyhow::Error> = None;
+
+        for path in candidates {
+            let content = match tokio::fs::read_to_string(&path).await {
+                Ok(c) => c,
+                Err(e) => {
+                    last_err = Some(e.into());
+                    continue;
+                }
+            };
+
+            let manifest: AppManifest = serde_yaml::from_str(&content)
+                .with_context(|| format!("parsing manifest {}", path.display()))?;
+            return Ok(manifest);
+        }
+
+        let msg = format!(
+            "manifest for {} not found in any search path (set ARCHIPELAGO_APPS_DIR or install /opt/archipelago/apps)",
+            app_id
+        );
+        Err(match last_err {
+            Some(e) => e.context(msg),
+            None => anyhow::anyhow!(msg),
+        })
+    }
+}
+
+fn candidate_manifest_paths(app_id: &str, data_dir: &Path) -> Vec<PathBuf> {
+    let mut roots: Vec<PathBuf> = Vec::new();
+
+    if let Ok(v) = std::env::var("ARCHIPELAGO_APPS_DIR") {
+        let v = v.trim();
+        if !v.is_empty() {
+            roots.push(PathBuf::from(v));
+        }
+    }
+
+    roots.push(PathBuf::from("/opt/archipelago/apps"));
+    roots.push(PathBuf::from("/home/archipelago/Projects/archy/apps"));
+    roots.push(data_dir.join("apps"));
+
+    let mut deduped: Vec<PathBuf> = Vec::new();
+    for root in roots {
+        if !deduped.iter().any(|p| p == &root) {
+            deduped.push(root);
+        }
+    }
+
+    deduped
+        .into_iter()
+        .map(|root| root.join(app_id).join("manifest.yml"))
+        .collect()
+}
+
+// ---------------------------------------------------------------------------
+// Trait impl (Step 4): expose the shared ContainerOrchestrator surface.
+// Forwards to the inherent methods, which internally apply the `-dev` suffix
+// and the port offset. The trait keeps the RPC layer mode-agnostic; Dev's
+// install_container (manifest_path-based) stays as an inherent method for the
+// ad-hoc dev-mode RPC and is not exposed on the trait.
+// ---------------------------------------------------------------------------
+
+#[async_trait]
+impl ContainerOrchestrator for DevContainerOrchestrator {
+    async fn install(&self, app_id: &str) -> Result<String> {
+        let manifest = self.load_manifest_for(app_id).await?;
+        let name = self.install_container(&manifest, "").await?;
+        Ok(name)
+    }
+
+    async fn start(&self, app_id: &str) -> Result<()> {
+        self.start_container(app_id).await
+    }
+
+    async fn stop(&self, app_id: &str) -> Result<()> {
+        self.stop_container(app_id).await
+    }
+
+    async fn restart(&self, app_id: &str) -> Result<()> {
+        let _ = self.stop_container(app_id).await;
+        self.start_container(app_id).await
+    }
+
+    async fn remove(&self, app_id: &str, preserve_data: bool) -> Result<()> {
+        self.remove_container(app_id, preserve_data).await
+    }
+
+    async fn upgrade(&self, app_id: &str) -> Result<()> {
+        // Dev upgrade: stop, remove (preserving data), re-install from the loaded manifest.
+        let _ = self.stop_container(app_id).await;
+        let _ = self.remove_container(app_id, true).await;
+        let manifest = self.load_manifest_for(app_id).await?;
+        self.install_container(&manifest, "").await?;
+        self.start_container(app_id).await
+    }
+
+    async fn status(&self, app_id: &str) -> Result<ContainerStatus> {
+        self.get_container_status(app_id).await
+    }
+
+    async fn list(&self) -> Result<Vec<ContainerStatus>> {
+        self.list_containers().await
+    }
+
+    async fn logs(&self, app_id: &str, lines: u32) -> Result<Vec<String>> {
+        self.get_container_logs(app_id, lines).await
+    }
+
+    async fn health(&self, app_id: &str) -> Result<String> {
+        self.get_health_status(app_id).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::candidate_manifest_paths;
+    use std::path::PathBuf;
+
+    #[test]
+    fn candidate_manifest_paths_include_expected_fallbacks() {
+        let app_id = "bitcoin-ui";
+        let paths = candidate_manifest_paths(app_id, &PathBuf::from("/var/lib/archipelago"));
+        let as_strings: Vec<String> = paths
+            .iter()
+            .map(|p| p.to_string_lossy().into_owned())
+            .collect();
+
+        assert!(as_strings
+            .iter()
+            .any(|p| p == "/opt/archipelago/apps/bitcoin-ui/manifest.yml"));
+        assert!(as_strings
+            .iter()
+            .any(|p| p == "/home/archipelago/Projects/archy/apps/bitcoin-ui/manifest.yml"));
+        assert!(as_strings
+            .iter()
+            .any(|p| p == "/var/lib/archipelago/apps/bitcoin-ui/manifest.yml"));
+    }
+}

core/archipelago/src/container/docker_packages.rs

@@ -0,0 +1,905 @@
+// Docker Package Scanner
+// Scans docker-compose containers and converts them to package data
+
+use anyhow::Result;
+use archipelago_container::{
+    ContainerRuntime as ContainerRuntimeTrait, ContainerState, PodmanClient,
+};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tracing::{debug, info};
+
+use super::image_versions;
+use crate::data_model::{
+    Description, InstalledPackageDataEntry, InterfaceAddress, Interfaces, MainInterface, Manifest,
+    PackageDataEntry, PackageState, ServiceStatus, StaticFiles,
+};
+
+pub struct DockerPackageScanner {
+    runtime: Arc<dyn ContainerRuntimeTrait>,
+}
+
+impl DockerPackageScanner {
+    pub fn new(runtime: Arc<dyn ContainerRuntimeTrait>) -> Self {
+        Self { runtime }
+    }
+
+    /// Scan Docker containers and convert to package data
+    pub async fn scan_containers(&self) -> Result<HashMap<String, PackageDataEntry>> {
+        let containers = self.runtime.list_containers().await?;
+
+        debug!("Found {} containers", containers.len());
+
+        let mut packages = HashMap::new();
+
+        // Backend services that should not appear as apps
+        let excluded_services = [
+            "btcpay-db",
+            "nbxplorer",
+            "mempool-db",
+            "mempool-api",
+            "immich_postgres",
+            "immich_redis",
+            "endurain-db",
+            "nextcloud-db",
+            "indeedhub-api",
+            "indeedhub-ffmpeg",
+            "indeedhub-postgres",
+            "indeedhub-redis",
+            "indeedhub-minio",
+            "indeedhub-relay",
+            "indeedhub-build_api_1",
+            "indeedhub-build_postgres_1",
+            "indeedhub-build_redis_1",
+            "indeedhub-build_minio_1",
+            "indeedhub-build_minio-init_1",
+            "indeedhub-build_relay_1",
+            "indeedhub-build_ffmpeg-worker_1",
+            "netbird-server",
+            "netbird-dashboard",
+            "pine-whisper",
+            "pine-piper",
+            "pine-openwakeword",
+            "buildx_buildkit_default",
+        ];
+
+        // First pass: collect running UI containers. Custom UI-backed apps must
+        // not advertise a launch URL unless their companion is actually alive.
+        let mut ui_containers: HashMap<String, String> = HashMap::new();
+        for container in &containers {
+            if container.name.ends_with("-ui") {
+                if !matches!(container.state, ContainerState::Running) {
+                    continue;
+                }
+                // Map fedimint-ui -> fedimint, lnd-ui -> lnd (normalize archy- prefix for lookup)
+                let parent_app = container
+                    .name
+                    .strip_suffix("-ui")
+                    .unwrap_or(&container.name);
+                let canonical_id = parent_app
+                    .strip_prefix("archy-")
+                    .unwrap_or(parent_app)
+                    .to_string();
+                let ui_address = extract_lan_address(&container.ports)
+                    .or_else(|| companion_lan_address(&canonical_id));
+                if let Some(ui_address) = ui_address {
+                    ui_containers.insert(canonical_id, ui_address);
+                }
+            }
+        }
+
+        debug!("Found {} UI containers", ui_containers.len());
+
+        for container in containers {
+            // Extract app ID from container name
+            // Support both archy-* containers (docker-compose) and plain names (manual)
+            let app_id = if container.name.starts_with("archy-") {
+                container
+                    .name
+                    .strip_prefix("archy-")
+                    .unwrap_or(&container.name)
+                    .to_string()
+            } else {
+                // Use the container name as-is for manually started containers
+                container.name.clone()
+            };
+
+            // Normalize multi-container app IDs to their canonical names
+            let app_id = match app_id.as_str() {
+                "immich_server" => "immich".to_string(),
+                _ => app_id,
+            };
+
+            // Skip backend services (databases, APIs, etc.)
+            if excluded_services.contains(&app_id.as_str()) {
+                debug!("Skipping backend service: {}", app_id);
+                continue;
+            }
+
+            if is_transient_podman_helper(&app_id, &container.ports) {
+                debug!("Skipping transient Podman helper container: {}", app_id);
+                continue;
+            }
+
+            // Skip podman-compose infrastructure containers (e.g. indeedhub-build_api_1)
+            // These have the project prefix pattern: {project}_{service}_{instance}
+            if app_id.starts_with("indeedhub-build_") {
+                debug!("Skipping IndeedHub compose service: {}", app_id);
+                continue;
+            }
+
+            if app_id.starts_with("buildx_buildkit") {
+                debug!("Skipping BuildKit helper container: {}", app_id);
+                continue;
+            }
+
+            // Skip UI containers (they're merged with their parent apps)
+            if app_id.ends_with("-ui") {
+                debug!("Skipping UI container: {}", app_id);
+                continue;
+            }
+
+            // Get metadata for this app
+            let metadata = get_app_metadata(&app_id);
+
+            // Resolve UI address: separate UI containers > static map > dynamic ports
+            let lan_address = if app_id == "netbird" {
+                reachable_lan_address(&app_id, netbird_configured_launch_url().await).await
+            } else if let Some(ui_address) = ui_containers.get(&app_id) {
+                // Apps with separate UI containers (e.g. archy-bitcoin-ui, archy-lnd-ui)
+                debug!("Using UI container for {}: {}", app_id, ui_address);
+                reachable_lan_address(&app_id, Some(ui_address.clone())).await
+            } else {
+                // Prefer the known web UI port over arbitrary first binding
+                // (for example Gitea exposes SSH on 2222 before web on 3001).
+                let candidate = if uses_allocated_launch_port(&app_id) {
+                    extract_lan_address(&container.ports)
+                        .or_else(|| PodmanClient::lan_address_for(&app_id))
+                } else {
+                    PodmanClient::lan_address_for(&app_id)
+                        .or_else(|| extract_lan_address(&container.ports))
+                };
+                reachable_lan_address(&app_id, candidate).await
+            };
+
+            debug!(
+                "Container {}: ports={:?}, lan_address={:?}",
+                app_id, container.ports, lan_address
+            );
+
+            // Convert container state to package/service state
+            let (package_state, service_status) = convert_state(&container.state);
+
+            let tor_address = read_tor_address(&app_id).await;
+
+            // Extract actual version from container image tag
+            let running_version = image_versions::extract_version_from_image(&container.image);
+
+            // Decoupled from the binary OTA: prefer the remote app catalog,
+            // falling back to the image-versions.sh pin when uncovered/offline.
+            let available_update =
+                crate::container::app_catalog::available_update_for_app(&app_id, &container.image);
+
+            let package = PackageDataEntry {
+                state: package_state.clone(),
+                health: container.health.clone(),
+                exit_code: if package_state == PackageState::Exited {
+                    container.exit_code
+                } else {
+                    None
+                },
+                static_files: StaticFiles {
+                    license: "MIT".to_string(),
+                    instructions: metadata.description.clone(),
+                    icon: metadata.icon.clone(),
+                },
+                manifest: Manifest {
+                    id: app_id.clone(),
+                    title: metadata.title.clone(),
+                    version: running_version,
+                    description: Description {
+                        short: metadata.description.clone(),
+                        long: metadata.description.clone(),
+                    },
+                    release_notes: "Docker container".to_string(),
+                    license: "MIT".to_string(),
+                    wrapper_repo: metadata.repo.clone(),
+                    upstream_repo: metadata.repo.clone(),
+                    support_site: metadata.repo.clone(),
+                    marketing_site: metadata.repo.clone(),
+                    donation_url: None,
+                    author: Some("Archipelago".to_string()),
+                    website: lan_address.clone(),
+                    tier: Some(metadata.tier.to_string()),
+                    interfaces: if lan_address.is_some() || tor_address.is_some() {
+                        // `ui` is no longer implied by a published port: a
+                        // headless backend with an exposed port is a service,
+                        // not a launchable app. ui_detection consults the
+                        // manifest declaration first, then HTTP-probes the
+                        // port. Addresses stay present either way so the
+                        // Services tab can still show where a backend lives.
+                        let has_ui = super::ui_detection::has_web_ui(
+                            &app_id,
+                            lan_address.as_deref(),
+                            package_state == PackageState::Running,
+                        )
+                        .await;
+                        Some(Interfaces {
+                            main: Some(MainInterface {
+                                ui: has_ui.then(|| "true".to_string()),
+                                tor_config: tor_address.clone(),
+                                lan_config: None,
+                            }),
+                        })
+                    } else {
+                        None
+                    },
+                },
+                available_update,
+                installed: Some(InstalledPackageDataEntry {
+                    current_dependents: HashMap::new(),
+                    current_dependencies: HashMap::new(),
+                    last_backup: None,
+                    interface_addresses: if lan_address.is_some() || tor_address.is_some() {
+                        let mut addresses = HashMap::new();
+                        // Only include tor_address if we have a real v3 .onion (not placeholder)
+                        let tor = tor_address
+                            .filter(|s| is_real_onion_address(s))
+                            .unwrap_or_default();
+                        addresses.insert(
+                            "main".to_string(),
+                            InterfaceAddress {
+                                tor_address: tor,
+                                lan_address,
+                            },
+                        );
+                        addresses
+                    } else {
+                        HashMap::new()
+                    },
+                    status: service_status,
+                }),
+                install_progress: None,
+                uninstall_stage: None,
+            };
+
+            packages.insert(app_id.clone(), package);
+            info!(
+                "Detected container: {} ({})",
+                metadata.title,
+                package_state_str(&package_state)
+            );
+        }
+
+        Ok(packages)
+    }
+}
+
+struct AppMetadata {
+    title: String,
+    description: String,
+    icon: String,
+    repo: String,
+    tier: &'static str,
+}
+
+/// Get the app tier: "core", "recommended", or "optional".
+fn get_app_tier(app_id: &str) -> &'static str {
+    match app_id {
+        // Core: required for basic Bitcoin node
+        "bitcoin" | "bitcoin-core" | "bitcoin-knots" => "core",
+        "lnd" => "core",
+        "mempool" | "mempool-web" | "mempool-api" | "electrumx" | "mempool-electrs" | "electrs" => {
+            "core"
+        }
+        "btcpay" | "btcpay-server" | "btcpayserver" => "core",
+        "filebrowser" => "core",
+        // Recommended: enhanced functionality
+        "fedimint" | "fedimint-gateway" => "recommended",
+        "vaultwarden" => "recommended",
+        "uptime-kuma" => "recommended",
+        "grafana" => "recommended",
+        "searxng" => "recommended",
+        "tailscale" | "netbird" => "recommended",
+        "portainer" => "recommended",
+        // Optional: everything else
+        _ => "optional",
+    }
+}
+
+fn is_transient_podman_helper(app_id: &str, ports: &[String]) -> bool {
+    if !ports.is_empty() {
+        return false;
+    }
+
+    let Some((left, right)) = app_id.split_once('_') else {
+        return false;
+    };
+
+    !left.is_empty()
+        && !right.is_empty()
+        && left.chars().all(|c| c.is_ascii_lowercase())
+        && right.chars().all(|c| c.is_ascii_lowercase())
+}
+
+fn get_app_metadata(app_id: &str) -> AppMetadata {
+    let mut meta = match app_id {
+        "bitcoin-core" => AppMetadata {
+            title: "Bitcoin Core".to_string(),
+            description: "Reference Bitcoin node implementation".to_string(),
+            icon: "/assets/img/app-icons/bitcoin-core.svg".to_string(),
+            repo: "https://github.com/bitcoin/bitcoin".to_string(),
+            tier: "",
+        },
+        "bitcoin" | "bitcoin-knots" => AppMetadata {
+            title: "Bitcoin Knots".to_string(),
+            description: "Enhanced Bitcoin node implementation".to_string(),
+            icon: "/assets/img/app-icons/bitcoin-knots.webp".to_string(),
+            repo: "https://github.com/bitcoinknots/bitcoin".to_string(),
+            tier: "",
+        },
+        "btcpay" | "btcpay-server" | "btcpayserver" => AppMetadata {
+            title: "BTCPay Server".to_string(),
+            description: "Self-hosted Bitcoin payment processor".to_string(),
+            icon: "/assets/img/app-icons/btcpay-server.png".to_string(),
+            repo: "https://github.com/btcpayserver/btcpayserver".to_string(),
+            tier: "",
+        },
+        "homeassistant" | "home-assistant" => AppMetadata {
+            title: "Home Assistant".to_string(),
+            description: "Open source home automation platform".to_string(),
+            icon: "/assets/img/app-icons/homeassistant.png".to_string(),
+            repo: "https://github.com/home-assistant/core".to_string(),
+            tier: "",
+        },
+        "grafana" => AppMetadata {
+            title: "Grafana".to_string(),
+            description: "Analytics and monitoring platform".to_string(),
+            icon: "/assets/img/app-icons/grafana.png".to_string(),
+            repo: "https://github.com/grafana/grafana".to_string(),
+            tier: "",
+        },
+        "endurain" => AppMetadata {
+            title: "Endurain".to_string(),
+            description: "Self-hosted fitness tracking platform".to_string(),
+            icon: "/assets/img/app-icons/endurain.png".to_string(),
+            repo: "https://github.com/joaovitoriasilva/endurain".to_string(),
+            tier: "",
+        },
+        "fedimint" | "fedimintd" => AppMetadata {
+            title: "Fedimint Guardian".to_string(),
+            description: "Federated Bitcoin mint — Guardian node for federation consensus".to_string(),
+            icon: "/assets/img/app-icons/fedimint.png".to_string(),
+            repo: "https://github.com/fedimint/fedimint".to_string(),
+            tier: "",
+        },
+        "fedimint-gateway" => AppMetadata {
+            title: "Fedimint Gateway".to_string(),
+            description: "Fedimint Lightning gateway for ecash payments".to_string(),
+            icon: "/assets/img/app-icons/fedimint.png".to_string(),
+            repo: "https://github.com/fedimint/fedimint".to_string(),
+            tier: "",
+        },
+        "fedimint-clientd" | "fmcd" => AppMetadata {
+            title: "Fedimint Client".to_string(),
+            description: "Fedimint ecash client daemon (fmcd) — lets your node hold Fedimint ecash and join federations".to_string(),
+            icon: "/assets/img/app-icons/fedimint.png".to_string(),
+            repo: "https://github.com/minmoto/fmcd".to_string(),
+            tier: "",
+        },
+        "barkd" | "bark" => AppMetadata {
+            title: "Ark Wallet".to_string(),
+            description: "Ark protocol wallet daemon (barkd) — self-custodial off-chain bitcoin via an Ark server (signet)".to_string(),
+            icon: "/assets/img/app-icons/bark.png".to_string(),
+            repo: "https://gitlab.com/ark-bitcoin/bark".to_string(),
+            tier: "",
+        },
+        "morphos" | "morphos-server" => AppMetadata {
+            title: "Morphos".to_string(),
+            description: "Self-hosted file converter".to_string(),
+            icon: "/assets/img/app-icons/morphos.png".to_string(),
+            repo: "https://github.com/danvergara/morphos".to_string(),
+            tier: "",
+        },
+        "lnd" | "lightning-stack" => AppMetadata {
+            title: "LND".to_string(),
+            description: "Lightning Network Daemon".to_string(),
+            icon: "/assets/img/app-icons/lnd.png".to_string(),
+            repo: "https://github.com/lightningnetwork/lnd".to_string(),
+            tier: "",
+        },
+        "mempool" | "mempool-web" => AppMetadata {
+            title: "Mempool".to_string(),
+            description: "Bitcoin blockchain explorer".to_string(),
+            icon: "/assets/img/app-icons/mempool.webp".to_string(),
+            repo: "https://github.com/mempool/mempool".to_string(),
+            tier: "",
+        },
+        "electrumx" | "mempool-electrs" | "electrs" => AppMetadata {
+            title: "ElectrumX".to_string(),
+            description: "ElectrumX server — full Electrum protocol indexer for Bitcoin. Powers Mempool and Electrum wallets.".to_string(),
+            icon: "/assets/img/app-icons/electrumx.png".to_string(),
+            repo: "https://github.com/spesmilo/electrumx".to_string(),
+            tier: "",
+        },
+        "ollama" => AppMetadata {
+            title: "Ollama".to_string(),
+            description: "Run large language models locally".to_string(),
+            icon: "/assets/img/app-icons/ollama.png".to_string(),
+            repo: "https://github.com/ollama/ollama".to_string(),
+            tier: "",
+        },
+        "monerod" | "monero" => AppMetadata {
+            title: "Monero".to_string(),
+            description: "Private cryptocurrency full node (Monero)".to_string(),
+            icon: "/assets/img/app-icons/monero.png".to_string(),
+            repo: "https://github.com/monero-project/monero".to_string(),
+            tier: "",
+        },
+        "elementsd" | "liquid" => AppMetadata {
+            title: "Liquid Network".to_string(),
+            description: "Bitcoin sidechain for confidential transactions and faster settlements".to_string(),
+            icon: "/assets/img/app-icons/liquid.png".to_string(),
+            repo: "https://github.com/ElementsProject/elements".to_string(),
+            tier: "",
+        },
+        "searxng" => AppMetadata {
+            title: "SearXNG".to_string(),
+            description: "Privacy-respecting metasearch engine".to_string(),
+            icon: "/assets/img/app-icons/searxng.png".to_string(),
+            repo: "https://github.com/searxng/searxng".to_string(),
+            tier: "",
+        },
+        "cryptpad" => AppMetadata {
+            title: "CryptPad".to_string(),
+            description: "End-to-end encrypted document collaboration".to_string(),
+            icon: "/assets/img/app-icons/cryptpad.webp".to_string(),
+            repo: "https://github.com/cryptpad/cryptpad".to_string(),
+            tier: "",
+        },
+        "nextcloud" => AppMetadata {
+            title: "Nextcloud".to_string(),
+            description: "Self-hosted cloud storage and file management".to_string(),
+            icon: "/assets/img/app-icons/nextcloud.webp".to_string(),
+            repo: "https://github.com/nextcloud/server".to_string(),
+            tier: "",
+        },
+        "vaultwarden" => AppMetadata {
+            title: "Vaultwarden".to_string(),
+            description: "Self-hosted password manager (Bitwarden compatible)".to_string(),
+            icon: "/assets/img/app-icons/vaultwarden.webp".to_string(),
+            repo: "https://github.com/dani-garcia/vaultwarden".to_string(),
+            tier: "",
+        },
+        "jellyfin" => AppMetadata {
+            title: "Jellyfin".to_string(),
+            description: "Free media server system".to_string(),
+            icon: "/assets/img/app-icons/jellyfin.webp".to_string(),
+            repo: "https://github.com/jellyfin/jellyfin".to_string(),
+            tier: "",
+        },
+        "photoprism" => AppMetadata {
+            title: "PhotoPrism".to_string(),
+            description: "AI-powered photo management".to_string(),
+            icon: "/assets/img/app-icons/photoprism.svg".to_string(),
+            repo: "https://github.com/photoprism/photoprism".to_string(),
+            tier: "",
+        },
+        "immich" | "immich_server" => AppMetadata {
+            title: "Immich".to_string(),
+            description: "High-performance self-hosted photo and video backup".to_string(),
+            icon: "/assets/img/app-icons/immich.png".to_string(),
+            repo: "https://github.com/immich-app/immich".to_string(),
+            tier: "",
+        },
+        "filebrowser" => AppMetadata {
+            title: "File Browser".to_string(),
+            description: "Web-based file manager".to_string(),
+            icon: "/assets/img/app-icons/file-browser.webp".to_string(),
+            repo: "https://github.com/filebrowser/filebrowser".to_string(),
+            tier: "",
+        },
+        "nginx-proxy-manager" => AppMetadata {
+            title: "Nginx Proxy Manager".to_string(),
+            description: "Easy proxy management with SSL".to_string(),
+            icon: "/assets/img/app-icons/nginx.svg".to_string(),
+            repo: "https://github.com/NginxProxyManager/nginx-proxy-manager".to_string(),
+            tier: "",
+        },
+        "portainer" => AppMetadata {
+            title: "Portainer".to_string(),
+            description: "Container management UI".to_string(),
+            icon: "/assets/img/app-icons/portainer.webp".to_string(),
+            repo: "https://github.com/portainer/portainer".to_string(),
+            tier: "",
+        },
+        "uptime-kuma" => AppMetadata {
+            title: "Uptime Kuma".to_string(),
+            description: "Self-hosted monitoring tool".to_string(),
+            icon: "/assets/img/app-icons/uptime-kuma.webp".to_string(),
+            repo: "https://github.com/louislam/uptime-kuma".to_string(),
+            tier: "",
+        },
+        "tailscale" => AppMetadata {
+            title: "Tailscale".to_string(),
+            description: "Zero-config VPN for secure remote access".to_string(),
+            icon: "/assets/img/app-icons/tailscale.webp".to_string(),
+            repo: "https://github.com/tailscale/tailscale".to_string(),
+            tier: "",
+        },
+        "netbird" => AppMetadata {
+            title: "NetBird".to_string(),
+            description: "Self-hosted WireGuard mesh VPN control plane and dashboard".to_string(),
+            icon: "/assets/img/app-icons/netbird.svg".to_string(),
+            repo: "https://github.com/netbirdio/netbird".to_string(),
+            tier: "",
+        },
+        "gitea" => AppMetadata {
+            title: "Gitea".to_string(),
+            description: "Self-hosted Git service with repository and package hosting".to_string(),
+            icon: "/assets/img/app-icons/gitea.svg".to_string(),
+            repo: "https://gitea.com".to_string(),
+            tier: "",
+        },
+        "indeedhub" | "indeehub" => AppMetadata {
+            title: "IndeedHub".to_string(),
+            description: "Decentralized media streaming platform".to_string(),
+            icon: "/assets/img/app-icons/indeedhub.png".to_string(),
+            repo: "https://github.com/indeedhub/indeedhub".to_string(),
+            tier: "",
+        },
+        "tor" | "archy-tor" => AppMetadata {
+            title: "Tor".to_string(),
+            description: "Anonymous overlay network for privacy".to_string(),
+            icon: "/assets/img/app-icons/tor.svg".to_string(),
+            repo: "https://gitlab.torproject.org/tpo/core/tor".to_string(),
+            tier: "",
+        },
+        "botfights" => AppMetadata {
+            title: "BotFights".to_string(),
+            description: "AI bot arena — build, train, and battle autonomous agents".to_string(),
+            icon: "/assets/img/app-icons/botfights.svg".to_string(),
+            repo: "https://botfights.net".to_string(),
+            tier: "",
+        },
+        _ => AppMetadata {
+            title: app_id.to_string(),
+            description: format!("{} application", app_id),
+            icon: "/assets/img/favico.png".to_string(),
+            repo: "#".to_string(),
+            tier: "",
+        },
+    };
+    apply_dynamic_metadata(app_id, &mut meta);
+    meta.tier = get_app_tier(app_id);
+    meta
+}
+
+fn apply_dynamic_metadata(app_id: &str, meta: &mut AppMetadata) {
+    let config_path = format!("/var/lib/archipelago/app-configs/{}.json", app_id);
+    let Ok(data) = std::fs::read_to_string(config_path) else {
+        return;
+    };
+    let Ok(cfg) = serde_json::from_str::<serde_json::Value>(&data) else {
+        return;
+    };
+    if let Some(title) = cfg
+        .get("title")
+        .and_then(|v| v.as_str())
+        .map(str::trim)
+        .filter(|s| !s.is_empty() && s.len() <= 80)
+    {
+        meta.title = title.to_string();
+    }
+    if let Some(description) = cfg
+        .get("description")
+        .and_then(|v| v.as_str())
+        .map(str::trim)
+        .filter(|s| !s.is_empty() && s.len() <= 240)
+    {
+        meta.description = description.to_string();
+    }
+}
+
+/// Map app_id to Tor hidden service directory name.
+/// "archipelago" is the main web UI (nginx port 80).
+/// Supports container names from deploy (archy-*, btcpay-server, etc.).
+fn tor_service_name(app_id: &str) -> Option<&'static str> {
+    match app_id {
+        "archipelago" => Some("archipelago"),
+        "bitcoin" | "bitcoin-knots" | "bitcoind" => Some("bitcoin"),
+        "electrumx" | "electrs" | "electrum" => Some("electrumx"),
+        "lnd" | "lnd-ui" => Some("lnd"),
+        "btcpay" | "btcpay-server" | "btcpayserver" => Some("btcpay"),
+        "mempool" | "mempool-web" | "mempool-frontend" => Some("mempool"),
+        "fedimint" | "fedimint-gateway" => Some("fedimint"),
+        "filebrowser" => Some("filebrowser"),
+        _ => None,
+    }
+}
+
+/// V3 onion addresses are 56 base32 chars + ".onion". Placeholders like "btcpay.onion" are not real.
+fn is_real_onion_address(s: &str) -> bool {
+    s.ends_with(".onion") && s.len() >= 60 && s.len() <= 70
+}
+
+/// Read real .onion address from Tor hidden service hostname file.
+/// Service name "archipelago" is for the main web UI (nginx port 80).
+/// Uses TOR_DATA_DIR env var if set, else /var/lib/archipelago/tor.
+pub async fn read_tor_address(app_id: &str) -> Option<String> {
+    let service = tor_service_name(app_id)?;
+    let base =
+        std::env::var("TOR_DATA_DIR").unwrap_or_else(|_| "/var/lib/archipelago/tor".to_string());
+
+    // Try readable hostname copy first (when system Tor owns hidden_service dirs)
+    let hostnames_path = std::path::Path::new(&base)
+        .parent()
+        .unwrap_or(std::path::Path::new("/var/lib/archipelago"))
+        .join("tor-hostnames")
+        .join(service);
+    if let Some(addr) = tokio::fs::read_to_string(&hostnames_path)
+        .await
+        .ok()
+        .map(|s| s.trim().to_string())
+        .filter(|s| s.ends_with(".onion") && !s.is_empty())
+    {
+        return Some(addr);
+    }
+
+    // Fall back to hidden_service directory
+    let path = std::path::Path::new(&base)
+        .join(format!("hidden_service_{}", service))
+        .join("hostname");
+    tokio::fs::read_to_string(&path)
+        .await
+        .ok()
+        .map(|s| s.trim().to_string())
+        .filter(|s| s.ends_with(".onion") && !s.is_empty())
+}
+
+/// Container-side ports that are essentially never a web UI, even when
+/// published alongside one — e.g. gitea publishes SSH (`2222->22`) before its
+/// web port (`3001->3000`), and podman's port list order isn't guaranteed to
+/// put the UI port first. Skipping these lets launch-URL guessing work for
+/// any future multi-port app without a per-app static override.
+const NON_HTTP_CONTAINER_PORTS: &[&str] = &["22", "21", "3306", "5432", "6379", "27017"];
+
+fn extract_lan_address(ports: &[String]) -> Option<String> {
+    let mut first_candidate = None;
+    for port_str in ports {
+        // Parse port strings like "0.0.0.0:18443->18443/tcp" or "0.0.0.0:18443-18444->18443-18444/tcp"
+        let Some(public_part) = port_str.split("->").next() else {
+            continue;
+        };
+        let Some(port_part) = public_part.split(':').nth(1) else {
+            continue;
+        };
+        // Extract just the first port if it's a range (e.g., "18443-18444" -> "18443")
+        let host_port = port_part.split('-').next().unwrap_or(port_part);
+        let candidate = format!("http://localhost:{}", host_port);
+        if first_candidate.is_none() {
+            first_candidate = Some(candidate.clone());
+        }
+
+        let container_port = port_str
+            .split("->")
+            .nth(1)
+            .and_then(|s| s.split('/').next())
+            .map(|s| s.split('-').next().unwrap_or(s));
+        if container_port.is_some_and(|p| NON_HTTP_CONTAINER_PORTS.contains(&p)) {
+            continue;
+        }
+        return Some(candidate);
+    }
+    // Nothing looked HTTP-like — fall back to whatever was published first
+    // rather than reporting no launch URL at all.
+    first_candidate
+}
+
+/// netbird's dashboard launch URL: HTTPS on 8087 (the proxy terminates TLS —
+/// the dashboard needs a secure context for OIDC PKCE, issue #15) at the node's
+/// primary host IP so it's reachable from the LAN. Manifest-driven netbird no
+/// longer writes `dashboard.env`, so this is derived from host facts (the same
+/// `{{HOST_IP}}` the orchestrator bakes into the cert/config); it falls back to
+/// the static localhost mapping when the host IP can't be read. URL shape is
+/// identical to the legacy installer's, so the existing https reachability
+/// wrapper still applies.
+async fn netbird_configured_launch_url() -> Option<String> {
+    if let Some(ip) = first_host_ip().await {
+        return Some(format!("https://{ip}:8087"));
+    }
+    PodmanClient::lan_address_for("netbird")
+}
+
+/// The node's primary host IP. Mirrors the orchestrator's `detect_host_ip`
+/// so launch URLs match the cert/config the orchestrator renders for
+/// `{{HOST_IP}}`.
+async fn first_host_ip() -> Option<String> {
+    crate::host_ip::primary_host_ipv4().await
+}
+
+async fn reachable_lan_address(app_id: &str, candidate: Option<String>) -> Option<String> {
+    let url = candidate?;
+    if !requires_reachable_launch(app_id) {
+        return Some(url);
+    }
+    let Some(port) = launch_url_port(&url) else {
+        return None;
+    };
+    if launch_port_reachable(port).await {
+        Some(url)
+    } else {
+        debug!(app_id = %app_id, port, "suppressing unreachable launch URL");
+        None
+    }
+}
+
+/// Extract the TCP port from a launch URL's authority.
+///
+/// The candidate URL can carry a path when it comes from a manifest
+/// `interfaces.main` declaration (e.g. `http://localhost:8096/`). A naive
+/// `rsplit(':')` then yields `"8096/"`, which fails to parse and silently
+/// drops a reachable launch URL. Reading digits after the final colon mirrors
+/// `port_from_url` in the RPC layer and tolerates a trailing path.
+pub(crate) fn launch_url_port(url: &str) -> Option<u16> {
+    let after_colon = url.rsplit_once(':')?.1;
+    after_colon
+        .chars()
+        .take_while(|c| c.is_ascii_digit())
+        .collect::<String>()
+        .parse::<u16>()
+        .ok()
+}
+
+async fn launch_port_reachable(port: u16) -> bool {
+    matches!(
+        tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            tokio::net::TcpStream::connect(("127.0.0.1", port)),
+        )
+        .await,
+        Ok(Ok(_))
+    )
+}
+
+fn requires_reachable_launch(app_id: &str) -> bool {
+    matches!(
+        app_id,
+        "botfights"
+            | "btcpay-server"
+            | "fedimint"
+            | "filebrowser"
+            | "grafana"
+            | "homeassistant"
+            | "home-assistant"
+            | "jellyfin"
+            | "mempool"
+            | "nginx-proxy-manager"
+            | "uptime-kuma"
+            | "gitea"
+            | "nextcloud"
+            | "portainer"
+            | "tailscale"
+            | "immich"
+            | "searxng"
+    )
+}
+
+fn companion_lan_address(app_id: &str) -> Option<String> {
+    match app_id {
+        "bitcoin" | "bitcoin-knots" | "bitcoin-core" => Some("http://localhost:8334".to_string()),
+        "electrumx" | "mempool-electrs" | "electrs" => Some("http://localhost:50002".to_string()),
+        _ => None,
+    }
+}
+
+fn uses_allocated_launch_port(app_id: &str) -> bool {
+    matches!(
+        app_id,
+        "filebrowser" | "nextcloud" | "nginx-proxy-manager" | "vaultwarden"
+    )
+}
+
+fn convert_state(container_state: &ContainerState) -> (PackageState, ServiceStatus) {
+    match container_state {
+        ContainerState::Running => (PackageState::Running, ServiceStatus::Running),
+        ContainerState::Stopping => (PackageState::Stopping, ServiceStatus::Stopped),
+        ContainerState::Stopped => (PackageState::Stopped, ServiceStatus::Stopped),
+        ContainerState::Exited => (PackageState::Exited, ServiceStatus::Stopped),
+        ContainerState::Created => (PackageState::Stopped, ServiceStatus::Stopped),
+        ContainerState::Paused => (PackageState::Stopped, ServiceStatus::Stopped),
+        ContainerState::Unknown(_) => (PackageState::Stopped, ServiceStatus::Stopped),
+    }
+}
+
+fn package_state_str(state: &PackageState) -> &str {
+    match state {
+        PackageState::Installing => "installing",
+        PackageState::Installed => "installed",
+        PackageState::Stopping => "stopping",
+        PackageState::Stopped => "stopped",
+        PackageState::Exited => "exited",
+        PackageState::Starting => "starting",
+        PackageState::Running => "running",
+        PackageState::Restarting => "restarting",
+        PackageState::CreatingBackup => "creating-backup",
+        PackageState::RestoringBackup => "restoring-backup",
+        PackageState::Removing => "removing",
+        PackageState::BackingUp => "backing-up",
+        PackageState::Updating => "updating",
+    }
+}
+
+#[cfg(test)]
+mod launch_url_port_tests {
+    use super::launch_url_port;
+
+    #[test]
+    fn parses_port_with_trailing_path() {
+        // Regression: manifest interfaces.main yields a path-suffixed URL.
+        // The old rsplit(':') parse produced "8096/" and dropped the URL.
+        assert_eq!(launch_url_port("http://localhost:8096/"), Some(8096));
+        assert_eq!(launch_url_port("http://localhost:8175/admin"), Some(8175));
+    }
+
+    #[test]
+    fn parses_bare_authority_port() {
+        assert_eq!(launch_url_port("http://localhost:8083"), Some(8083));
+    }
+
+    #[test]
+    fn rejects_url_without_port() {
+        assert_eq!(launch_url_port("http://localhost/"), None);
+    }
+}
+
+#[cfg(test)]
+mod extract_lan_address_tests {
+    use super::extract_lan_address;
+
+    #[test]
+    fn skips_ssh_port_when_web_port_is_published() {
+        // gitea: SSH published before the web port, in podman's list order.
+        let ports = vec![
+            "0.0.0.0:2222->22/tcp".to_string(),
+            "0.0.0.0:3001->3000/tcp".to_string(),
+        ];
+        assert_eq!(
+            extract_lan_address(&ports).as_deref(),
+            Some("http://localhost:3001")
+        );
+    }
+
+    #[test]
+    fn falls_back_to_first_port_when_nothing_looks_like_http() {
+        let ports = vec!["0.0.0.0:2222->22/tcp".to_string()];
+        assert_eq!(
+            extract_lan_address(&ports).as_deref(),
+            Some("http://localhost:2222")
+        );
+    }
+
+    #[test]
+    fn single_http_port_still_resolves() {
+        let ports = vec!["0.0.0.0:8096->8096/tcp".to_string()];
+        assert_eq!(
+            extract_lan_address(&ports).as_deref(),
+            Some("http://localhost:8096")
+        );
+    }
+
+    #[test]
+    fn handles_port_ranges() {
+        let ports = vec!["0.0.0.0:18443-18444->18443-18444/tcp".to_string()];
+        assert_eq!(
+            extract_lan_address(&ports).as_deref(),
+            Some("http://localhost:18443")
+        );
+    }
+
+    #[test]
+    fn no_ports_returns_none() {
+        let ports: Vec<String> = vec![];
+        assert_eq!(extract_lan_address(&ports), None);
+    }
+}

core/archipelago/src/container/filebrowser.rs

@@ -0,0 +1,154 @@
+//! filebrowser config bootstrap helper.
+//!
+//! Mirrors the legacy first-boot behavior that writes
+//! `/var/lib/archipelago/filebrowser-data/.filebrowser.json` before
+//! starting the container with `--config /data/.filebrowser.json`.
+
+use anyhow::{Context, Result};
+use std::path::PathBuf;
+use tokio::fs;
+
+use crate::update::host_sudo;
+
+pub const DEFAULT_SRV_ROOT: &str = "/var/lib/archipelago/filebrowser";
+pub const DEFAULT_DATA_DIR: &str = "/var/lib/archipelago/filebrowser-data";
+pub const DEFAULT_CONFIG_PATH: &str = "/var/lib/archipelago/filebrowser-data/.filebrowser.json";
+
+const DEFAULT_CONFIG_JSON: &str =
+    "{\"port\":80,\"baseURL\":\"\",\"address\":\"0.0.0.0\",\"database\":\"/data/filebrowser.db\",\"root\":\"/srv\",\"log\":\"stdout\"}\n";
+
+#[derive(Debug, Clone)]
+pub struct EnsurePaths {
+    pub srv_root: PathBuf,
+    pub data_dir: PathBuf,
+    pub config_path: PathBuf,
+}
+
+impl Default for EnsurePaths {
+    fn default() -> Self {
+        Self {
+            srv_root: PathBuf::from(DEFAULT_SRV_ROOT),
+            data_dir: PathBuf::from(DEFAULT_DATA_DIR),
+            config_path: PathBuf::from(DEFAULT_CONFIG_PATH),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EnsureOutcome {
+    Written,
+    Unchanged,
+}
+
+pub async fn ensure_config(paths: &EnsurePaths) -> Result<EnsureOutcome> {
+    create_dir_all_or_sudo(&paths.srv_root).await?;
+    create_dir_all_or_sudo(&paths.data_dir).await?;
+
+    for d in ["Documents", "Photos", "Music", "Downloads", "Builds"] {
+        create_dir_all_or_sudo(&paths.srv_root.join(d)).await?;
+    }
+
+    if paths.config_path.exists() {
+        return Ok(EnsureOutcome::Unchanged);
+    }
+
+    let parent = paths
+        .config_path
+        .parent()
+        .ok_or_else(|| anyhow::anyhow!("config_path has no parent directory"))?;
+    create_dir_all_or_sudo(parent).await?;
+
+    write_config_atomically(paths).await?;
+
+    Ok(EnsureOutcome::Written)
+}
+
+async fn create_dir_all_or_sudo(path: &std::path::Path) -> Result<()> {
+    match fs::create_dir_all(path).await {
+        Ok(()) => Ok(()),
+        Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
+            let path = path.to_string_lossy();
+            let status = host_sudo(&["mkdir", "-p", &path])
+                .await
+                .with_context(|| format!("creating {path} via sudo"))?;
+            if !status.success() {
+                anyhow::bail!("mkdir -p {path} via sudo exited with {status}");
+            }
+            Ok(())
+        }
+        Err(e) => Err(e).with_context(|| format!("creating {}", path.display())),
+    }
+}
+
+async fn write_config_atomically(paths: &EnsurePaths) -> Result<()> {
+    let tmp = paths.config_path.with_extension("tmp");
+    match fs::write(&tmp, DEFAULT_CONFIG_JSON).await {
+        Ok(()) => {
+            fs::rename(&tmp, &paths.config_path)
+                .await
+                .with_context(|| {
+                    format!(
+                        "renaming {} -> {}",
+                        tmp.display(),
+                        paths.config_path.display()
+                    )
+                })?;
+            Ok(())
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
+            let script = format!(
+                "set -eu\ncat > '{}' <<'FILEBROWSERCONF'\n{}FILEBROWSERCONF\n",
+                shell_quote(&paths.config_path.to_string_lossy()),
+                DEFAULT_CONFIG_JSON
+            );
+            let status = host_sudo(&["sh", "-lc", &script])
+                .await
+                .context("writing .filebrowser.json via sudo")?;
+            if !status.success() {
+                anyhow::bail!("writing .filebrowser.json via sudo exited with {status}");
+            }
+            Ok(())
+        }
+        Err(e) => Err(e).with_context(|| format!("writing tmp {}", tmp.display())),
+    }
+}
+
+fn shell_quote(s: &str) -> String {
+    s.replace('\'', "'\\''")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn ensure_config_creates_dirs_and_file() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            srv_root: tmp.path().join("filebrowser"),
+            data_dir: tmp.path().join("filebrowser-data"),
+            config_path: tmp.path().join("filebrowser-data/.filebrowser.json"),
+        };
+
+        let out = ensure_config(&paths).await.unwrap();
+        assert_eq!(out, EnsureOutcome::Written);
+        assert!(paths.config_path.exists());
+        assert!(paths.srv_root.join("Documents").exists());
+        assert!(paths.srv_root.join("Photos").exists());
+    }
+
+    #[tokio::test]
+    async fn ensure_config_is_idempotent() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            srv_root: tmp.path().join("filebrowser"),
+            data_dir: tmp.path().join("filebrowser-data"),
+            config_path: tmp.path().join("filebrowser-data/.filebrowser.json"),
+        };
+
+        let first = ensure_config(&paths).await.unwrap();
+        assert_eq!(first, EnsureOutcome::Written);
+        let second = ensure_config(&paths).await.unwrap();
+        assert_eq!(second, EnsureOutcome::Unchanged);
+    }
+}

core/archipelago/src/container/hooks.rs

@@ -0,0 +1,198 @@
+//! Manifest-driven lifecycle hook executor (Task #20).
+//!
+//! Runs an app's declarative `post_install` hooks against its **own** running
+//! container. Hooks are an allowlisted, reviewed escape hatch — NOT arbitrary
+//! host scripts:
+//!
+//! - `exec` runs *inside the container* (`podman exec`), never on the host, and
+//!   inherits the container's (already dropped) capabilities.
+//! - `copy_from_host.src` is resolved against an allowlist root, canonicalised,
+//!   and rejected on any escape; only then is it `podman cp`'d into the container.
+//! - Execution is **best-effort + idempotent**: each step is logged, a failure is
+//!   warned and the remaining steps still run, so a transient hook error never
+//!   bricks an install. Authors must make steps safe to re-run (e.g. `grep -q … ||`).
+//!
+//! See `docs/manifest-hooks-design.md`.
+
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::{bail, Result};
+use archipelago_container::{AppManifest, HookStep};
+
+/// Upper bound on a single hook command. Generous — config rewrites + nginx
+/// reloads are fast, but an image with a hung entrypoint shouldn't wedge install.
+const HOOK_TIMEOUT: Duration = Duration::from_secs(60);
+
+/// Roots a `copy_from_host.src` may resolve within. A src is joined onto each
+/// root, canonicalised, and accepted only if it stays inside that root:
+/// - the app's own data dir (`<data_dir>/<app_id>`), and
+/// - `/opt/archipelago` (covers the orchestrator's bundled `web-ui/` assets,
+///   e.g. indeedhub's `web-ui/nostr-provider.js`).
+fn allowlist_roots(app_id: &str, data_dir: &Path) -> Vec<PathBuf> {
+    vec![data_dir.join(app_id), PathBuf::from("/opt/archipelago")]
+}
+
+/// Resolve a hook copy source against the allowlist. Returns the canonical
+/// absolute path iff it exists and lies within an allowlist root. Defence in
+/// depth: `AppManifest::validate` already rejects absolute / `..` srcs, but we
+/// re-check here and canonicalise so a symlink inside a root can't escape it.
+fn resolve_copy_src(src: &str, app_id: &str, data_dir: &Path) -> Result<PathBuf> {
+    if src.is_empty() || src.starts_with('/') || src.contains("..") {
+        bail!("hook copy src '{src}' is not an allowlisted relative path");
+    }
+    for root in allowlist_roots(app_id, data_dir) {
+        let Ok(root_canon) = root.canonicalize() else {
+            continue;
+        };
+        let Ok(canon) = root.join(src).canonicalize() else {
+            continue;
+        };
+        if canon.starts_with(&root_canon) {
+            return Ok(canon);
+        }
+    }
+    bail!("hook copy src '{src}' did not resolve inside an allowlist root")
+}
+
+/// Run an app's declarative `post_install` hooks against its running container.
+/// Best-effort: never returns an error — a failed step is warned and skipped.
+/// Called from the install path after the container is created + running, and
+/// only when a fresh container was created (see `install_fresh`).
+pub async fn run_post_install(manifest: &AppManifest, container_name: &str, data_dir: &Path) {
+    let steps = &manifest.app.hooks.post_install;
+    if steps.is_empty() {
+        return;
+    }
+    let app_id = &manifest.app.id;
+    tracing::info!(
+        app_id = %app_id,
+        container = %container_name,
+        steps = steps.len(),
+        "running manifest post_install hooks"
+    );
+    for (i, step) in steps.iter().enumerate() {
+        match run_step(step, container_name, app_id, data_dir).await {
+            Ok(()) => tracing::debug!(app_id = %app_id, step = i, "post_install hook step ok"),
+            Err(err) => tracing::warn!(
+                app_id = %app_id,
+                container = %container_name,
+                step = i,
+                error = %err,
+                "post_install hook step failed (continuing best-effort)"
+            ),
+        }
+    }
+}
+
+async fn run_step(step: &HookStep, container: &str, app_id: &str, data_dir: &Path) -> Result<()> {
+    match step {
+        HookStep::Exec { exec } => {
+            let mut args: Vec<&str> = Vec::with_capacity(exec.len() + 2);
+            args.push("exec");
+            args.push(container);
+            args.extend(exec.iter().map(String::as_str));
+            // `exec` spawns a process INSIDE the container's cgroup. When the
+            // container was started by archipelago.service, that cgroup is under
+            // the service's slice and a bare `podman exec` from the service can't
+            // write its `cgroup.procs` ("crun: ... Permission denied / OCI
+            // permission denied"). Run it in a transient user scope (its own
+            // delegated cgroup) — mirrors `podman_user_scope` for pasta starts.
+            run_podman(&args, /* scoped */ true).await
+        }
+        HookStep::CopyFromHost { copy_from_host } => {
+            let abs = resolve_copy_src(&copy_from_host.src, app_id, data_dir)?;
+            let abs = abs.to_string_lossy().into_owned();
+            let dest = format!("{container}:{}", copy_from_host.dest);
+            // `cp` is a host-side copy (no in-container process), so no scope needed.
+            run_podman(&["cp", &abs, &dest], /* scoped */ false).await
+        }
+    }
+}
+
+/// Run a podman command, optionally inside a transient systemd user scope. The
+/// scope gives the invocation its own delegated cgroup so `podman exec` can
+/// place its child process — without it, an exec launched from the service's
+/// own cgroup is denied write to the container's `cgroup.procs`.
+async fn run_podman(args: &[&str], scoped: bool) -> Result<()> {
+    let rendered = args.join(" ");
+    let mut cmd = if scoped {
+        let mut c = tokio::process::Command::new("systemd-run");
+        c.args(["--user", "--scope", "--quiet", "--collect", "podman"]);
+        c.args(args);
+        c
+    } else {
+        let mut c = tokio::process::Command::new("podman");
+        c.args(args);
+        c
+    };
+    let out = tokio::time::timeout(HOOK_TIMEOUT, cmd.output())
+        .await
+        .map_err(|_| anyhow::anyhow!("podman {rendered} timed out after {:?}", HOOK_TIMEOUT))?
+        .map_err(|e| anyhow::anyhow!("podman {rendered}: {e}"))?;
+
+    if !out.status.success() {
+        bail!(
+            "podman {rendered} exited {}: {}",
+            out.status,
+            String::from_utf8_lossy(&out.stderr).trim()
+        );
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolve_copy_src_accepts_file_in_app_data_dir() {
+        let tmp = tempfile::tempdir().unwrap();
+        let data_dir = tmp.path();
+        let app_dir = data_dir.join("myapp/web-ui");
+        std::fs::create_dir_all(&app_dir).unwrap();
+        std::fs::write(app_dir.join("provider.js"), b"x").unwrap();
+
+        let got = resolve_copy_src("web-ui/provider.js", "myapp", data_dir).unwrap();
+        assert!(got.ends_with("myapp/web-ui/provider.js"));
+        assert!(got.is_absolute());
+    }
+
+    #[test]
+    fn resolve_copy_src_rejects_absolute() {
+        let tmp = tempfile::tempdir().unwrap();
+        assert!(resolve_copy_src("/etc/passwd", "myapp", tmp.path()).is_err());
+    }
+
+    #[test]
+    fn resolve_copy_src_rejects_traversal() {
+        let tmp = tempfile::tempdir().unwrap();
+        assert!(resolve_copy_src("web-ui/../../etc/shadow", "myapp", tmp.path()).is_err());
+    }
+
+    #[test]
+    fn resolve_copy_src_rejects_missing_file() {
+        // Inside the allowlist shape but the file doesn't exist → canonicalize fails.
+        let tmp = tempfile::tempdir().unwrap();
+        std::fs::create_dir_all(tmp.path().join("myapp")).unwrap();
+        assert!(resolve_copy_src("nope.js", "myapp", tmp.path()).is_err());
+    }
+
+    #[test]
+    fn resolve_copy_src_rejects_symlink_escape() {
+        // A symlink inside the app dir pointing outside it must be rejected by
+        // the post-canonicalisation prefix check.
+        let tmp = tempfile::tempdir().unwrap();
+        let app_dir = tmp.path().join("myapp");
+        std::fs::create_dir_all(&app_dir).unwrap();
+        let secret = tmp.path().join("secret.txt");
+        std::fs::write(&secret, b"s").unwrap();
+        let link = app_dir.join("link.js");
+        if std::os::unix::fs::symlink(&secret, &link).is_ok() {
+            // `secret.txt` lives in the tmp root, NOT under <data_dir>/myapp, so
+            // the canonical target escapes the app-data root. It also isn't under
+            // /opt/archipelago. Must be rejected.
+            assert!(resolve_copy_src("link.js", "myapp", tmp.path()).is_err());
+        }
+    }
+}

core/archipelago/src/container/image_policy.rs

@@ -0,0 +1,114 @@
+//! Trusted-registry policy for container image references — the single
+//! source of truth. The RPC boundary (`api::rpc::package::config`) and the
+//! orchestrator's pull sites both validate against this, so a catalog- or
+//! manifest-supplied ref can't reach `pull_image` unchecked (§A of the
+//! 1.8.0 hardening plan).
+
+/// The registry's previous address, before it moved behind a domain.
+///
+/// TRANSITIONAL — remove once the app catalog has been regenerated and
+/// re-signed against `source.archipelago-foundation.org`. The catalog is a
+/// signed artifact, so its image refs cannot be rewritten in place without
+/// invalidating the signature; until the signing ceremony runs, deployed
+/// nodes still resolve every app through a catalog that names this host.
+/// Dropping it from the trusted list before then makes each catalog-driven
+/// install fail with "not from a trusted registry".
+pub const LEGACY_REGISTRY_HOST: &str = "146.59.87.168:3000";
+
+/// Registries images may be pulled from with an explicit host part.
+/// (git.tx1138.com was removed 2026-07-10: the host is retired and must
+/// never be pulled through again.)
+pub const TRUSTED_REGISTRIES: &[&str] = &[
+    "docker.io",
+    "ghcr.io",
+    "localhost",
+    "source.archipelago-foundation.org",
+    LEGACY_REGISTRY_HOST,
+];
+
+/// Validate a container image reference.
+///
+/// Accepts:
+///   * refs whose explicit registry host is on [`TRUSTED_REGISTRIES`]
+///     (`docker.io/grafana/grafana`, `source.archipelago-foundation.org/archy/x:1`), and
+///   * registry-less Docker Hub shorthand (`nginx`, `grafana/grafana`) —
+///     the first segment has no `.`/`:` so it cannot name an attacker host;
+///     resolution follows the host's registries.conf search order.
+///
+/// Rejects empty/oversized refs, shell metacharacters, and any ref whose
+/// explicit registry host is not on the allowlist.
+pub fn is_valid_docker_image(image: &str) -> bool {
+    if image.is_empty() || image.len() > 256 {
+        return false;
+    }
+    // Reject shell metacharacters
+    let dangerous_chars = [
+        '&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r', ' ', '\t',
+    ];
+    if image.chars().any(|c| dangerous_chars.contains(&c)) {
+        return false;
+    }
+    let first_segment = match image.split('/').next() {
+        Some(r) if !r.is_empty() => r,
+        _ => return false,
+    };
+    if TRUSTED_REGISTRIES.contains(&first_segment) {
+        return true;
+    }
+    // No dot/colon in the first segment ⇒ it's a Docker Hub namespace or a
+    // bare repo name, not a registry host — allowed. Anything that *looks*
+    // like a host (has a dot or port) but isn't allowlisted is rejected.
+    !first_segment.contains('.') && !first_segment.contains(':')
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn accepts_trusted_registries() {
+        for img in [
+            "docker.io/library/nginx:1.25",
+            "ghcr.io/owner/app:latest",
+            "localhost/archy-dev:1",
+            "source.archipelago-foundation.org/archy/bitcoin-knots:28.1",
+        ] {
+            assert!(is_valid_docker_image(img), "{img} should be accepted");
+        }
+    }
+
+    #[test]
+    fn rejects_retired_tx1138_registry() {
+        // Retired 2026-07-10 — refs through the dead host must be refused
+        // at the pull site, not time out against it.
+        assert!(!is_valid_docker_image("git.tx1138.com/lfg2025/x:2"));
+    }
+
+    #[test]
+    fn accepts_docker_hub_shorthand() {
+        for img in ["nginx", "grafana/grafana:11.2.0", "lightninglabs/lnd:v0.18"] {
+            assert!(is_valid_docker_image(img), "{img} should be accepted");
+        }
+    }
+
+    #[test]
+    fn rejects_untrusted_registry_hosts() {
+        for img in [
+            "evil.com/backdoor:latest",
+            "203.0.113.7:5000/x",
+            "registry.gitlab.com/x/y",
+            "quay.io/x/y",
+        ] {
+            assert!(!is_valid_docker_image(img), "{img} should be rejected");
+        }
+    }
+
+    #[test]
+    fn rejects_malformed_refs() {
+        assert!(!is_valid_docker_image(""));
+        assert!(!is_valid_docker_image(&"a".repeat(257)));
+        assert!(!is_valid_docker_image("docker.io/x; rm -rf /"));
+        assert!(!is_valid_docker_image("docker.io/$(curl evil)"));
+        assert!(!is_valid_docker_image("/leading-slash"));
+    }
+}

core/archipelago/src/container/image_versions.rs

@@ -0,0 +1,532 @@
+//! Parser for image-versions.sh — single source of truth for pinned container images.
+//!
+//! Reads the deployed file at /opt/archipelago/scripts/image-versions.sh (the canonical
+//! location installed by the image-recipe) with fallbacks for older layouts and the
+//! repo-local scripts/image-versions.sh for development runs from the repo root.
+
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::Mutex;
+use std::time::SystemTime;
+use tracing::debug;
+
+/// Cached parse result, invalidated when file mtime changes.
+static CACHE: Mutex<Option<CacheEntry>> = Mutex::new(None);
+
+struct CacheEntry {
+    mtime: SystemTime,
+    images: HashMap<String, String>,
+}
+
+/// File search order — canonical production path first, older layout second,
+/// repo-local for dev last. The canonical deployed path is
+/// /opt/archipelago/scripts/image-versions.sh; earlier builds put it directly
+/// in /opt/archipelago/, so that path is kept as a fallback for not-yet-updated
+/// nodes. The repo-relative entry matches `cargo run` from the repo root.
+const PATHS: &[&str] = &[
+    "/opt/archipelago/scripts/image-versions.sh",
+    "/opt/archipelago/image-versions.sh",
+    "/home/archipelago/Projects/archy/scripts/image-versions.sh",
+    "scripts/image-versions.sh",
+];
+
+/// Parse image-versions.sh and return map of variable names to full image refs.
+/// Result is cached and only re-parsed when the file's mtime changes.
+fn load_image_versions() -> HashMap<String, String> {
+    let (path, mtime) = match find_file() {
+        Some(v) => v,
+        None => {
+            debug!("image-versions.sh not found in any search path");
+            return HashMap::new();
+        }
+    };
+
+    // Check cache
+    {
+        let cache = CACHE.lock().unwrap();
+        if let Some(ref entry) = *cache {
+            if entry.mtime == mtime {
+                return entry.images.clone();
+            }
+        }
+    }
+
+    // Parse fresh
+    let content = match std::fs::read_to_string(&path) {
+        Ok(c) => c,
+        Err(e) => {
+            debug!("Failed to read {}: {}", path, e);
+            return HashMap::new();
+        }
+    };
+
+    let images = parse_image_versions(&content);
+    debug!("Parsed {} image versions from {}", images.len(), path);
+
+    // Update cache
+    {
+        let mut cache = CACHE.lock().unwrap();
+        *cache = Some(CacheEntry {
+            mtime,
+            images: images.clone(),
+        });
+    }
+
+    images
+}
+
+fn find_file() -> Option<(String, SystemTime)> {
+    for p in PATHS {
+        let path = Path::new(p);
+        if let Ok(meta) = path.metadata() {
+            if let Ok(mtime) = meta.modified() {
+                return Some((p.to_string(), mtime));
+            }
+        }
+    }
+    None
+}
+
+/// Parse shell variable assignments, expanding $ARCHY_REGISTRY.
+fn parse_image_versions(content: &str) -> HashMap<String, String> {
+    let mut vars = HashMap::new();
+    let mut registry = String::new();
+
+    for line in content.lines() {
+        let line = line.trim();
+        if line.is_empty() || line.starts_with('#') {
+            continue;
+        }
+
+        // Match VAR="value" or VAR=value
+        if let Some((key, val)) = parse_assignment(line) {
+            let expanded = val.replace("$ARCHY_REGISTRY", &registry);
+            if key == "ARCHY_REGISTRY" {
+                registry = expanded.clone();
+            }
+            vars.insert(key.to_string(), expanded);
+        }
+    }
+
+    // Keep only *_IMAGE entries whose value looks like a container image
+    // reference (contains a `:` tag separator and at least one `/` path
+    // component). Rejects placeholder values like "something" so a
+    // hand-edit typo in image-versions.sh never gets treated as an image.
+    vars.retain(|k, v| k.ends_with("_IMAGE") && v.contains(':') && v.contains('/'));
+    vars
+}
+
+fn parse_assignment(line: &str) -> Option<(&str, &str)> {
+    let eq = line.find('=')?;
+    let key = &line[..eq];
+
+    // Validate key is a shell variable name
+    if !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
+        return None;
+    }
+
+    let val = &line[eq + 1..];
+    // Strip surrounding quotes
+    let val = val
+        .strip_prefix('"')
+        .and_then(|v| v.strip_suffix('"'))
+        .unwrap_or(val);
+
+    Some((key, val))
+}
+
+/// Map app ID (as seen by the container scanner) to image variable name.
+fn image_var_for_app(app_id: &str) -> Option<&'static str> {
+    match app_id {
+        // Bitcoin stack
+        "bitcoin-knots" | "bitcoin" | "bitcoin-core" => Some("BITCOIN_KNOTS_IMAGE"),
+        "lnd" => Some("LND_IMAGE"),
+        "electrumx" => Some("ELECTRUMX_IMAGE"),
+        "electrs" | "mempool-electrs" => Some("ELECTRUMX_IMAGE"),
+        "bitcoin-ui" | "archy-bitcoin-ui" => Some("BITCOIN_UI_IMAGE"),
+        "lnd-ui" | "archy-lnd-ui" => Some("LND_UI_IMAGE"),
+        "electrs-ui" | "archy-electrs-ui" => Some("ELECTRS_UI_IMAGE"),
+
+        // Mempool stack (primary = web)
+        "mempool" | "mempool-web" | "archy-mempool-web" => Some("MEMPOOL_WEB_IMAGE"),
+
+        // BTCPay stack (primary = server)
+        "btcpay" | "btcpay-server" | "btcpayserver" | "archy-btcpay-ui" => Some("BTCPAY_IMAGE"),
+
+        // Apps
+        "homeassistant" | "home-assistant" => Some("HOMEASSISTANT_IMAGE"),
+        "grafana" => Some("GRAFANA_IMAGE"),
+        "uptime-kuma" => Some("UPTIME_KUMA_IMAGE"),
+        "jellyfin" => Some("JELLYFIN_IMAGE"),
+        "photoprism" => Some("PHOTOPRISM_IMAGE"),
+        "ollama" => Some("OLLAMA_IMAGE"),
+        "vaultwarden" => Some("VAULTWARDEN_IMAGE"),
+        "nextcloud" => Some("NEXTCLOUD_IMAGE"),
+        "searxng" => Some("SEARXNG_IMAGE"),
+        "cryptpad" => Some("CRYPTPAD_IMAGE"),
+        "filebrowser" => Some("FILEBROWSER_IMAGE"),
+        "nginx-proxy-manager" => Some("NPM_IMAGE"),
+        "portainer" => Some("PORTAINER_IMAGE"),
+        "tailscale" => Some("TAILSCALE_IMAGE"),
+        "netbird" => Some("NETBIRD_DASHBOARD_IMAGE"),
+        "netbird-dashboard" => Some("NETBIRD_DASHBOARD_IMAGE"),
+        "netbird-server" => Some("NETBIRD_SERVER_IMAGE"),
+
+        // Fedimint
+        "fedimint" | "fedimintd" => Some("FEDIMINT_IMAGE"),
+        "fedimint-gateway" => Some("FEDIMINT_GATEWAY_IMAGE"),
+
+        // Nostr / VPN
+        "nostr-rs-relay" => Some("NOSTR_RS_RELAY_IMAGE"),
+        "nostr-vpn" => Some("NOSTR_VPN_IMAGE"),
+        "fips" => Some("FIPS_IMAGE"),
+
+        // Immich (primary = server)
+        "immich" | "immich_server" => Some("IMMICH_SERVER_IMAGE"),
+
+        // Penpot (primary = frontend)
+        "penpot" | "penpot-frontend" => Some("PENPOT_FRONTEND_IMAGE"),
+
+        // AI
+        "routstr" => Some("ROUTSTR_IMAGE"),
+
+        // Networking
+        "adguardhome" => Some("ADGUARDHOME_IMAGE"),
+        "tor" | "archy-tor" => Some("ALPINE_TOR_IMAGE"),
+
+        _ => None,
+    }
+}
+
+/// Get the full pinned image reference for an app ID.
+pub fn pinned_image_for_app(app_id: &str) -> Option<String> {
+    let var = image_var_for_app(app_id)?;
+    let images = load_image_versions();
+    images.get(var).cloned()
+}
+
+/// Return the pinned tag only when the running image is genuinely behind.
+/// Registry host changes alone are not app updates, and floating tags are not
+/// explicit versions we should advertise to users as available updates.
+pub fn available_update_for_app(app_id: &str, running_image: &str) -> Option<String> {
+    let pinned = pinned_image_for_app(app_id)?;
+    available_update_for_images(&pinned, running_image)
+}
+
+pub fn available_update_for_images(pinned: &str, running_image: &str) -> Option<String> {
+    let pinned_version = extract_version_from_image(&pinned);
+    if is_floating_tag(&pinned_version) {
+        return None;
+    }
+
+    let running_version = extract_version_from_image(running_image);
+    if pinned_version == running_version {
+        return None;
+    }
+
+    let pinned_repo = image_without_registry_or_tag(&pinned);
+    let running_repo = image_without_registry_or_tag(running_image);
+    if pinned_repo != running_repo {
+        return None;
+    }
+
+    // Never advertise a LOWER version as an update.
+    //
+    // Everything upstream of here is a version claim that can go stale: the
+    // signed catalog, a legacy catalog entry with no manifest, the
+    // image-versions.sh baseline pin. When one lags behind what a node is
+    // actually running, a bare `pinned != running` check turns that staleness
+    // into an "Update" button that rolls the node BACKWARDS — and a rollback
+    // to a version withdrawn for a vulnerability is precisely the case where
+    // that must not happen. Observed with BTCPay: 2.4.2 installed, a stale
+    // 2.3.9 pin, and the UI offering "update" to the exploited release.
+    //
+    // Only suppress when both tags parse as comparable version numbers, so
+    // apps with opaque tags (RELEASE.2024-11-07T00-52-20Z, 14-vectorchord0.4.3)
+    // keep the previous behaviour rather than silently losing updates.
+    if let (Some(p), Some(r)) = (
+        parse_version_parts(&pinned_version),
+        parse_version_parts(&running_version),
+    ) {
+        if p < r {
+            return None;
+        }
+    }
+
+    Some(pinned_version)
+}
+
+/// Numeric components of a version tag, for ordering comparisons only.
+///
+/// Accepts a leading `v` and a trailing pre-release suffix (`v0.18.4-beta`),
+/// comparing on the dotted numbers alone. Returns None when the tag is not a
+/// recognisable dotted-numeric version, which the caller treats as "cannot
+/// order these" rather than as equality.
+fn parse_version_parts(tag: &str) -> Option<Vec<u64>> {
+    let core = tag.strip_prefix('v').unwrap_or(tag);
+    // Drop a pre-release/build suffix: 0.18.4-beta -> 0.18.4
+    let core = core.split(['-', '+', '_']).next().unwrap_or(core);
+    if core.is_empty() {
+        return None;
+    }
+    let parts: Vec<&str> = core.split('.').collect();
+    let mut out = Vec::with_capacity(parts.len());
+    for part in parts {
+        // Any non-numeric component makes the whole tag unorderable.
+        out.push(part.parse::<u64>().ok()?);
+    }
+    Some(out)
+}
+
+/// Extract version tag from a full image reference.
+/// e.g. "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta" → "v0.18.4-beta"
+/// Returns "latest" if no tag or tag is empty.
+pub fn extract_version_from_image(image: &str) -> String {
+    // Split off the tag after the last colon, but only if it comes after the last slash
+    // (to avoid splitting on registry port like "registry.example.com:3000")
+    if let Some(slash_pos) = image.rfind('/') {
+        let after_slash = &image[slash_pos..];
+        if let Some(colon_pos) = after_slash.rfind(':') {
+            let tag = &after_slash[colon_pos + 1..];
+            if !tag.is_empty() {
+                return tag.to_string();
+            }
+        }
+    }
+    "latest".to_string()
+}
+
+fn is_floating_tag(tag: &str) -> bool {
+    matches!(tag, "latest" | "stable" | "release" | "main")
+}
+
+pub fn image_without_registry_or_tag(image: &str) -> &str {
+    let without_tag = strip_tag(image);
+    match without_tag.split_once('/') {
+        Some((first, rest))
+            if first.contains('.') || first.contains(':') || first == "localhost" =>
+        {
+            rest
+        }
+        _ => without_tag,
+    }
+}
+
+fn strip_tag(image: &str) -> &str {
+    if let Some(slash_pos) = image.rfind('/') {
+        let after_slash = &image[slash_pos..];
+        if let Some(colon_pos) = after_slash.rfind(':') {
+            return &image[..slash_pos + colon_pos];
+        }
+    }
+    image
+}
+
+/// Container names and their image variable names for multi-container stacks.
+/// Returns empty vec for single-container apps.
+pub fn containers_for_stack(app_id: &str) -> Vec<(&'static str, &'static str)> {
+    match app_id {
+        "mempool" | "mempool-web" => vec![
+            ("archy-mempool-db", "MARIADB_IMAGE"),
+            ("mempool-api", "MEMPOOL_BACKEND_IMAGE"),
+            ("archy-mempool-web", "MEMPOOL_WEB_IMAGE"),
+        ],
+        "btcpay" | "btcpay-server" | "btcpayserver" => vec![
+            ("archy-btcpay-db", "BTCPAY_POSTGRES_IMAGE"),
+            ("archy-nbxplorer", "NBXPLORER_IMAGE"),
+            ("btcpay-server", "BTCPAY_IMAGE"),
+        ],
+        "immich" | "immich_server" => vec![
+            ("immich_postgres", "IMMICH_POSTGRES_IMAGE"),
+            ("immich_redis", "REDIS_IMAGE"),
+            ("immich_server", "IMMICH_SERVER_IMAGE"),
+        ],
+        "penpot" | "penpot-frontend" => vec![
+            ("penpot-postgres", "PENPOT_POSTGRES_IMAGE"),
+            ("penpot-valkey", "PENPOT_VALKEY_IMAGE"),
+            ("penpot-backend", "PENPOT_BACKEND_IMAGE"),
+            ("penpot-exporter", "PENPOT_EXPORTER_IMAGE"),
+            ("penpot-frontend", "PENPOT_FRONTEND_IMAGE"),
+        ],
+        "netbird" => vec![
+            ("netbird", "NETBIRD_PROXY_IMAGE"),
+            ("netbird-dashboard", "NETBIRD_DASHBOARD_IMAGE"),
+            ("netbird-server", "NETBIRD_SERVER_IMAGE"),
+        ],
+        _ => vec![],
+    }
+}
+
+/// Get all pinned images for a stack update. Returns vec of (container_name, full_image_ref).
+pub fn pinned_images_for_stack(app_id: &str) -> Vec<(String, String)> {
+    let images = load_image_versions();
+    containers_for_stack(app_id)
+        .into_iter()
+        .filter_map(|(name, var)| images.get(var).map(|img| (name.to_string(), img.clone())))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_version() {
+        assert_eq!(
+            extract_version_from_image(
+                "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta"
+            ),
+            "v0.18.4-beta"
+        );
+        assert_eq!(
+            extract_version_from_image("source.archipelago-foundation.org/lfg2025/grafana:10.2.0"),
+            "10.2.0"
+        );
+        assert_eq!(
+            extract_version_from_image("localhost/myapp:latest"),
+            "latest"
+        );
+        assert_eq!(
+            extract_version_from_image(
+                "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest"
+            ),
+            "latest"
+        );
+    }
+
+    #[test]
+    fn strips_registry_and_tag_for_image_identity() {
+        assert_eq!(
+            image_without_registry_or_tag(
+                "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta"
+            ),
+            "lfg2025/lnd"
+        );
+        assert_eq!(
+            image_without_registry_or_tag(
+                "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta"
+            ),
+            "lfg2025/lnd"
+        );
+    }
+
+    #[test]
+    fn floating_tags_are_not_explicit_updates() {
+        assert!(is_floating_tag("latest"));
+        assert!(is_floating_tag("stable"));
+        assert!(!is_floating_tag("v0.18.4-beta"));
+    }
+
+    #[test]
+    fn available_update_ignores_registry_only_changes() {
+        assert_eq!(
+            available_update_for_images(
+                "source.archipelago-foundation.org/lfg2025/nextcloud:29",
+                "source.archipelago-foundation.org/lfg2025/nextcloud:29",
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn available_update_returns_pinned_version_for_same_repo_newer_tag() {
+        assert_eq!(
+            available_update_for_images(
+                "source.archipelago-foundation.org/lfg2025/nextcloud:29",
+                "source.archipelago-foundation.org/lfg2025/nextcloud:28",
+            ),
+            Some("29".to_string())
+        );
+    }
+
+    #[test]
+    fn test_parse_image_versions() {
+        let content = r#"
+ARCHY_REGISTRY="source.archipelago-foundation.org/lfg2025"
+LND_IMAGE="$ARCHY_REGISTRY/lnd:v0.18.4-beta"
+GRAFANA_IMAGE="$ARCHY_REGISTRY/grafana:10.2.0"
+# comment
+NOT_AN_IMAGE="something"
+"#;
+        let parsed = parse_image_versions(content);
+        assert_eq!(
+            parsed.get("LND_IMAGE"),
+            Some(&"source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta".to_string())
+        );
+        assert_eq!(
+            parsed.get("GRAFANA_IMAGE"),
+            Some(&"source.archipelago-foundation.org/lfg2025/grafana:10.2.0".to_string())
+        );
+        assert!(!parsed.contains_key("NOT_AN_IMAGE"));
+        assert!(!parsed.contains_key("ARCHY_REGISTRY"));
+    }
+
+    #[test]
+    fn test_image_var_mapping() {
+        assert_eq!(image_var_for_app("lnd"), Some("LND_IMAGE"));
+        assert_eq!(
+            image_var_for_app("bitcoin-knots"),
+            Some("BITCOIN_KNOTS_IMAGE")
+        );
+        assert_eq!(image_var_for_app("unknown-app"), None);
+    }
+
+    /// The BTCPay case that prompted the guard: 2.4.2 shipped for an actively
+    /// exploited 2FA bypass, a stale 2.3.9 pin left in a legacy catalog entry,
+    /// and the UI offering the withdrawn release as an "update".
+    #[test]
+    fn never_advertises_a_downgrade_as_an_update() {
+        let stale = "docker.io/btcpayserver/btcpayserver:2.3.9";
+        let running = "docker.io/btcpayserver/btcpayserver:2.4.2";
+        assert_eq!(available_update_for_images(stale, running), None);
+    }
+
+    #[test]
+    fn still_advertises_a_genuine_upgrade() {
+        let pinned = "docker.io/btcpayserver/btcpayserver:2.4.2";
+        let running = "docker.io/btcpayserver/btcpayserver:2.3.9";
+        assert_eq!(
+            available_update_for_images(pinned, running),
+            Some("2.4.2".to_string())
+        );
+    }
+
+    #[test]
+    fn equal_versions_offer_nothing() {
+        let same = "docker.io/btcpayserver/btcpayserver:2.4.2";
+        assert_eq!(available_update_for_images(same, same), None);
+    }
+
+    #[test]
+    fn prerelease_suffixes_compare_on_their_numbers() {
+        let older = "example.test/lfg2025/lnd:v0.18.3-beta";
+        let newer = "example.test/lfg2025/lnd:v0.18.4-beta";
+        assert_eq!(available_update_for_images(older, newer), None);
+        assert_eq!(
+            available_update_for_images(newer, older),
+            Some("v0.18.4-beta".to_string())
+        );
+    }
+
+    /// Opaque tags stay on the old behaviour: we cannot order them, so a
+    /// difference is still reported rather than silently swallowed.
+    #[test]
+    fn unorderable_tags_keep_previous_behaviour() {
+        let a = "example.test/lfg2025/minio:RELEASE.2024-11-07T00-52-20Z";
+        let b = "example.test/lfg2025/minio:RELEASE.2024-10-01T00-00-00Z";
+        assert_eq!(
+            available_update_for_images(a, b),
+            Some("RELEASE.2024-11-07T00-52-20Z".to_string())
+        );
+    }
+
+    #[test]
+    fn parse_version_parts_rejects_non_numeric() {
+        assert_eq!(parse_version_parts("2.4.2"), Some(vec![2, 4, 2]));
+        assert_eq!(parse_version_parts("v0.18.4-beta"), Some(vec![0, 18, 4]));
+        assert_eq!(parse_version_parts("28.4"), Some(vec![28, 4]));
+        assert_eq!(parse_version_parts("RELEASE.2024-11-07T00-52-20Z"), None);
+        assert_eq!(parse_version_parts("14-vectorchord0.4.3"), Some(vec![14]));
+        assert_eq!(parse_version_parts("latest"), None);
+    }
+}

core/archipelago/src/container/lnd.rs

@@ -0,0 +1,988 @@
+//! lnd config bootstrap helper.
+
+use anyhow::{Context, Result};
+use base64::Engine;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use tokio::fs;
+
+use crate::update::host_sudo;
+
+pub const DEFAULT_DATA_DIR: &str = "/var/lib/archipelago/lnd";
+pub const DEFAULT_CONF_PATH: &str = "/var/lib/archipelago/lnd/lnd.conf";
+const LND_REST_BASE_URL: &str = "https://127.0.0.1:18080";
+
+/// Per-node LND wallet password file (random, 0600). Replaces the old
+/// fleet-wide hardcoded constant: each node's wallet password is now unique,
+/// high-entropy, and recorded here so the unattended boot path can auto-unlock.
+const WALLET_PASSWORD_SECRET: &str = "/var/lib/archipelago/secrets/lnd-wallet-password";
+
+/// Legacy fleet-wide wallet password (builds that hardcoded it). Kept ONLY as an
+/// unlock fallback so wallets created by those builds still open; new wallets
+/// never use it, and the login-path migration rotates away from it.
+const LEGACY_WALLET_PASSWORD: &str = "hellohello";
+
+/// How many one-second passes `unlock_existing_wallet_via_rest` will make while
+/// LND's unlocker is still not listening (~10 minutes). See the comment at the
+/// retry loop for why this is measured in minutes rather than seconds.
+const UNLOCK_NOT_READY_ATTEMPTS: u32 = 600;
+
+#[derive(Debug, Clone)]
+pub struct EnsurePaths {
+    pub data_dir: PathBuf,
+    pub conf_path: PathBuf,
+}
+
+impl Default for EnsurePaths {
+    fn default() -> Self {
+        Self {
+            data_dir: PathBuf::from(DEFAULT_DATA_DIR),
+            conf_path: PathBuf::from(DEFAULT_CONF_PATH),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EnsureOutcome {
+    Written,
+    Unchanged,
+}
+
+pub async fn ensure_config(
+    paths: &EnsurePaths,
+    rpc_pass: &str,
+    bitcoin_host: &str,
+) -> Result<EnsureOutcome> {
+    fs::create_dir_all(&paths.data_dir)
+        .await
+        .with_context(|| format!("creating {}", paths.data_dir.display()))?;
+
+    if paths.conf_path.exists() {
+        let existing = fs::read_to_string(&paths.conf_path)
+            .await
+            .with_context(|| format!("reading {}", paths.conf_path.display()))?;
+        if has_required_lnd_flags(&existing, rpc_pass, bitcoin_host) {
+            return Ok(EnsureOutcome::Unchanged);
+        }
+    }
+
+    let conf = format!(
+        "debuglevel=info\n\
+maxpendingchannels=10\n\
+alias=Archipelago Node\n\
+color=#f7931a\n\
+listen=0.0.0.0:9735\n\
+rpclisten=0.0.0.0:10009\n\
+restlisten=0.0.0.0:8080\n\
+bitcoin.active=true\n\
+bitcoin.mainnet=true\n\
+bitcoin.node=bitcoind\n\
+bitcoind.rpchost={bitcoin_host}:8332\n\
+bitcoind.rpcuser=archipelago\n\
+bitcoind.rpcpass={rpc_pass}\n\
+bitcoind.rpcpolling=true\n\
+bitcoind.estimatemode=ECONOMICAL\n"
+    );
+
+    write_config_atomically(paths, &conf).await?;
+
+    Ok(EnsureOutcome::Written)
+}
+
+pub async fn ensure_wallet_initialized() -> Result<()> {
+    let admin_macaroon = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
+    let wallet_db = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/wallet.db";
+    if file_exists_as_root(wallet_db).await {
+        if file_exists_as_root(admin_macaroon).await && lnd_getinfo_ready(admin_macaroon).await {
+            return Ok(());
+        }
+        match unlock_existing_wallet().await? {
+            true => {
+                wait_for_admin_macaroon(admin_macaroon).await?;
+                return Ok(());
+            }
+            false => {
+                // Every candidate password was actively rejected: this wallet was
+                // created with a password this node no longer has, so it can never
+                // auto-unlock unattended. Alpha nodes hold no real funds and a wallet
+                // locked with an unknown password is already inaccessible, so wipe +
+                // recreate it on the per-node secret to self-heal at boot.
+                recreate_wallet_destructively().await?;
+                wait_for_admin_macaroon(admin_macaroon).await?;
+                return Ok(());
+            }
+        }
+    }
+
+    init_wallet_via_rest().await?;
+    wait_for_admin_macaroon(admin_macaroon).await
+}
+
+/// LND data subdirectories holding wallet + channel + graph state. Removing them
+/// returns LND to a NON_EXISTING wallet state. Funds-bearing data lives here too,
+/// so deletion is destructive — only done once the wallet is already unrecoverable.
+const LND_STATE_DIRS: &[&str] = &[
+    "/var/lib/archipelago/lnd/data/chain",
+    "/var/lib/archipelago/lnd/data/graph",
+];
+
+/// Podman container name for the core LND app (see `compute_container_name`:
+/// non-UI core apps keep their bare id). LND runs as a plain bridge-network
+/// container, not a Quadlet unit, so it is restarted via `podman`, not systemctl.
+const LND_CONTAINER: &str = "lnd";
+
+/// Archipelago data dir (default; not overridden in prod). Holds the
+/// `user-stopped.json` that gates health-monitor auto-restart.
+const ARCHY_DATA_DIR: &str = "/var/lib/archipelago";
+
+/// Destroy an unrecoverable LND wallet and recreate a fresh one keyed to the
+/// per-node secret. Suppresses health-monitor auto-restart for the wipe window,
+/// stops LND, deletes its wallet/chain/graph state as root, restarts it, waits
+/// for NON_EXISTING, then inits a fresh wallet. Destructive — only called when no
+/// candidate password can open the existing wallet.
+async fn recreate_wallet_destructively() -> Result<()> {
+    tracing::warn!(
+        "[lnd] wallet is locked with an unknown password and cannot auto-unlock; \
+         wiping and recreating it on the per-node secret (DESTRUCTIVE)"
+    );
+
+    // The health monitor restarts any container it sees stopped; mark LND
+    // user-stopped so it doesn't re-launch (and re-open the wallet) mid-wipe.
+    // Always cleared below so LND auto-recovers normally afterwards.
+    let data_dir = std::path::Path::new(ARCHY_DATA_DIR);
+    crate::crash_recovery::mark_user_stopped(data_dir, LND_CONTAINER).await;
+    let result = wipe_and_reinit_wallet().await;
+    crate::crash_recovery::clear_user_stopped(data_dir, LND_CONTAINER).await;
+    result
+}
+
+async fn wipe_and_reinit_wallet() -> Result<()> {
+    podman_user_scoped(&["stop", LND_CONTAINER])
+        .await
+        .context("stopping lnd before wallet wipe")?;
+
+    for dir in LND_STATE_DIRS {
+        let status = host_sudo(&["rm", "-rf", dir])
+            .await
+            .with_context(|| format!("removing {dir}"))?;
+        if !status.success() {
+            anyhow::bail!("removing {dir} exited with {status}");
+        }
+    }
+
+    podman_user_scoped(&["start", LND_CONTAINER])
+        .await
+        .context("restarting lnd after wallet wipe")?;
+
+    wait_for_wallet_state("NON_EXISTING").await?;
+    init_wallet_via_rest().await
+}
+
+/// Run `podman <args>` inside a transient `systemd-run --user --scope`, matching
+/// how the orchestrator/health-monitor manage rootless containers (keeps the
+/// container out of the archipelago service's cgroup).
+async fn podman_user_scoped(args: &[&str]) -> Result<()> {
+    let out = tokio::process::Command::new("systemd-run")
+        .args(["--user", "--scope", "--quiet", "--collect", "podman"])
+        .args(args)
+        .output()
+        .await
+        .with_context(|| format!("systemd-run --user --scope podman {}", args.join(" ")))?;
+    if !out.status.success() {
+        anyhow::bail!(
+            "podman {} failed: {}",
+            args.join(" "),
+            String::from_utf8_lossy(&out.stderr).trim()
+        );
+    }
+    Ok(())
+}
+
+/// Poll `/v1/state` until LND reports `target`, or time out after ~120s.
+async fn wait_for_wallet_state(target: &str) -> Result<()> {
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(std::time::Duration::from_secs(5))
+        .danger_accept_invalid_certs(true)
+        .build()
+        .context("building LND REST client")?;
+    for _ in 0..120 {
+        if wallet_state(&client).await.as_deref() == Some(target) {
+            return Ok(());
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    }
+    anyhow::bail!("LND did not reach state {target} after wallet wipe")
+}
+
+async fn file_exists_as_root(path: &str) -> bool {
+    if std::path::Path::new(path).exists() {
+        return true;
+    }
+    tokio::process::Command::new("sudo")
+        .args(["test", "-f", path])
+        .status()
+        .await
+        .map(|status| status.success())
+        .unwrap_or(false)
+}
+
+async fn read_file_as_root(path: &str) -> Result<Vec<u8>> {
+    match fs::read(path).await {
+        Ok(bytes) => Ok(bytes),
+        Err(direct_err) => {
+            let out = tokio::process::Command::new("sudo")
+                .args(["cat", path])
+                .output()
+                .await
+                .with_context(|| format!("reading {path} via sudo"))?;
+            if out.status.success() {
+                Ok(out.stdout)
+            } else {
+                anyhow::bail!(
+                    "reading {path} failed (direct: {direct_err}; sudo: {})",
+                    String::from_utf8_lossy(&out.stderr).trim()
+                )
+            }
+        }
+    }
+}
+
+/// Read the per-node wallet password from the secrets file, if present.
+/// Never generates one — absence means "fall back to legacy / not set yet".
+async fn read_wallet_password() -> Option<String> {
+    let bytes = fs::read(WALLET_PASSWORD_SECRET).await.ok()?;
+    let pw = String::from_utf8_lossy(&bytes).trim().to_string();
+    (!pw.is_empty()).then_some(pw)
+}
+
+/// Return the per-node wallet password, generating and persisting a fresh
+/// 256-bit one (base64, 0600) if none exists. Use ONLY when creating a NEW
+/// wallet — calling it merely to unlock an existing wallet would record a
+/// password that doesn't match it.
+pub(crate) async fn ensure_wallet_password() -> Result<String> {
+    if let Some(pw) = read_wallet_password().await {
+        return Ok(pw);
+    }
+    use rand::RngCore;
+    let mut raw = [0u8; 32];
+    rand::rngs::OsRng.fill_bytes(&mut raw);
+    let pw = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw);
+    let path = std::path::Path::new(WALLET_PASSWORD_SECRET);
+    if let Some(dir) = path.parent() {
+        fs::create_dir_all(dir)
+            .await
+            .with_context(|| format!("creating {}", dir.display()))?;
+    }
+    fs::write(path, &pw)
+        .await
+        .with_context(|| format!("writing {WALLET_PASSWORD_SECRET}"))?;
+    use std::os::unix::fs::PermissionsExt;
+    let _ = fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).await;
+    Ok(pw)
+}
+
+/// Candidate passwords to try when unlocking an EXISTING wallet, in order: the
+/// per-node secret (current scheme) first, then the legacy constant so wallets
+/// created by older builds still open.
+async fn unlock_password_candidates() -> Vec<String> {
+    let mut v = Vec::new();
+    if let Some(pw) = read_wallet_password().await {
+        v.push(pw);
+    }
+    v.push(LEGACY_WALLET_PASSWORD.to_string());
+    v
+}
+
+/// Outcome of a single unlock attempt — lets the caller fail fast on a wrong
+/// password (no point retrying) vs keep waiting for LND to come up.
+enum UnlockAttempt {
+    Unlocked,
+    WrongPassword,
+    NotReady,
+}
+
+/// One unlock POST, no internal retry. Distinguishes "invalid passphrase"
+/// (WrongPassword — try the next candidate, don't retry) from transient
+/// not-ready / connection errors (NotReady — worth retrying).
+async fn try_unlock_once(client: &reqwest::Client, password: &str) -> UnlockAttempt {
+    let body = serde_json::json!({
+        "wallet_password": base64::engine::general_purpose::STANDARD.encode(password)
+    });
+    match client
+        .post(format!("{LND_REST_BASE_URL}/v1/unlockwallet"))
+        .json(&body)
+        .send()
+        .await
+    {
+        Ok(resp) => {
+            let status = resp.status();
+            let text = resp.text().await.unwrap_or_default();
+            if status.is_success() || text.contains("already unlocked") {
+                UnlockAttempt::Unlocked
+            } else if text.contains("invalid passphrase") {
+                UnlockAttempt::WrongPassword
+            } else {
+                UnlockAttempt::NotReady
+            }
+        }
+        Err(_) => UnlockAttempt::NotReady,
+    }
+}
+
+/// Unlock an existing wallet. Ok(true) = unlocked; Ok(false) = every candidate
+/// password was actively rejected (unrecoverable — caller should recreate);
+/// Err = transient (LND not ready / timeout — caller should retry, NOT wipe).
+async fn unlock_existing_wallet() -> Result<bool> {
+    unlock_existing_wallet_via_rest().await
+}
+
+async fn unlock_existing_wallet_via_rest() -> Result<bool> {
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(std::time::Duration::from_secs(20))
+        .danger_accept_invalid_certs(true)
+        .build()
+        .context("building LND REST client")?;
+
+    let candidates = unlock_password_candidates().await;
+    // Retry only while LND's unlocker isn't ready yet. If every candidate is
+    // *actively rejected* (invalid passphrase), retrying can't help — fail fast
+    // with a clear message instead of hanging the boot path for 60s+ (the wallet
+    // was created with a password this node doesn't have → migration/recovery).
+    //
+    // The not-ready budget is deliberately generous. LND opens channel.db,
+    // graph.db and wallet.db before it starts serving the unlocker at all, and
+    // on a busy node that is genuinely slow — observed at 2m38s on a box running
+    // 30 containers, where a 60s budget could never succeed. Timing out here is
+    // not a harmless retry: reconcile records the post-start hook as failed,
+    // which restarts LND, which starts the slow database open over again. The
+    // result is a restart loop that leaves the wallet permanently locked and
+    // every LND-dependent app (BTCPay's internal node included) broken, on
+    // exactly the nodes least able to afford it. Waiting longer costs nothing —
+    // a wrong password still exits on the first pass via `all_rejected`.
+    for _ in 0..UNLOCK_NOT_READY_ATTEMPTS {
+        let mut all_rejected = true;
+        for pw in &candidates {
+            match try_unlock_once(&client, pw).await {
+                UnlockAttempt::Unlocked => return Ok(true),
+                UnlockAttempt::WrongPassword => {}
+                UnlockAttempt::NotReady => all_rejected = false,
+            }
+        }
+        if all_rejected {
+            tracing::warn!(
+                "[lnd] none of the {} candidate password(s) unlock the wallet — it was created \
+                 with a password this node does not have",
+                candidates.len()
+            );
+            return Ok(false);
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    }
+    anyhow::bail!(
+        "LND wallet unlock timed out after ~{}s waiting for the unlocker to become ready",
+        UNLOCK_NOT_READY_ATTEMPTS
+    )
+}
+
+/// Unlock an existing wallet WITHOUT the destructive fallback.
+///
+/// `ensure_wallet_initialized` wipes and recreates a wallet no candidate
+/// password can open — correct for a boot path that must self-heal, and exactly
+/// wrong for macaroon rotation, which restarts LND against a wallet the operator
+/// still wants. Rotation calls this instead, so there is no code path from
+/// "rotate my credentials" to "delete my wallet": a rejected password surfaces
+/// as an error the caller reports, never as a wipe.
+pub(crate) async fn unlock_existing_wallet_no_wipe() -> Result<()> {
+    match unlock_existing_wallet().await? {
+        true => Ok(()),
+        false => anyhow::bail!(
+            "LND rejected every candidate wallet password — refusing to touch the wallet. \
+             The wallet is intact and still locked; its password is not one this node holds."
+        ),
+    }
+}
+
+/// Current LND wallet state via the unauthenticated `/v1/state` endpoint
+/// (NON_EXISTING / LOCKED / UNLOCKED / RPC_ACTIVE / …). None if unreachable.
+async fn wallet_state(client: &reqwest::Client) -> Option<String> {
+    let resp = client
+        .get(format!("{LND_REST_BASE_URL}/v1/state"))
+        .send()
+        .await
+        .ok()?;
+    let v: serde_json::Value = resp.json().await.ok()?;
+    v.get("state")
+        .and_then(|s| s.as_str())
+        .map(|s| s.to_string())
+}
+
+/// ChangePassword via WalletUnlocker (wallet must be LOCKED). Both passwords are
+/// base64-encoded. Ok(true) = current accepted and rotated; Ok(false) = current
+/// rejected (wrong password — try the next candidate); Err = transport/other.
+async fn change_wallet_password(
+    client: &reqwest::Client,
+    current: &str,
+    new: &str,
+) -> Result<bool> {
+    let body = serde_json::json!({
+        "current_password": base64::engine::general_purpose::STANDARD.encode(current),
+        "new_password": base64::engine::general_purpose::STANDARD.encode(new),
+    });
+    let resp = client
+        .post(format!("{LND_REST_BASE_URL}/v1/changepassword"))
+        .json(&body)
+        .send()
+        .await
+        .context("calling LND changepassword")?;
+    let status = resp.status();
+    let text = resp.text().await.unwrap_or_default();
+    if status.is_success() {
+        Ok(true)
+    } else if text.contains("invalid passphrase") {
+        Ok(false)
+    } else {
+        anyhow::bail!("LND changepassword returned {status}: {text}")
+    }
+}
+
+/// Best-effort migration of a LOCKED wallet onto the per-node secret. Called at
+/// login, when the onboarding password is available as a candidate. If the
+/// per-node secret already opens the wallet, just unlock. Otherwise try each
+/// candidate as the CURRENT password and ChangePassword it to a fresh per-node
+/// secret so all future boots auto-unlock. Ok(true) = healed/unlocked;
+/// Ok(false) = not locked, or no candidate worked (seed-recovery required).
+pub(crate) async fn migrate_locked_wallet(candidates: &[String]) -> Result<bool> {
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(std::time::Duration::from_secs(20))
+        .danger_accept_invalid_certs(true)
+        .build()
+        .context("building LND REST client")?;
+
+    // Only act on a wallet that is actually LOCKED.
+    if wallet_state(&client).await.as_deref() != Some("LOCKED") {
+        return Ok(false);
+    }
+
+    // If the per-node secret already opens it, nothing to rotate — just unlock.
+    if let Some(secret) = read_wallet_password().await {
+        if matches!(
+            try_unlock_once(&client, &secret).await,
+            UnlockAttempt::Unlocked
+        ) {
+            return Ok(true);
+        }
+    }
+
+    // The wallet's new password becomes the per-node secret (generate if absent).
+    let new_secret = ensure_wallet_password().await?;
+
+    // ChangePassword requires LOCKED; bail out if a prior step already unlocked.
+    if wallet_state(&client).await.as_deref() != Some("LOCKED") {
+        return Ok(true);
+    }
+
+    for cand in candidates {
+        if cand.is_empty() || *cand == new_secret {
+            continue;
+        }
+        match change_wallet_password(&client, cand, &new_secret).await {
+            Ok(true) => {
+                tracing::info!("[lnd-migrate] rotated locked wallet onto the per-node secret");
+                return Ok(true);
+            }
+            Ok(false) => continue, // wrong current password — try next candidate
+            Err(e) => tracing::debug!("[lnd-migrate] changepassword error: {e}"),
+        }
+    }
+    tracing::warn!(
+        "[lnd-migrate] no candidate password opened the wallet — seed-recovery required"
+    );
+    Ok(false)
+}
+
+#[derive(Debug, Deserialize)]
+struct GenSeedResponse {
+    cipher_seed_mnemonic: Vec<String>,
+}
+
+#[derive(Debug)]
+enum UnlockerResponse<T> {
+    Value(T),
+    WalletAlreadyExists,
+}
+
+#[derive(Debug, Serialize)]
+struct InitWalletRequest {
+    wallet_password: String,
+    cipher_seed_mnemonic: Vec<String>,
+}
+
+async fn init_wallet_via_rest() -> Result<()> {
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(std::time::Duration::from_secs(20))
+        .danger_accept_invalid_certs(true)
+        .build()
+        .context("building LND REST client")?;
+
+    let seed: GenSeedResponse = match get_lnd_unlocker_json(&client, "/v1/genseed")
+        .await
+        .context("generating LND wallet seed")?
+    {
+        UnlockerResponse::Value(seed) => seed,
+        UnlockerResponse::WalletAlreadyExists => {
+            unlock_existing_wallet().await?;
+            return Ok(());
+        }
+    };
+    if seed.cipher_seed_mnemonic.is_empty() {
+        anyhow::bail!("LND genseed returned no seed words");
+    }
+
+    let node_secret = ensure_wallet_password().await?;
+    let wallet_password = base64::engine::general_purpose::STANDARD.encode(&node_secret);
+    let req = InitWalletRequest {
+        wallet_password,
+        cipher_seed_mnemonic: seed.cipher_seed_mnemonic.clone(),
+    };
+    match post_lnd_unlocker_json::<serde_json::Value>(
+        &client,
+        "/v1/initwallet",
+        serde_json::to_value(req)?,
+    )
+    .await
+    .context("initializing LND wallet")?
+    {
+        UnlockerResponse::Value(_) => {
+            persist_aezeed_backup(
+                std::path::Path::new(ARCHY_DATA_DIR),
+                &seed.cipher_seed_mnemonic,
+                &node_secret,
+            )
+            .await;
+        }
+        UnlockerResponse::WalletAlreadyExists => {
+            unlock_existing_wallet().await?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Persist the just-created wallet's aezeed words encrypted with the per-node
+/// secret so they can be revealed later (`lnd.seed-reveal`). aezeed cannot be
+/// re-derived after init, so this is the only capture point on the unattended
+/// boot path. Best-effort: a failed backup must not fail wallet creation.
+pub(crate) async fn persist_aezeed_backup(
+    data_dir: &std::path::Path,
+    words: &[String],
+    node_secret: &str,
+) {
+    match crate::seed::save_lnd_aezeed_encrypted(data_dir, words, node_secret).await {
+        Ok(()) => {
+            // A fresh wallet means a fresh seed — any previous "user backed it
+            // up" acknowledgment no longer applies.
+            crate::seed::clear_lnd_aezeed_acknowledged(data_dir).await;
+            tracing::info!("[lnd] aezeed backup saved (encrypted with per-node secret)");
+        }
+        Err(e) => tracing::warn!("[lnd] failed to save aezeed backup: {e:#}"),
+    }
+}
+
+/// The per-node wallet password if one has been persisted; never generates.
+/// Used by the reveal RPC to decrypt the aezeed backup.
+pub(crate) async fn wallet_password_if_exists() -> Option<String> {
+    read_wallet_password().await
+}
+
+async fn get_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
+    client: &reqwest::Client,
+    path: &str,
+) -> Result<UnlockerResponse<T>> {
+    let url = format!("{LND_REST_BASE_URL}{path}");
+    let mut last_err = None;
+    for _ in 0..60 {
+        match client.get(&url).send().await {
+            Ok(resp) => match decode_lnd_unlocker_response(resp, path).await {
+                Ok(value) => return Ok(value),
+                Err(e) => last_err = Some(e.to_string()),
+            },
+            Err(e) => last_err = Some(e.to_string()),
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    }
+    anyhow::bail!(
+        "LND REST {path} unavailable: {}",
+        last_err.unwrap_or_else(|| "unknown error".to_string())
+    )
+}
+
+async fn post_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
+    client: &reqwest::Client,
+    path: &str,
+    body: serde_json::Value,
+) -> Result<UnlockerResponse<T>> {
+    let url = format!("{LND_REST_BASE_URL}{path}");
+    let mut last_err = None;
+    for _ in 0..60 {
+        match client.post(&url).json(&body).send().await {
+            Ok(resp) => match decode_lnd_unlocker_response(resp, path).await {
+                Ok(value) => return Ok(value),
+                Err(e) => last_err = Some(e.to_string()),
+            },
+            Err(e) => last_err = Some(e.to_string()),
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    }
+    anyhow::bail!(
+        "LND REST {path} unavailable: {}",
+        last_err.unwrap_or_else(|| "unknown error".to_string())
+    )
+}
+
+async fn decode_lnd_unlocker_response<T: for<'de> Deserialize<'de>>(
+    resp: reqwest::Response,
+    path: &str,
+) -> Result<UnlockerResponse<T>> {
+    let status = resp.status();
+    let text = resp.text().await.unwrap_or_default();
+    if status.is_success() {
+        let value = serde_json::from_str(&text)
+            .with_context(|| format!("parsing LND REST response from {path}"))?;
+        return Ok(UnlockerResponse::Value(value));
+    }
+    if text.contains("wallet already exists") {
+        return Ok(UnlockerResponse::WalletAlreadyExists);
+    }
+    anyhow::bail!("LND REST {path} returned {status}: {text}")
+}
+
+async fn lnd_getinfo_ready(admin_macaroon: &str) -> bool {
+    let Ok(macaroon) = read_file_as_root(admin_macaroon).await else {
+        return false;
+    };
+    let Ok(client) = reqwest::Client::builder()
+        .no_proxy()
+        .timeout(std::time::Duration::from_secs(5))
+        .danger_accept_invalid_certs(true)
+        .build()
+    else {
+        return false;
+    };
+    client
+        .get(format!("{LND_REST_BASE_URL}/v1/getinfo"))
+        .header("Grpc-Metadata-macaroon", hex::encode(macaroon))
+        .send()
+        .await
+        .map(|resp| resp.status().is_success())
+        .unwrap_or(false)
+}
+
+async fn wait_for_admin_macaroon(admin_macaroon: &str) -> Result<()> {
+    for _ in 0..60 {
+        if file_exists_as_root(admin_macaroon).await {
+            return Ok(());
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+    }
+    anyhow::bail!("LND admin macaroon not created after wallet init")
+}
+
+async fn write_config_atomically(paths: &EnsurePaths, conf: &str) -> Result<()> {
+    let tmp = paths.conf_path.with_extension("tmp");
+    match fs::write(&tmp, conf).await {
+        Ok(()) => {
+            fs::rename(&tmp, &paths.conf_path).await.with_context(|| {
+                format!(
+                    "renaming {} -> {}",
+                    tmp.display(),
+                    paths.conf_path.display()
+                )
+            })?;
+            Ok(())
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
+            let script = format!(
+                "set -eu\ncat > '{}' <<'LNDCONF'\n{}LNDCONF\n",
+                shell_quote(&paths.conf_path.to_string_lossy()),
+                conf
+            );
+            let status = host_sudo(&["sh", "-lc", &script])
+                .await
+                .context("writing lnd.conf via sudo")?;
+            if !status.success() {
+                anyhow::bail!("writing lnd.conf via sudo exited with {status}");
+            }
+            Ok(())
+        }
+        Err(e) => Err(e).with_context(|| format!("writing tmp {}", tmp.display())),
+    }
+}
+
+fn shell_quote(s: &str) -> String {
+    s.replace('\'', "'\\''")
+}
+
+fn has_required_lnd_flags(conf: &str, rpc_pass: &str, bitcoin_host: &str) -> bool {
+    let rpc_pass_line = format!("bitcoind.rpcpass={rpc_pass}");
+    let rpc_host_line = format!("bitcoind.rpchost={bitcoin_host}:8332");
+    [
+        "bitcoin.active=true",
+        "bitcoin.mainnet=true",
+        "bitcoin.node=bitcoind",
+        rpc_host_line.as_str(),
+        rpc_pass_line.as_str(),
+    ]
+    .iter()
+    .all(|needle| conf.lines().any(|line| line.trim() == *needle))
+}
+
+/// Secret file consumed by btcpay-server's optional `BTCPAY_BTCLIGHTNING`
+/// secret_env (see apps/btcpay-server/manifest.yml).
+const BTCPAY_LND_CONNECTION_SECRET: &str = "btcpay-lnd-connection";
+
+/// Materialise the BTCPay→internal-LND connection-string secret.
+///
+/// LND's datadir is owned by its container subuid (100999 on a stock node),
+/// so btcpay cannot bind-mount the macaroon — EACCES across the userns
+/// boundary. The connection string therefore carries the macaroon inline as
+/// hex, delivered by reference through the podman secret store.
+///
+/// No-op when LND isn't provisioned yet (missing tls.cert or macaroon) —
+/// btcpay's secret_env entry is `optional`, so it simply starts without an
+/// internal Lightning node and picks it up on a later reconcile tick.
+/// Rewrites when the pinned cert thumbprint no longer matches (LND TLS cert
+/// rotation). Macaroon rotation without cert rotation is not auto-detected here
+/// (reading the macaroon needs sudo; probing it every tick is not worth the
+/// churn) — the rotation path calls `rewrite_btcpay_lnd_connection_secret`
+/// instead, and deleting the secret file also forces regeneration.
+pub async fn ensure_btcpay_lnd_connection_secret(secrets_dir: &std::path::Path) -> Result<()> {
+    build_btcpay_lnd_connection_secret(secrets_dir, false)
+        .await
+        .map(|_| ())
+}
+
+/// Rewrite the BTCPay→LND connection secret unconditionally, ignoring the
+/// cert-thumbprint fast path.
+///
+/// Rotating LND's macaroons invalidates the one embedded in this secret, and it
+/// is embedded *inline* rather than referenced by path — LND's datadir is owned
+/// by its container subuid, so btcpay cannot bind-mount the file and the string
+/// cannot self-heal. Nothing else notices: the TLS cert is untouched by macaroon
+/// rotation, so `ensure_…` takes its fast path forever and BTCPay keeps
+/// presenting a credential LND no longer honours. A node in that state looks
+/// entirely healthy — btcpay is up, LND is up — while every Lightning invoice it
+/// tries to create fails.
+///
+/// Writing the new value makes the change *visible*: `secret_env_hash` is
+/// derived from the resolved secret contents, so a changed file reads as label
+/// drift on the running container. It is not sufficient on its own — btcpay is
+/// restart-sensitive, and boot reconcile deliberately leaves running
+/// restart-sensitive apps untouched on drift. The caller must also call
+/// `ContainerOrchestrator::mark_credential_rotated("btcpay-server")`, which is
+/// the carve-out for exactly this case: a container that is up and healthy while
+/// holding a credential that no longer works. The orchestrator's own recreate
+/// path then rebuilds it around an unchanged data directory. No teardown here,
+/// deliberately — a hand-rolled remove-and-run is the anti-pattern CLAUDE.md
+/// names.
+///
+/// Returns `false` when LND isn't provisioned enough to derive a value.
+pub async fn rewrite_btcpay_lnd_connection_secret(secrets_dir: &std::path::Path) -> Result<bool> {
+    build_btcpay_lnd_connection_secret(secrets_dir, true).await
+}
+
+/// Shared body. `force` skips the "already pins the current cert" fast path.
+/// Returns whether a value was written.
+async fn build_btcpay_lnd_connection_secret(
+    secrets_dir: &std::path::Path,
+    force: bool,
+) -> Result<bool> {
+    let cert_path = format!("{DEFAULT_DATA_DIR}/tls.cert");
+    let pem = match fs::read_to_string(&cert_path).await {
+        Ok(s) => s,
+        Err(_) => return Ok(false), // LND not installed/provisioned yet
+    };
+    let thumbprint = cert_sha256_thumbprint(&pem).context("computing LND tls.cert thumbprint")?;
+
+    let target = secrets_dir.join(BTCPAY_LND_CONNECTION_SECRET);
+    // Fast path (no sudo): existing secret already pins the current cert.
+    if !force {
+        if let Ok(existing) = fs::read_to_string(&target).await {
+            if !existing.trim().is_empty()
+                && existing.contains(&format!("certthumbprint={thumbprint}"))
+            {
+                return Ok(false);
+            }
+        }
+    }
+
+    let macaroon_path = format!("{DEFAULT_DATA_DIR}/data/chain/bitcoin/mainnet/admin.macaroon");
+    if !file_exists_as_root(&macaroon_path).await {
+        return Ok(false); // wallet not created yet; next tick retries
+    }
+    let macaroon = read_file_as_root(&macaroon_path).await?;
+    let value = format!(
+        "type=lnd-rest;server=https://lnd:8080/;macaroon={};certthumbprint={}",
+        hex::encode(macaroon),
+        thumbprint
+    );
+    crate::container::secrets::write_secret_file(&target, &value)
+        .context("writing btcpay-lnd-connection secret")?;
+    Ok(true)
+}
+
+/// Does the on-disk BTCPay connection secret still carry the macaroon LND is
+/// currently issuing? `None` when there is nothing to compare — no secret file
+/// (BTCPay has no internal node configured) or no macaroon (LND unprovisioned).
+///
+/// Compares only hex text that is already on this host; the value is never
+/// logged, returned over RPC, or placed in an error.
+pub(crate) async fn btcpay_lnd_connection_is_current(
+    secrets_dir: &std::path::Path,
+) -> Option<bool> {
+    let target = secrets_dir.join(BTCPAY_LND_CONNECTION_SECRET);
+    let existing = fs::read_to_string(&target).await.ok()?;
+    let embedded = existing
+        .split("macaroon=")
+        .nth(1)?
+        .split(';')
+        .next()?
+        .to_string();
+    let macaroon_path = format!("{DEFAULT_DATA_DIR}/data/chain/bitcoin/mainnet/admin.macaroon");
+    let current = read_file_as_root(&macaroon_path).await.ok()?;
+    Some(embedded.eq_ignore_ascii_case(&hex::encode(current)))
+}
+
+/// SHA256 over the DER certificate body (matches
+/// `openssl x509 -fingerprint -sha256` without colons) — the format BTCPay's
+/// `certthumbprint=` connection-string parameter expects.
+fn cert_sha256_thumbprint(pem: &str) -> Result<String> {
+    use sha2::{Digest, Sha256};
+    let b64: String = pem
+        .lines()
+        .filter(|l| !l.starts_with("-----"))
+        .collect::<Vec<_>>()
+        .join("");
+    let der = base64::engine::general_purpose::STANDARD
+        .decode(b64.trim())
+        .context("decoding tls.cert PEM body")?;
+    Ok(hex::encode_upper(Sha256::digest(&der)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn ensure_config_writes_required_bitcoin_network_flags() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            data_dir: tmp.path().join("lnd"),
+            conf_path: tmp.path().join("lnd/lnd.conf"),
+        };
+
+        let out = ensure_config(&paths, "secret", "bitcoin-knots")
+            .await
+            .unwrap();
+        assert_eq!(out, EnsureOutcome::Written);
+        let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
+        assert!(conf.contains("bitcoin.active=true"));
+        assert!(conf.contains("bitcoin.mainnet=true"));
+        assert!(conf.contains("bitcoin.node=bitcoind"));
+        assert!(conf.contains("bitcoind.rpchost=bitcoin-knots:8332"));
+        assert!(conf.contains("bitcoind.rpcpass=secret"));
+    }
+
+    #[tokio::test]
+    async fn ensure_config_repairs_rpc_password_drift() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            data_dir: tmp.path().join("lnd"),
+            conf_path: tmp.path().join("lnd/lnd.conf"),
+        };
+
+        assert_eq!(
+            ensure_config(&paths, "first", "bitcoin-knots")
+                .await
+                .unwrap(),
+            EnsureOutcome::Written
+        );
+        assert_eq!(
+            ensure_config(&paths, "second", "bitcoin-knots")
+                .await
+                .unwrap(),
+            EnsureOutcome::Written
+        );
+        let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
+        assert!(conf.contains("bitcoind.rpcpass=second"));
+    }
+
+    #[tokio::test]
+    async fn ensure_config_repairs_bitcoin_host_drift() {
+        // A conf written against bitcoin-knots must be rewritten when the
+        // node's Bitcoin variant is bitcoin-core, or LND dials a hostname
+        // that doesn't exist on archy-net and dies on startup.
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            data_dir: tmp.path().join("lnd"),
+            conf_path: tmp.path().join("lnd/lnd.conf"),
+        };
+
+        assert_eq!(
+            ensure_config(&paths, "pw", "bitcoin-knots").await.unwrap(),
+            EnsureOutcome::Written
+        );
+        assert_eq!(
+            ensure_config(&paths, "pw", "bitcoin-core").await.unwrap(),
+            EnsureOutcome::Written
+        );
+        let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
+        assert!(conf.contains("bitcoind.rpchost=bitcoin-core:8332"));
+        assert!(!conf.contains("bitcoind.rpchost=bitcoin-knots:8332"));
+
+        assert_eq!(
+            ensure_config(&paths, "pw", "bitcoin-core").await.unwrap(),
+            EnsureOutcome::Unchanged
+        );
+    }
+
+    #[tokio::test]
+    async fn ensure_config_repairs_incomplete_existing_config() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let paths = EnsurePaths {
+            data_dir: tmp.path().join("lnd"),
+            conf_path: tmp.path().join("lnd/lnd.conf"),
+        };
+        fs::create_dir_all(&paths.data_dir).await.unwrap();
+        fs::write(&paths.conf_path, "debuglevel=info\n")
+            .await
+            .unwrap();
+
+        assert_eq!(
+            ensure_config(&paths, "repaired", "bitcoin-knots")
+                .await
+                .unwrap(),
+            EnsureOutcome::Written
+        );
+        let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
+        assert!(conf.contains("bitcoin.mainnet=true"));
+        assert!(conf.contains("bitcoind.rpcpass=repaired"));
+    }
+
+    #[test]
+    fn legacy_wallet_password_is_valid_for_lncli() {
+        // Legacy fallback must still be a valid lncli passphrase (>8 chars).
+        assert!(LEGACY_WALLET_PASSWORD.len() > 8);
+    }
+
+    #[tokio::test]
+    async fn unlock_candidates_always_include_legacy_fallback() {
+        // With no per-node secret on disk in the test env, candidates fall back
+        // to the legacy constant so old wallets still open.
+        let cands = unlock_password_candidates().await;
+        assert!(cands.iter().any(|p| p == LEGACY_WALLET_PASSWORD));
+    }
+}

core/archipelago/src/container/mock_podman.rs

@@ -0,0 +1,289 @@
+//! Mock container runtime for unit testing orchestration logic.
+//!
+//! Simulates podman behavior in-memory: container lifecycle, health checks,
+//! image pulls (with configurable failures for retry testing).
+
+use std::collections::HashMap;
+use std::sync::{
+    atomic::{AtomicBool, AtomicU32, Ordering},
+    Arc, Mutex,
+};
+
+/// Container state matching podman's real states.
+#[derive(Debug, Clone, PartialEq)]
+#[allow(dead_code)]
+pub enum MockContainerState {
+    Created,
+    Running,
+    Exited(i32), // exit code
+    Stopped,
+}
+
+impl MockContainerState {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Created => "created",
+            Self::Running => "running",
+            Self::Exited(_) => "exited",
+            Self::Stopped => "stopped",
+        }
+    }
+}
+
+/// A simulated container.
+#[derive(Debug, Clone)]
+#[allow(dead_code)]
+pub struct MockContainer {
+    pub name: String,
+    pub image: String,
+    pub state: MockContainerState,
+    pub stop_timeout_used: Option<u64>,
+}
+
+/// Mock podman runtime for testing orchestration logic without real containers.
+#[allow(dead_code)]
+pub struct MockPodman {
+    containers: Arc<Mutex<HashMap<String, MockContainer>>>,
+    /// When true, `podman pull` will fail (simulates registry down).
+    pub fail_pull: Arc<AtomicBool>,
+    /// When true, containers exit immediately after start (simulates crash).
+    pub fail_start: Arc<AtomicBool>,
+    /// Count of pull attempts (for retry testing).
+    pub pull_attempt_count: Arc<AtomicU32>,
+    /// Count of start attempts.
+    pub start_attempt_count: Arc<AtomicU32>,
+    /// Images that have been "pulled" (exist locally).
+    images: Arc<Mutex<Vec<String>>>,
+}
+
+#[allow(dead_code)]
+impl MockPodman {
+    pub fn new() -> Self {
+        Self {
+            containers: Arc::new(Mutex::new(HashMap::new())),
+            fail_pull: Arc::new(AtomicBool::new(false)),
+            fail_start: Arc::new(AtomicBool::new(false)),
+            pull_attempt_count: Arc::new(AtomicU32::new(0)),
+            start_attempt_count: Arc::new(AtomicU32::new(0)),
+            images: Arc::new(Mutex::new(Vec::new())),
+        }
+    }
+
+    /// Simulate `podman pull <image>`. Respects fail_pull flag.
+    pub fn pull_image(&self, image: &str) -> Result<(), String> {
+        self.pull_attempt_count.fetch_add(1, Ordering::SeqCst);
+        if self.fail_pull.load(Ordering::SeqCst) {
+            return Err(format!(
+                "Error: initializing source docker://{}: connection refused",
+                image
+            ));
+        }
+        self.images.lock().unwrap().push(image.to_string());
+        Ok(())
+    }
+
+    /// Check if an image exists locally (was pulled).
+    pub fn image_exists(&self, image: &str) -> bool {
+        self.images.lock().unwrap().iter().any(|i| i == image)
+    }
+
+    /// Simulate `podman run -d --name <name> <image>`.
+    pub fn create_and_start(&self, name: &str, image: &str) -> Result<String, String> {
+        self.start_attempt_count.fetch_add(1, Ordering::SeqCst);
+
+        if !self.image_exists(image) {
+            return Err(format!("Error: {} not found", image));
+        }
+
+        let state = if self.fail_start.load(Ordering::SeqCst) {
+            MockContainerState::Exited(1)
+        } else {
+            MockContainerState::Running
+        };
+
+        let container = MockContainer {
+            name: name.to_string(),
+            image: image.to_string(),
+            state,
+            stop_timeout_used: None,
+        };
+
+        self.containers
+            .lock()
+            .unwrap()
+            .insert(name.to_string(), container);
+        Ok(format!("abc123def456_{}", name))
+    }
+
+    /// Simulate `podman start <name>`.
+    pub fn start(&self, name: &str) -> Result<(), String> {
+        let mut containers = self.containers.lock().unwrap();
+        match containers.get_mut(name) {
+            Some(c) => {
+                if self.fail_start.load(Ordering::SeqCst) {
+                    c.state = MockContainerState::Exited(1);
+                } else {
+                    c.state = MockContainerState::Running;
+                }
+                Ok(())
+            }
+            None => Err(format!("Error: no such container {}", name)),
+        }
+    }
+
+    /// Simulate `podman stop -t <timeout> <name>`.
+    pub fn stop(&self, name: &str, timeout: u64) -> Result<(), String> {
+        let mut containers = self.containers.lock().unwrap();
+        match containers.get_mut(name) {
+            Some(c) => {
+                c.state = MockContainerState::Stopped;
+                c.stop_timeout_used = Some(timeout);
+                Ok(())
+            }
+            None => Err(format!("Error: no such container {}", name)),
+        }
+    }
+
+    /// Simulate `podman rm -f <name>`.
+    pub fn remove(&self, name: &str) -> Result<(), String> {
+        self.containers.lock().unwrap().remove(name);
+        Ok(())
+    }
+
+    /// Simulate `podman inspect <name> --format {{.State.Status}}`.
+    pub fn inspect_state(&self, name: &str) -> Option<String> {
+        self.containers
+            .lock()
+            .unwrap()
+            .get(name)
+            .map(|c| c.state.as_str().to_string())
+    }
+
+    /// List all containers (like `podman ps -a`).
+    pub fn list_all(&self) -> Vec<MockContainer> {
+        self.containers.lock().unwrap().values().cloned().collect()
+    }
+
+    /// Get a specific container.
+    pub fn get(&self, name: &str) -> Option<MockContainer> {
+        self.containers.lock().unwrap().get(name).cloned()
+    }
+
+    /// Pre-load an image (as if it was already pulled or bundled).
+    pub fn preload_image(&self, image: &str) {
+        self.images.lock().unwrap().push(image.to_string());
+    }
+
+    /// Pre-load a container in a specific state.
+    pub fn preload_container(&self, name: &str, image: &str, state: MockContainerState) {
+        self.containers.lock().unwrap().insert(
+            name.to_string(),
+            MockContainer {
+                name: name.to_string(),
+                image: image.to_string(),
+                state,
+                stop_timeout_used: None,
+            },
+        );
+    }
+
+    /// Get the stop timeout that was used for a container.
+    pub fn get_stop_timeout(&self, name: &str) -> Option<u64> {
+        self.containers
+            .lock()
+            .unwrap()
+            .get(name)
+            .and_then(|c| c.stop_timeout_used)
+    }
+
+    /// Reset all counters and state.
+    pub fn reset(&self) {
+        self.containers.lock().unwrap().clear();
+        self.images.lock().unwrap().clear();
+        self.fail_pull.store(false, Ordering::SeqCst);
+        self.fail_start.store(false, Ordering::SeqCst);
+        self.pull_attempt_count.store(0, Ordering::SeqCst);
+        self.start_attempt_count.store(0, Ordering::SeqCst);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pull_and_start() {
+        let mock = MockPodman::new();
+        mock.pull_image("test:latest").unwrap();
+        assert!(mock.image_exists("test:latest"));
+        mock.create_and_start("test-container", "test:latest")
+            .unwrap();
+        assert_eq!(
+            mock.inspect_state("test-container"),
+            Some("running".to_string())
+        );
+    }
+
+    #[test]
+    fn test_pull_failure() {
+        let mock = MockPodman::new();
+        mock.fail_pull.store(true, Ordering::SeqCst);
+        assert!(mock.pull_image("test:latest").is_err());
+        assert!(!mock.image_exists("test:latest"));
+        assert_eq!(mock.pull_attempt_count.load(Ordering::SeqCst), 1);
+    }
+
+    #[test]
+    fn test_start_failure() {
+        let mock = MockPodman::new();
+        mock.preload_image("test:latest");
+        mock.fail_start.store(true, Ordering::SeqCst);
+        mock.create_and_start("crasher", "test:latest").unwrap();
+        assert_eq!(mock.inspect_state("crasher"), Some("exited".to_string()));
+    }
+
+    #[test]
+    fn test_stop_records_timeout() {
+        let mock = MockPodman::new();
+        mock.preload_image("test:latest");
+        mock.create_and_start("test", "test:latest").unwrap();
+        mock.stop("test", 600).unwrap();
+        assert_eq!(mock.get_stop_timeout("test"), Some(600));
+        assert_eq!(mock.inspect_state("test"), Some("stopped".to_string()));
+    }
+
+    #[test]
+    fn test_remove() {
+        let mock = MockPodman::new();
+        mock.preload_image("test:latest");
+        mock.create_and_start("removeme", "test:latest").unwrap();
+        mock.remove("removeme").unwrap();
+        assert!(mock.inspect_state("removeme").is_none());
+    }
+
+    #[test]
+    fn test_start_without_image_fails() {
+        let mock = MockPodman::new();
+        assert!(mock.create_and_start("nope", "missing:latest").is_err());
+    }
+
+    #[test]
+    fn test_preload_container() {
+        let mock = MockPodman::new();
+        mock.preload_container("existing", "img:1.0", MockContainerState::Running);
+        assert_eq!(mock.inspect_state("existing"), Some("running".to_string()));
+        assert_eq!(mock.list_all().len(), 1);
+    }
+
+    #[test]
+    fn test_reset() {
+        let mock = MockPodman::new();
+        mock.preload_image("img:1");
+        mock.preload_container("c1", "img:1", MockContainerState::Running);
+        mock.fail_pull.store(true, Ordering::SeqCst);
+        mock.reset();
+        assert!(!mock.image_exists("img:1"));
+        assert!(mock.list_all().is_empty());
+        assert!(!mock.fail_pull.load(Ordering::SeqCst));
+    }
+}

core/archipelago/src/container/mod.rs

@@ -0,0 +1,25 @@
+pub mod app_catalog;
+pub mod bitcoin_ui;
+pub mod boot_reconciler;
+pub mod companion;
+pub mod data_manager;
+pub mod dev_orchestrator;
+pub mod docker_packages;
+pub mod filebrowser;
+pub mod hooks;
+pub mod image_policy;
+pub mod image_versions;
+pub mod lnd;
+pub mod prod_orchestrator;
+pub mod quadlet;
+pub mod registry;
+pub mod secrets;
+pub mod traits;
+pub mod ui_detection;
+pub mod version_config;
+
+pub use boot_reconciler::{BootReconciler, DEFAULT_INTERVAL as RECONCILER_DEFAULT_INTERVAL};
+pub use dev_orchestrator::DevContainerOrchestrator;
+pub use docker_packages::DockerPackageScanner;
+pub use prod_orchestrator::ProdContainerOrchestrator;
+pub use traits::ContainerOrchestrator;

core/archipelago/src/container/registry.rs

@@ -0,0 +1,310 @@
+//! Dynamic container registry configuration.
+//!
+//! Manages a list of container registries that the node uses to pull app images.
+//! Registries are tried in order — if the first fails, the next is attempted.
+//! Configuration is persisted to disk and editable via RPC.
+
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+use tokio::fs;
+
+const REGISTRY_FILE: &str = "config/registries.json";
+const OVH_REGISTRY_URL: &str = "source.archipelago-foundation.org/lfg2025";
+/// Retired registry host (release server retired 2026-06-13; the registry
+/// frontend was fully dead by 2026-07-10 — 500 on every /v2 manifest read).
+/// Never a default, never force-enabled; stripped from saved configs on
+/// load. The literal exists ONLY so the strip can match — nothing may pull
+/// through this host.
+const RETIRED_TX1138_HOST: &str = "git.tx1138.com";
+
+/// A single container registry.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Registry {
+    /// Registry URL (e.g., "source.archipelago-foundation.org/lfg2025").
+    pub url: String,
+    /// Human-readable name.
+    pub name: String,
+    /// Whether TLS verification is required (false for HTTP registries).
+    pub tls_verify: bool,
+    /// Whether this registry is enabled.
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+    /// Priority (lower = tried first).
+    #[serde(default)]
+    pub priority: u32,
+}
+
+fn default_true() -> bool {
+    true
+}
+
+/// Registry configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RegistryConfig {
+    pub registries: Vec<Registry>,
+}
+
+impl Default for RegistryConfig {
+    fn default() -> Self {
+        Self {
+            registries: vec![Registry {
+                url: OVH_REGISTRY_URL.to_string(),
+                name: "Server 1 (OVH)".to_string(),
+                tls_verify: false,
+                enabled: true,
+                priority: 0,
+            }],
+        }
+    }
+}
+
+impl RegistryConfig {
+    /// Get enabled registries sorted by priority.
+    pub fn active_registries(&self) -> Vec<&Registry> {
+        let mut regs: Vec<&Registry> = self.registries.iter().filter(|r| r.enabled).collect();
+        regs.sort_by_key(|r| r.priority);
+        regs
+    }
+
+    /// Rewrite an image reference to use a specific registry.
+    /// E.g., "docker.io/lfg2025/bitcoin-knots:latest" with registry "source.archipelago-foundation.org/lfg2025"
+    /// becomes "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest".
+    pub fn rewrite_image(&self, image: &str, registry: &Registry) -> String {
+        // Extract the image name (last component after the org/namespace)
+        // Handles: "registry/org/image:tag" -> "image:tag"
+        let image_name = extract_image_name(image);
+        format!("{}/{}", registry.url, image_name)
+    }
+}
+
+/// Extract the image name from a full image reference.
+/// "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest" -> "bitcoin-knots:latest"
+/// "docker.io/gitea/gitea:1.23" -> "gitea:1.23"
+fn extract_image_name(image: &str) -> &str {
+    // Split by '/' and take the last segment (image:tag)
+    image.rsplit('/').next().unwrap_or(image)
+}
+
+/// Load registry config from disk, merging in any default registries
+/// that the operator hasn't explicitly removed. This lets us roll out
+/// new default mirrors (e.g. a new Server 3) to existing nodes without
+/// them having to edit their saved config. Explicit removals stick —
+/// if the URL is absent from disk AND absent from current defaults, it
+/// stays gone.
+pub async fn load_registries(data_dir: &Path) -> Result<RegistryConfig> {
+    let path = data_dir.join(REGISTRY_FILE);
+    if !path.exists() {
+        return Ok(RegistryConfig::default());
+    }
+    let content = fs::read_to_string(&path)
+        .await
+        .context("Failed to read registry config")?;
+    let mut config: RegistryConfig =
+        serde_json::from_str(&content).unwrap_or_else(|_| RegistryConfig::default());
+
+    // One-time migration: the Hetzner VPS at 23.182.128.160 was
+    // decommissioned 2026-04-23. Existing nodes have it baked into
+    // their saved registry list (was the original Server 1). Strip it
+    // on load so every container pull doesn't pay a connection-refused
+    // timeout against a dead host. Exception to the usual "explicit
+    // removals stick" rule: the user never chose to add this — it
+    // was a default.
+    let before = config.registries.len();
+    config
+        .registries
+        .retain(|r| !r.url.contains("23.182.128.160"));
+    // Same treatment for the retired tx1138 registry (was Server 2 in older
+    // defaults): strip it on load so nothing ever pulls through the dead
+    // host again.
+    config
+        .registries
+        .retain(|r| !r.url.contains(RETIRED_TX1138_HOST));
+    // And the release server's own bare-IP twin (146.59.87.168:3000 — the
+    // same host as source.archipelago-foundation.org): older defaults listed
+    // both, so the registry UI showed one server twice. Bare-IP origins were
+    // retired 2026-08-11; the named entry stays and covers the same pulls.
+    config
+        .registries
+        .retain(|r| !r.url.contains("146.59.87.168"));
+    let mut changed = config.registries.len() != before;
+
+    // Migrate: any default registry URL that isn't already in the
+    // saved list gets appended at the end (so existing priority order
+    // is preserved for anything the operator already configured).
+    let defaults = RegistryConfig::default();
+    let known: std::collections::HashSet<String> =
+        config.registries.iter().map(|r| r.url.clone()).collect();
+    let max_priority = config
+        .registries
+        .iter()
+        .map(|r| r.priority)
+        .max()
+        .unwrap_or(0);
+    for (i, def) in defaults.registries.iter().enumerate() {
+        if !known.contains(&def.url) {
+            let mut cloned = def.clone();
+            cloned.priority = max_priority.saturating_add(10 + i as u32);
+            config.registries.push(cloned);
+            changed = true;
+        }
+    }
+    let before_order: Vec<(String, bool, u32)> = config
+        .registries
+        .iter()
+        .map(|r| (r.url.clone(), r.enabled, r.priority))
+        .collect();
+    force_ovh_registry_primary(&mut config);
+    changed = changed
+        || before_order
+            != config
+                .registries
+                .iter()
+                .map(|r| (r.url.clone(), r.enabled, r.priority))
+                .collect::<Vec<_>>();
+    if changed {
+        // Persist so the next load doesn't have to re-merge.
+        if let Err(e) = save_registries(data_dir, &config).await {
+            tracing::warn!("Failed to persist migrated registry config: {e:#}");
+        }
+    }
+    Ok(config)
+}
+
+fn force_ovh_registry_primary(config: &mut RegistryConfig) {
+    let defaults = RegistryConfig::default();
+    for def in defaults.registries {
+        if !config.registries.iter().any(|r| r.url == def.url) {
+            config.registries.push(def);
+        }
+    }
+
+    for registry in config.registries.iter_mut() {
+        match registry.url.as_str() {
+            OVH_REGISTRY_URL => {
+                registry.name = "Server 1 (OVH)".to_string();
+                registry.tls_verify = false;
+                registry.enabled = true;
+                registry.priority = 0;
+            }
+            _ => {
+                if registry.priority <= 10 {
+                    registry.priority = registry.priority.saturating_add(20);
+                }
+            }
+        }
+    }
+}
+
+/// Save registry config to disk.
+pub async fn save_registries(data_dir: &Path, config: &RegistryConfig) -> Result<()> {
+    let dir = data_dir.join("config");
+    fs::create_dir_all(&dir)
+        .await
+        .context("Failed to create config dir")?;
+    let path = data_dir.join(REGISTRY_FILE);
+    let content =
+        serde_json::to_string_pretty(config).context("Failed to serialize registry config")?;
+    fs::write(&path, content)
+        .await
+        .context("Failed to write registry config")?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_extract_image_name() {
+        assert_eq!(
+            extract_image_name("source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest"),
+            "bitcoin-knots:latest"
+        );
+        assert_eq!(
+            extract_image_name("docker.io/gitea/gitea:1.23"),
+            "gitea:1.23"
+        );
+        assert_eq!(extract_image_name("localhost/myimage:v1"), "myimage:v1");
+    }
+
+    #[test]
+    fn test_rewrite_image() {
+        let config = RegistryConfig::default();
+        // An image hardcoded to some other registry rewrites to OVH when
+        // asked for the primary mirror.
+        let primary = &config.registries[0];
+        assert_eq!(
+            config.rewrite_image("docker.io/lfg2025/bitcoin-knots:latest", primary),
+            "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest"
+        );
+    }
+
+    #[test]
+    fn test_active_registries_sorted() {
+        let config = RegistryConfig::default();
+        let active = config.active_registries();
+        assert_eq!(active.len(), 1);
+        assert_eq!(active[0].url, OVH_REGISTRY_URL);
+    }
+
+    #[tokio::test]
+    async fn test_load_default() {
+        let tmp = TempDir::new().unwrap();
+        let config = load_registries(tmp.path()).await.unwrap();
+        assert_eq!(config.registries.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_load_strips_retired_tx1138_registry() {
+        // Nodes provisioned before the retirement have the tx1138 registry
+        // baked into their saved config (was Server 2). It must be stripped
+        // on load and never re-added by the defaults merge.
+        let tmp = TempDir::new().unwrap();
+        let config = RegistryConfig {
+            registries: vec![
+                Registry {
+                    url: format!("{RETIRED_TX1138_HOST}/lfg2025"),
+                    name: "Server 2 (tx1138)".into(),
+                    tls_verify: true,
+                    enabled: true,
+                    priority: 10,
+                },
+                Registry {
+                    url: OVH_REGISTRY_URL.into(),
+                    name: "Server 1 (OVH)".into(),
+                    tls_verify: false,
+                    enabled: true,
+                    priority: 0,
+                },
+            ],
+        };
+        save_registries(tmp.path(), &config).await.unwrap();
+        let loaded = load_registries(tmp.path()).await.unwrap();
+        assert!(
+            !loaded
+                .registries
+                .iter()
+                .any(|r| r.url.contains(RETIRED_TX1138_HOST)),
+            "retired tx1138 registry must be stripped on load; got {:?}",
+            loaded.registries
+        );
+    }
+
+    #[tokio::test]
+    async fn test_save_load_roundtrip() {
+        let tmp = TempDir::new().unwrap();
+        let mut config = RegistryConfig::default();
+        config.registries.push(Registry {
+            url: "myregistry.com/apps".into(),
+            name: "Custom".into(),
+            tls_verify: true,
+            enabled: true,
+            priority: 5,
+        });
+        save_registries(tmp.path(), &config).await.unwrap();
+        let loaded = load_registries(tmp.path()).await.unwrap();
+        assert_eq!(loaded.registries.len(), 2);
+    }
+}

core/archipelago/src/container/secrets.rs

@@ -0,0 +1,561 @@
+//! Declarative, self-healing generation of app secrets.
+//!
+//! An app declares `generated_secrets` in its manifest; this module materialises
+//! them just before `secret_env` is resolved. That keeps the migration's
+//! data-driven bar: an app installs from its manifest alone — no host
+//! provisioning and no per-app Rust — and every secret lands `0600`, owned by
+//! the unprivileged (rootless) service user.
+//!
+//! Two properties make it safe to call on every install/reconcile tick:
+//!
+//! * **Idempotent** — a target file that already exists, is readable and
+//!   non-empty is left untouched, so values are stable across ticks.
+//! * **Self-healing without privilege** — a target file that exists but is
+//!   *unreadable* (the classic `root:root`-owned secret left by some earlier
+//!   path) is unlinked and rewritten. Unlinking needs write on the
+//!   service-owned secrets dir, not on the file, so this recovers the broken
+//!   state with no `chown` and no root — exactly what a rootless node needs.
+
+use anyhow::{Context, Result};
+use archipelago_container::{AppManifest, GeneratedSecret, SecretGenKind};
+use rand::RngCore;
+use std::fs;
+use std::io::Write;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::Path;
+
+/// Plaintext-password length (bytes of entropy) for [`SecretGenKind::Bcrypt`].
+const BCRYPT_PASSWORD_BYTES: usize = 24;
+
+/// Materialise every declared generated secret for `manifest` under
+/// `secrets_dir`. No-op when the manifest declares none. Safe to call on every
+/// reconcile/install tick (idempotent + self-healing).
+pub fn ensure_generated_secrets(secrets_dir: &Path, manifest: &AppManifest) -> Result<()> {
+    let specs = &manifest.app.container.generated_secrets;
+    if specs.is_empty() {
+        return Ok(());
+    }
+    fs::create_dir_all(secrets_dir)
+        .with_context(|| format!("creating secrets dir {}", secrets_dir.display()))?;
+    for gs in specs {
+        ensure_one(secrets_dir, gs).with_context(|| format!("generating secret '{}'", gs.name))?;
+    }
+    Ok(())
+}
+
+fn ensure_one(dir: &Path, gs: &GeneratedSecret) -> Result<()> {
+    let files = gs.target_files();
+
+    // Idempotent fast path: every target file present, readable and non-empty.
+    if files.iter().all(|f| readable_nonempty(&dir.join(f))) {
+        return Ok(());
+    }
+
+    // Self-heal: drop any stale/unreadable target so the write below recreates
+    // it owned by us. Unlinking uses the (service-owned) dir's write bit, so a
+    // wrongly root-owned secret is recovered with no privilege escalation.
+    for f in &files {
+        let p = dir.join(f);
+        if p.exists() && !readable_nonempty(&p) {
+            tracing::warn!("regenerating unreadable/stale secret {}", p.display());
+            fs::remove_file(&p)
+                .with_context(|| format!("removing stale secret {}", p.display()))?;
+        }
+    }
+
+    match gs.kind {
+        SecretGenKind::Hex16 => write_secret(&dir.join(&gs.name), &random_hex(16))?,
+        SecretGenKind::Hex32 => write_secret(&dir.join(&gs.name), &random_hex(32))?,
+        SecretGenKind::Base64 => write_secret(&dir.join(&gs.name), &random_base64(32))?,
+        SecretGenKind::Bcrypt => write_bcrypt_pair(dir, &gs.name)?,
+    }
+    Ok(())
+}
+
+/// Generate a fresh bcrypt credential pair for `name` under `dir`: the
+/// server-facing hash at `<name>` and its plaintext sibling at `<name>.pw`,
+/// both 0600 through the atomic [`write_secret`].
+///
+/// The single implementation of bcrypt generation on this platform —
+/// [`ensure_one`]'s `Bcrypt` arm and
+/// [`rotate_compromised_gateway_credential`] both go through here, so there is
+/// one place where a credential comes into existence.
+fn write_bcrypt_pair(dir: &Path, name: &str) -> Result<()> {
+    let password = random_hex(BCRYPT_PASSWORD_BYTES);
+    let hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST)
+        .context("bcrypt-hashing generated password")?;
+    // Primary (server-facing hash) first, then the plaintext sibling.
+    write_secret(&dir.join(name), &hash)?;
+    write_secret(&dir.join(format!("{}.pw", name)), &password)?;
+    Ok(())
+}
+
+/// True when `path` exists, is readable by this process, and is non-empty after
+/// trimming. Any error (missing, permission denied, empty) reads as false.
+fn readable_nonempty(path: &Path) -> bool {
+    fs::read_to_string(path)
+        .map(|s| !s.trim().is_empty())
+        .unwrap_or(false)
+}
+
+/// Fill `buf` from an explicitly named `OsRng`, guarded when it is long enough
+/// for the degenerate predicate's false-positive bound to hold.
+///
+/// KEY-05 / F-10: these are the manifest-declared `generated_secrets` — app
+/// passwords and API keys — and were the original F-10 finding. Every production
+/// caller requests 16 or 32 bytes, so the guard is live in practice; the short
+/// branch exists so a future caller asking for fewer cannot trip the guard's
+/// length assertion, which is a programmer-error panic and not an input
+/// condition.
+fn fill_secret_bytes(buf: &mut [u8]) {
+    if buf.len() >= crate::entropy::MIN_GUARDED_LEN {
+        crate::entropy::draw_key_bytes(&mut rand::rngs::OsRng, buf).unwrap_or_else(|e| {
+            panic!("refusing to generate an app secret from degenerate entropy: {e} (KEY-05)")
+        });
+    } else {
+        rand::rngs::OsRng.fill_bytes(buf);
+    }
+}
+
+fn random_hex(bytes: usize) -> String {
+    let mut buf = vec![0u8; bytes];
+    fill_secret_bytes(&mut buf);
+    hex::encode(buf)
+}
+
+/// `bytes` of entropy, standard base64 (with padding). For keys that a service
+/// base64-decodes to recover the raw bytes (e.g. netbird's store encryptionKey).
+fn random_base64(bytes: usize) -> String {
+    use base64::Engine as _;
+    let mut buf = vec![0u8; bytes];
+    fill_secret_bytes(&mut buf);
+    base64::engine::general_purpose::STANDARD.encode(buf)
+}
+
+/// Canonical secret name for the Fedimint gateway's admin bcrypt hash — must
+/// match `generated_secrets: fedimint-gateway-hash` in
+/// `apps/fedimint-gateway/manifest.yml` so the Rust orchestrator, first-boot
+/// script, reconcile script and both deploy scripts all agree on one file
+/// (FED-07: before this, scripts wrote `fedimint-gateway-password` while the
+/// daemon read `fedimint-gateway-hash`).
+pub const GATEWAY_HASH_SECRET_NAME: &str = "fedimint-gateway-hash";
+
+/// Detection-only denylist of bcrypt hashes that shipped as hardcoded
+/// fallback credentials in this repository before FED-07. `t9YjjxkiktrlYvjajB
+/// /zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC` was substituted for the Fedimint
+/// gateway's admin password whenever the real per-install secret was
+/// missing — in `config.rs`, `dependencies.rs`, and every shell install path
+/// — meaning anyone holding a copy of this repo held the admin credential for
+/// every gateway that ever took that fallback.
+///
+/// This value exists **only** so an install still carrying it can be
+/// detected and rotated (plan 01-16 owns the migration). It must NEVER be
+/// passed to a container, written to a fresh install, or handed back to a
+/// caller by [`gateway_bcrypt_hash`] — that function returns `Err` instead.
+/// This is the one and only place this value may appear in the tree.
+const KNOWN_DEFAULT_GATEWAY_HASHES: &[&str] =
+    &["$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC"];
+
+/// Idempotently ensure the Fedimint gateway's admin credential exists under
+/// `secrets_dir`: a fresh per-install bcrypt hash plus its `.pw` plaintext
+/// sibling, both 0600. Delegates to [`ensure_one`] for the actual bcrypt
+/// generation so there is exactly one implementation of that logic — this
+/// also means a second call is a no-op (idempotent fast path) and a
+/// present-but-unreadable file self-heals, so a reconcile tick never rotates
+/// a working gateway credential out from under it.
+pub fn ensure_gateway_credential(secrets_dir: &Path) -> Result<()> {
+    fs::create_dir_all(secrets_dir)
+        .with_context(|| format!("creating secrets dir {}", secrets_dir.display()))?;
+    let gs = GeneratedSecret {
+        name: GATEWAY_HASH_SECRET_NAME.to_string(),
+        kind: SecretGenKind::Bcrypt,
+    };
+    ensure_one(secrets_dir, &gs)
+}
+
+/// Read the Fedimint gateway's canonical per-install bcrypt hash.
+///
+/// Returns `Err` naming the missing file when it is absent, empty, or
+/// unreadable — callers must propagate that error rather than substitute a
+/// literal, so an install with no credential fails loudly instead of quietly
+/// starting an unauthenticated/default-credentialed gateway. Also returns
+/// `Err` when the stored value matches [`KNOWN_DEFAULT_GATEWAY_HASHES`]: a
+/// node carrying the shipped default must not be handed that value back by
+/// this codebase, even to reconfigure itself with the same value it already
+/// (insecurely) has.
+pub fn gateway_bcrypt_hash(secrets_dir: &Path) -> Result<String> {
+    let path = secrets_dir.join(GATEWAY_HASH_SECRET_NAME);
+    let hash = fs::read_to_string(&path).with_context(|| {
+        format!(
+            "gateway credential missing at {} — call ensure_gateway_credential (or wait for the \
+             next reconcile tick) to generate a per-install credential before starting the gateway",
+            path.display()
+        )
+    })?;
+    let hash = hash.trim();
+    if hash.is_empty() {
+        anyhow::bail!("gateway credential {} is empty", path.display());
+    }
+    if KNOWN_DEFAULT_GATEWAY_HASHES.contains(&hash) {
+        anyhow::bail!(
+            "gateway credential {} is a publicly known default that shipped hardcoded in this \
+             repository before FED-07 — this install must rotate it (see plan 01-16) before the \
+             gateway can be (re)configured",
+            path.display()
+        );
+    }
+    Ok(hash.to_string())
+}
+
+/// Detect and rotate a Fedimint gateway credential that is a publicly known
+/// shipped default (FED-07 migration).
+///
+/// Returns `Ok(true)` only when the stored hash was an EXACT match for a
+/// [`KNOWN_DEFAULT_GATEWAY_HASHES`] entry and has been replaced with a freshly
+/// generated pair. An absent, unreadable, or simply unrecognised-but-unique
+/// value returns `Ok(false)` and writes nothing: rotation must never fire on
+/// "anything I did not generate this run", or an operator who deliberately set
+/// their own credential would have it silently replaced.
+///
+/// Generating a credential where none exists is
+/// [`ensure_gateway_credential`]'s job, not this function's.
+///
+/// **Rollback:** the replacement goes through [`write_secret`]'s atomic
+/// temp-file-plus-rename, so a failure part-way through leaves the previous
+/// credential file intact and the gateway keeps working with it. Do NOT
+/// "improve" this into a truncate-in-place or a remove-then-write — that turns
+/// a failed rotation into a gateway configured against a credential nobody
+/// holds.
+///
+/// **Self-terminating:** the value written is freshly generated and therefore
+/// not on the denylist, so the next reconcile tick detects nothing and changes
+/// nothing. Rotation happens at most once per affected node.
+pub fn rotate_compromised_gateway_credential(secrets_dir: &Path) -> Result<bool> {
+    let path = secrets_dir.join(GATEWAY_HASH_SECRET_NAME);
+    let Ok(current) = fs::read_to_string(&path) else {
+        // Absent or unreadable: nothing to rotate. ensure_gateway_credential
+        // owns materialising it.
+        return Ok(false);
+    };
+    if !KNOWN_DEFAULT_GATEWAY_HASHES.contains(&current.trim()) {
+        return Ok(false);
+    }
+    write_bcrypt_pair(secrets_dir, GATEWAY_HASH_SECRET_NAME).with_context(|| {
+        format!(
+            "rotating compromised gateway credential at {}",
+            path.display()
+        )
+    })?;
+    Ok(true)
+}
+
+/// Write an externally computed secret value (0600, atomic). For derived
+/// secrets that aren't random generators — e.g. the btcpay internal-LND
+/// connection string assembled in `container::lnd`.
+pub(crate) fn write_secret_file(path: &Path, value: &str) -> Result<()> {
+    if let Some(dir) = path.parent() {
+        fs::create_dir_all(dir)
+            .with_context(|| format!("creating secrets dir {}", dir.display()))?;
+    }
+    write_secret(path, value)
+}
+
+/// Atomically write a `0600` secret: a temp file in the same dir (so the rename
+/// is atomic), fsynced, then renamed over the target.
+fn write_secret(path: &Path, value: &str) -> Result<()> {
+    let dir = path
+        .parent()
+        .context("secret path has no parent directory")?;
+    let name = path
+        .file_name()
+        .and_then(|n| n.to_str())
+        .context("secret path has no filename")?;
+    let tmp = dir.join(format!(".{name}.tmp"));
+
+    let mut f = fs::OpenOptions::new()
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .mode(0o600)
+        .open(&tmp)
+        .with_context(|| format!("creating temp secret {}", tmp.display()))?;
+    f.write_all(value.as_bytes())
+        .with_context(|| format!("writing temp secret {}", tmp.display()))?;
+    f.sync_all()
+        .with_context(|| format!("fsync temp secret {}", tmp.display()))?;
+    drop(f);
+
+    fs::rename(&tmp, path)
+        .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use archipelago_container::SecretGenKind;
+    use std::os::unix::fs::PermissionsExt;
+
+    fn manifest_with(secrets: Vec<GeneratedSecret>) -> AppManifest {
+        let mut m: AppManifest = serde_yaml::from_str(
+            "app:\n  id: t\n  name: t\n  version: 1.0.0\n  container:\n    image: x:y\n",
+        )
+        .unwrap();
+        m.app.container.generated_secrets = secrets;
+        m
+    }
+
+    fn gs(name: &str, kind: SecretGenKind) -> GeneratedSecret {
+        GeneratedSecret {
+            name: name.to_string(),
+            kind,
+        }
+    }
+
+    #[test]
+    fn generates_hex_and_bcrypt_with_0600() {
+        let dir = tempfile::tempdir().unwrap();
+        let m = manifest_with(vec![
+            gs("tok", SecretGenKind::Hex16),
+            gs("admin", SecretGenKind::Bcrypt),
+        ]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+
+        let tok = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(tok.trim().len(), 32, "hex16 = 16 bytes = 32 hex chars");
+
+        let hash = std::fs::read_to_string(dir.path().join("admin")).unwrap();
+        let pw = std::fs::read_to_string(dir.path().join("admin.pw")).unwrap();
+        assert!(hash.starts_with("$2"), "bcrypt hash shape");
+        assert!(
+            bcrypt::verify(pw.trim(), hash.trim()).unwrap(),
+            "pw matches hash"
+        );
+
+        for f in ["tok", "admin", "admin.pw"] {
+            let mode = std::fs::metadata(dir.path().join(f))
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777;
+            assert_eq!(mode, 0o600, "{f} must be 0600");
+        }
+    }
+
+    #[test]
+    fn idempotent_value_is_stable() {
+        let dir = tempfile::tempdir().unwrap();
+        let m = manifest_with(vec![gs("tok", SecretGenKind::Hex32)]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let first = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let second = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(
+            first, second,
+            "a present readable secret is never rewritten"
+        );
+    }
+
+    #[test]
+    fn gateway_credential_fresh_generation_verifies_and_is_0600() {
+        let dir = tempfile::tempdir().unwrap();
+        ensure_gateway_credential(dir.path()).unwrap();
+
+        let hash = std::fs::read_to_string(dir.path().join(GATEWAY_HASH_SECRET_NAME)).unwrap();
+        let pw = std::fs::read_to_string(dir.path().join(format!("{GATEWAY_HASH_SECRET_NAME}.pw")))
+            .unwrap();
+        assert!(bcrypt::verify(pw.trim(), hash.trim()).unwrap());
+
+        for f in [
+            GATEWAY_HASH_SECRET_NAME.to_string(),
+            format!("{GATEWAY_HASH_SECRET_NAME}.pw"),
+        ] {
+            let mode = std::fs::metadata(dir.path().join(&f))
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777;
+            assert_eq!(mode, 0o600, "{f} must be 0600");
+        }
+
+        let read_back = gateway_bcrypt_hash(dir.path()).unwrap();
+        assert_eq!(read_back, hash.trim());
+    }
+
+    #[test]
+    fn gateway_credential_is_idempotent() {
+        let dir = tempfile::tempdir().unwrap();
+        ensure_gateway_credential(dir.path()).unwrap();
+        let first = gateway_bcrypt_hash(dir.path()).unwrap();
+        ensure_gateway_credential(dir.path()).unwrap();
+        let second = gateway_bcrypt_hash(dir.path()).unwrap();
+        assert_eq!(first, second, "second call must not rotate the credential");
+    }
+
+    #[test]
+    fn gateway_credential_missing_is_a_named_error() {
+        let dir = tempfile::tempdir().unwrap();
+        let err = gateway_bcrypt_hash(dir.path()).unwrap_err();
+        assert!(
+            err.to_string().contains(GATEWAY_HASH_SECRET_NAME),
+            "error must name the missing secret file: {err}"
+        );
+    }
+
+    #[test]
+    fn gateway_credential_rejects_known_default() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+        let err = gateway_bcrypt_hash(dir.path()).unwrap_err();
+        assert!(
+            err.to_string().to_lowercase().contains("default"),
+            "error must explain the denylisted value: {err}"
+        );
+    }
+
+    #[test]
+    fn gateway_credential_is_per_install_not_per_build() {
+        let dir_a = tempfile::tempdir().unwrap();
+        let dir_b = tempfile::tempdir().unwrap();
+        ensure_gateway_credential(dir_a.path()).unwrap();
+        ensure_gateway_credential(dir_b.path()).unwrap();
+        let hash_a = gateway_bcrypt_hash(dir_a.path()).unwrap();
+        let hash_b = gateway_bcrypt_hash(dir_b.path()).unwrap();
+        assert_ne!(hash_a, hash_b, "two fresh installs must not share a hash");
+    }
+
+    // ── FED-07 migration: rotating a shipped default off an existing node ──
+
+    #[test]
+    fn rotates_a_denylisted_gateway_credential() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+
+        // The new value is readable through the normal accessor, which means
+        // it is neither empty nor still denylisted.
+        let rotated = gateway_bcrypt_hash(dir.path()).unwrap();
+        assert!(!KNOWN_DEFAULT_GATEWAY_HASHES.contains(&rotated.as_str()));
+
+        // The plaintext sibling was written too and verifies against the hash,
+        // so the operator can actually get back into the gateway.
+        let pw = std::fs::read_to_string(dir.path().join(format!("{GATEWAY_HASH_SECRET_NAME}.pw")))
+            .unwrap();
+        assert!(bcrypt::verify(pw.trim(), rotated.trim()).unwrap());
+
+        for f in [
+            GATEWAY_HASH_SECRET_NAME.to_string(),
+            format!("{GATEWAY_HASH_SECRET_NAME}.pw"),
+        ] {
+            let mode = std::fs::metadata(dir.path().join(&f))
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777;
+            assert_eq!(mode, 0o600, "{f} must stay 0600 after rotation");
+        }
+    }
+
+    #[test]
+    fn leaves_a_unique_gateway_credential_alone() {
+        let dir = tempfile::tempdir().unwrap();
+        ensure_gateway_credential(dir.path()).unwrap();
+        let before = gateway_bcrypt_hash(dir.path()).unwrap();
+
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(before, gateway_bcrypt_hash(dir.path()).unwrap());
+    }
+
+    #[test]
+    fn leaves_an_unrecognised_credential_alone() {
+        // The adjacency edge that matters: an operator's own hand-set value is
+        // not on the denylist and must survive. Rotation is denylist-exact,
+        // never "anything I did not generate".
+        let dir = tempfile::tempdir().unwrap();
+        let operator_set = "$2y$10$operatorChosenValueThatWeMustNeverTouchAAAAAAAAAAAAAAAAAAAAA";
+        std::fs::write(dir.path().join(GATEWAY_HASH_SECRET_NAME), operator_set).unwrap();
+
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(
+            std::fs::read_to_string(dir.path().join(GATEWAY_HASH_SECRET_NAME)).unwrap(),
+            operator_set
+        );
+    }
+
+    #[test]
+    fn no_op_when_no_gateway_credential_exists() {
+        let dir = tempfile::tempdir().unwrap();
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert!(!dir.path().join(GATEWAY_HASH_SECRET_NAME).exists());
+    }
+
+    #[test]
+    fn rotation_is_idempotent() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+        let after_first = gateway_bcrypt_hash(dir.path()).unwrap();
+
+        // Second tick: nothing detected, nothing changed. This is what stops a
+        // reconcile loop from recreating the gateway on every pass.
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(after_first, gateway_bcrypt_hash(dir.path()).unwrap());
+    }
+
+    #[test]
+    fn rotation_touches_no_other_secret() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+        let bystanders = [
+            ("mempool-db-password", "mempool-value"),
+            ("immich-db-password", "immich-value"),
+            ("fmcd-password", "fmcd-value"),
+            ("bitcoin-rpc-password", "bitcoin-value"),
+        ];
+        for (name, value) in bystanders {
+            std::fs::write(dir.path().join(name), value).unwrap();
+        }
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+
+        for (name, value) in bystanders {
+            assert_eq!(
+                std::fs::read_to_string(dir.path().join(name)).unwrap(),
+                value,
+                "{name} must be byte-identical after a gateway rotation"
+            );
+        }
+    }
+
+    #[test]
+    fn self_heals_unreadable_secret() {
+        // Simulate the root-owned case: a present-but-unreadable file. We can't
+        // chmod-away read as the owner in a unit test, so emulate "unreadable"
+        // via the empty-file branch (readable_nonempty == false), which drives
+        // the same unlink+regenerate path.
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(dir.path().join("tok"), "").unwrap();
+        let m = manifest_with(vec![gs("tok", SecretGenKind::Hex16)]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let v = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(v.trim().len(), 32, "stale/empty secret was regenerated");
+    }
+}

core/archipelago/src/container/traits.rs

@@ -0,0 +1,93 @@
+//! Orchestrator trait — the shared surface the RPC layer talks to.
+//!
+//! Step 4 of the rust-orchestrator migration. Unifies the container lifecycle
+//! surface of `DevContainerOrchestrator` and `ProdContainerOrchestrator` so
+//! `RpcHandler` can hold `Arc<dyn ContainerOrchestrator>` and stop caring
+//! which mode it is in.
+//!
+//! The trait takes `app_id: &str` everywhere (never a manifest path). Dev and
+//! Prod both resolve app_id → manifest internally. The legacy
+//! `container-install { manifest_path }` RPC shape is preserved as a concrete
+//! `install_container_from_path` method on `DevContainerOrchestrator` only,
+//! since that ad-hoc workflow is a dev convenience and has no prod meaning.
+//!
+//! See `docs/rust-orchestrator-migration.md`.
+
+use anyhow::Result;
+use archipelago_container::ContainerStatus;
+use async_trait::async_trait;
+
+/// Lifecycle + query operations every orchestrator exposes to the RPC layer.
+#[async_trait]
+pub trait ContainerOrchestrator: Send + Sync {
+    /// Build-or-pull the image, create the container, and start it. Returns the
+    /// podman container name that was created. Assumes the app_id corresponds
+    /// to a manifest the orchestrator already knows about.
+    async fn install(&self, app_id: &str) -> Result<String>;
+
+    /// True when this orchestrator holds a manifest for `app_id` (disk or
+    /// signed-catalog overlay) — i.e. `install(app_id)` would not fail with
+    /// "unknown app_id". Lets the RPC layer route any manifest-driven app
+    /// through the orchestrator without a per-app allowlist. Defaults to
+    /// `false` so orchestrators without a manifest registry keep routing
+    /// through the legacy install flow.
+    async fn knows_app(&self, _app_id: &str) -> bool {
+        false
+    }
+
+    /// Rebuild the in-memory manifest map (disk + signed-catalog overlay).
+    /// Called after a runtime catalog refresh detects changed bytes so catalog
+    /// manifest changes take effect without a service restart — without this,
+    /// `load_manifests` only runs at startup and a freshly published manifest
+    /// sits dormant until the next restart. Returns the merged manifest count.
+    /// Defaults to a no-op for orchestrators without a manifest registry.
+    async fn reload_manifests(&self) -> Result<usize> {
+        Ok(0)
+    }
+
+    /// Start an already-created container.
+    async fn start(&self, app_id: &str) -> Result<()>;
+
+    /// Stop a running container. No-op on Prod if already stopped.
+    async fn stop(&self, app_id: &str) -> Result<()>;
+
+    /// Stop-then-start. Best-effort: ignores stop failure.
+    async fn restart(&self, app_id: &str) -> Result<()>;
+
+    /// Remove the container. `preserve_data = true` keeps the volumes; `false`
+    /// is honored on a best-effort basis (Dev cleans, Prod leaves the volume
+    /// management to the data layer).
+    async fn remove(&self, app_id: &str, preserve_data: bool) -> Result<()>;
+
+    /// Pull/rebuild the image and recreate the container from scratch.
+    async fn upgrade(&self, app_id: &str) -> Result<()>;
+
+    /// Current state of a single container.
+    async fn status(&self, app_id: &str) -> Result<ContainerStatus>;
+
+    /// All containers this orchestrator knows about.
+    async fn list(&self) -> Result<Vec<ContainerStatus>>;
+
+    /// Tail the container's stdout+stderr.
+    async fn logs(&self, app_id: &str, lines: u32) -> Result<Vec<String>>;
+
+    /// Coarse health summary: "healthy", "unhealthy", "starting", "paused", "unknown".
+    async fn health(&self, app_id: &str) -> Result<String>;
+
+    /// Declare that a credential this app consumes has just been rotated, so
+    /// the running container is now holding an invalid one.
+    ///
+    /// Restart-sensitivity normally protects apps like `btcpay-server` from
+    /// being recreated on drift — correct when the running container is
+    /// working, and exactly wrong when it is working only in appearance. After
+    /// an LND macaroon rotation, BTCPay is up and healthy while every Lightning
+    /// operation it attempts fails against a credential LND no longer honours;
+    /// leaving it untouched perpetuates the breakage rather than protecting
+    /// anything. This is the same carve-out FED-07 uses for the Fedimint
+    /// gateway, reached from the RPC layer instead of from inside a reconcile.
+    ///
+    /// Consumed by the next drift check, which recreates the container around
+    /// its unchanged data directory, ports and volumes. Default no-op: an
+    /// orchestrator without restart-sensitivity has nothing to override.
+    async fn mark_credential_rotated(&self, _app_id: &str) {}
+}

core/archipelago/src/container/ui_detection.rs

@@ -0,0 +1,243 @@
+//! Web-UI detection for discovered containers.
+//!
+//! The packages list used to mark every container that published a port (or
+//! carried an onion address) as a UI app, which gave headless backends —
+//! databases, media servers, self-deployed compose stacks — a Launch button.
+//! UI-ness is decided here instead, for manifest apps and ad-hoc containers
+//! alike:
+//!
+//! 1. A manifest that declares an `interfaces:` block is definitive: any
+//!    entry of `type: ui` means a browsable UI, a block without one means a
+//!    backend service. The signed catalog overlay is consulted before disk
+//!    manifests (catalog supremacy).
+//! 2. Manifests without an `interfaces:` block (the overwhelming majority)
+//!    and manifest-less containers fall through to a short HTTP probe of the
+//!    launch port: an HTML page, a redirect, or a browser-auth wall means a
+//!    UI; JSON APIs, raw TCP protocols, and dead ports mean a service.
+//!
+//! Verdicts are cached — positives longer than negatives, since "no UI yet"
+//! is often just an app that hasn't finished starting.
+
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+use std::time::{Duration, Instant};
+
+use tracing::debug;
+
+const PROBE_TIMEOUT: Duration = Duration::from_secs(2);
+/// A confirmed UI stays a UI — re-check rarely.
+const POSITIVE_TTL: Duration = Duration::from_secs(15 * 60);
+/// A "no UI" verdict may be a slow-starting app — re-check sooner.
+const NEGATIVE_TTL: Duration = Duration::from_secs(2 * 60);
+
+/// Decide whether `app_id` exposes a browsable web UI. `lan_address` is the
+/// launch candidate already computed by the package scanner (host-published),
+/// and `running` gates the probe: a stopped container can't answer, and a
+/// dead-port verdict against it would poison the cache.
+pub async fn has_web_ui(app_id: &str, lan_address: Option<&str>, running: bool) -> bool {
+    if let Some(declared) = manifest_declares_ui(app_id) {
+        return declared;
+    }
+    let Some(port) = lan_address.and_then(super::docker_packages::launch_url_port) else {
+        return false;
+    };
+    if !running {
+        // Serve a cached verdict if we have one, but never record one.
+        return cached_verdict(app_id, port).unwrap_or(false);
+    }
+    if let Some(v) = cached_verdict(app_id, port) {
+        return v;
+    }
+    let verdict = probe_port(port).await;
+    debug!(app_id, port, verdict, "web-UI probe");
+    cache_verdict(app_id, port, verdict);
+    verdict
+}
+
+// ─── Manifest declarations ───────────────────────────────────────────────
+
+/// `Some(true)`: a manifest covers this app and declares a `type: ui`
+/// interface. `Some(false)`: a manifest declares interfaces, none of them UI.
+/// `None`: no manifest, or one without an `interfaces:` block — undeclared,
+/// let the probe decide.
+fn manifest_declares_ui(app_id: &str) -> Option<bool> {
+    // Catalog overlay first: disk manifests don't apply to catalog-covered
+    // apps, so the catalog must win where it speaks.
+    for (id, value) in super::app_catalog::catalog_manifest_values() {
+        if id == app_id {
+            if let Some(v) = declared_ui_in_value(&value) {
+                return Some(v);
+            }
+            break;
+        }
+    }
+    for path in disk_manifest_candidates(app_id) {
+        if !path.exists() {
+            continue;
+        }
+        match archipelago_container::AppManifest::from_file(&path) {
+            Ok(m) => {
+                if m.app.interfaces.is_empty() {
+                    return None;
+                }
+                return Some(m.app.interfaces.values().any(|i| i.interface_type == "ui"));
+            }
+            // Malformed manifests are already reported by the orchestrator's
+            // loader; here they simply don't count as a declaration.
+            Err(_) => return None,
+        }
+    }
+    None
+}
+
+/// Same declaration logic against a catalog manifest carried as raw JSON.
+fn declared_ui_in_value(manifest: &serde_json::Value) -> Option<bool> {
+    let interfaces = manifest.get("app")?.get("interfaces")?.as_object()?;
+    if interfaces.is_empty() {
+        return None;
+    }
+    Some(interfaces.values().any(|i| {
+        // An omitted `type` defaults to "ui", mirroring the YAML schema.
+        i.get("type")
+            .and_then(|t| t.as_str())
+            .map(|t| t == "ui")
+            .unwrap_or(true)
+    }))
+}
+
+fn disk_manifest_candidates(app_id: &str) -> Vec<std::path::PathBuf> {
+    let mut roots: Vec<std::path::PathBuf> = Vec::new();
+    if let Ok(v) = std::env::var("ARCHIPELAGO_APPS_DIR") {
+        let v = v.trim();
+        if !v.is_empty() {
+            roots.push(v.into());
+        }
+    }
+    roots.push("/opt/archipelago/apps".into());
+    roots
+        .into_iter()
+        .map(|root| root.join(app_id).join("manifest.yml"))
+        .collect()
+}
+
+// ─── HTTP probe ──────────────────────────────────────────────────────────
+
+async fn probe_port(port: u16) -> bool {
+    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+    let client = CLIENT.get_or_init(|| {
+        reqwest::Client::builder()
+            .timeout(PROBE_TIMEOUT)
+            .redirect(reqwest::redirect::Policy::none())
+            .build()
+            .expect("static probe client")
+    });
+    match client.get(format!("http://127.0.0.1:{port}/")).send().await {
+        Ok(resp) => {
+            let content_type = resp
+                .headers()
+                .get(reqwest::header::CONTENT_TYPE)
+                .and_then(|v| v.to_str().ok())
+                .unwrap_or("");
+            let auth_challenge = resp
+                .headers()
+                .contains_key(reqwest::header::WWW_AUTHENTICATE);
+            ui_verdict(resp.status().as_u16(), content_type, auth_challenge)
+        }
+        // Connection refused, timeout, or a non-HTTP protocol on the port.
+        Err(_) => false,
+    }
+}
+
+/// The pure classification rule, split out for tests: a browsable UI is an
+/// HTML response (any status — error pages included), a redirect (login
+/// flows), or a browser auth prompt. JSON/plaintext APIs are services.
+fn ui_verdict(status: u16, content_type: &str, auth_challenge: bool) -> bool {
+    if content_type.contains("text/html") {
+        return true;
+    }
+    if (300..400).contains(&status) {
+        return true;
+    }
+    if (status == 401 || status == 403) && auth_challenge {
+        return true;
+    }
+    false
+}
+
+// ─── Verdict cache ───────────────────────────────────────────────────────
+
+fn cache() -> &'static Mutex<HashMap<String, (bool, Instant)>> {
+    static CACHE: OnceLock<Mutex<HashMap<String, (bool, Instant)>>> = OnceLock::new();
+    CACHE.get_or_init(Default::default)
+}
+
+fn cached_verdict(app_id: &str, port: u16) -> Option<bool> {
+    let cache = cache().lock().ok()?;
+    let (verdict, at) = cache.get(&format!("{app_id}:{port}"))?;
+    let ttl = if *verdict { POSITIVE_TTL } else { NEGATIVE_TTL };
+    (at.elapsed() < ttl).then_some(*verdict)
+}
+
+fn cache_verdict(app_id: &str, port: u16, verdict: bool) {
+    if let Ok(mut cache) = cache().lock() {
+        cache.insert(format!("{app_id}:{port}"), (verdict, Instant::now()));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn html_is_ui_regardless_of_status() {
+        assert!(ui_verdict(200, "text/html; charset=utf-8", false));
+        assert!(ui_verdict(404, "text/html", false));
+        assert!(ui_verdict(401, "text/html", false));
+    }
+
+    #[test]
+    fn redirects_are_ui() {
+        assert!(ui_verdict(302, "", false));
+        assert!(ui_verdict(307, "text/plain", false));
+    }
+
+    #[test]
+    fn auth_walls_are_ui_only_with_challenge() {
+        assert!(ui_verdict(401, "text/plain", true));
+        assert!(!ui_verdict(401, "application/json", false));
+    }
+
+    #[test]
+    fn apis_and_plain_ports_are_services() {
+        assert!(!ui_verdict(200, "application/json", false));
+        assert!(!ui_verdict(200, "text/plain", false));
+        assert!(!ui_verdict(500, "", false));
+    }
+
+    #[test]
+    fn catalog_value_declaration() {
+        let ui = json!({"app": {"interfaces": {"main": {"type": "ui", "port": 80}}}});
+        assert_eq!(declared_ui_in_value(&ui), Some(true));
+
+        // Omitted type defaults to "ui", mirroring the YAML schema default.
+        let default_ty = json!({"app": {"interfaces": {"main": {"port": 80}}}});
+        assert_eq!(declared_ui_in_value(&default_ty), Some(true));
+
+        let api_only = json!({"app": {"interfaces": {"rpc": {"type": "api", "port": 80}}}});
+        assert_eq!(declared_ui_in_value(&api_only), Some(false));
+
+        // No interfaces block ⇒ undeclared, not "no UI".
+        let none = json!({"app": {"id": "x"}});
+        assert_eq!(declared_ui_in_value(&none), None);
+        let empty = json!({"app": {"interfaces": {}}});
+        assert_eq!(declared_ui_in_value(&empty), None);
+    }
+
+    #[test]
+    fn verdict_cache_roundtrip() {
+        cache_verdict("test-app", 1234, true);
+        assert_eq!(cached_verdict("test-app", 1234), Some(true));
+        assert_eq!(cached_verdict("test-app", 9999), None);
+    }
+}

core/archipelago/src/container/version_config.rs

@@ -0,0 +1,272 @@
+//! Per-app version preferences — the persistence layer for multi-version support.
+//!
+//! Multi-version support (`docs/bitcoin-multi-version-design.md`) lets a node
+//! runner pin Bitcoin Core / Knots to a specific version and opt into
+//! auto-update-to-latest. Both choices live in the existing per-app config file
+//! at `/var/lib/archipelago/app-configs/<id>.json` as two keys:
+//!
+//! ```jsonc
+//! { "pinnedVersion": "29.3.knots20260508", "autoUpdate": false }
+//! ```
+//!
+//! This is the single source of truth the orchestrator's install path reads to
+//! resolve the image, and that the auto-update tick + "available update" badge
+//! consult. Reads/writes are merge-preserving so they never clobber any
+//! `containerConfig` (ports/volumes/env) a generic app may also store here.
+//!
+//! Platform-managed apps (bitcoin-core/knots/…) never use the
+//! `containerConfig`-style keys (see `config.rs::dynamic_app_config`, which
+//! returns early for them), so adding these keys to their file is collision-free.
+
+use serde_json::{Map, Value};
+use std::path::PathBuf;
+
+/// Resolved version preferences for one app. Defaults: no pin, auto-update off
+/// (consensus-critical apps opt in explicitly — design open-question #4).
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AppVersionConfig {
+    /// The version string the runner pinned, if any. Suppresses the update badge
+    /// and overrides the catalog default at install/recreate time.
+    pub pinned_version: Option<String>,
+    /// When true, the hourly catalog tick updates this app to the catalog
+    /// default automatically. Ignored while a version is pinned.
+    pub auto_update: bool,
+}
+
+fn config_dir() -> PathBuf {
+    let base = std::env::var("ARCHIPELAGO_DATA_DIR")
+        .unwrap_or_else(|_| "/var/lib/archipelago".to_string());
+    PathBuf::from(base).join("app-configs")
+}
+
+fn config_path(app_id: &str) -> PathBuf {
+    config_dir().join(format!("{app_id}.json"))
+}
+
+/// App ids that have opted into auto-update-to-latest AND are not pinned (a pin
+/// is an explicit "stay here"). Drives the hourly per-app auto-update tick. The
+/// app id is the config file stem. Returns empty when the dir is absent.
+pub fn auto_update_apps() -> Vec<String> {
+    let mut out = Vec::new();
+    let Ok(entries) = std::fs::read_dir(config_dir()) else {
+        return out;
+    };
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if path.extension().and_then(|e| e.to_str()) != Some("json") {
+            continue;
+        }
+        let Some(app_id) = path.file_stem().and_then(|s| s.to_str()) else {
+            continue;
+        };
+        let cfg = read(app_id);
+        if cfg.auto_update && cfg.pinned_version.is_none() {
+            out.push(app_id.to_string());
+        }
+    }
+    out
+}
+
+fn read_raw(app_id: &str) -> Map<String, Value> {
+    let path = config_path(app_id);
+    match std::fs::read_to_string(&path) {
+        Ok(s) => serde_json::from_str::<Value>(&s)
+            .ok()
+            .and_then(|v| v.as_object().cloned())
+            .unwrap_or_default(),
+        Err(_) => Map::new(),
+    }
+}
+
+/// Read the version preferences for `app_id`. Returns defaults when the file is
+/// absent or the keys are unset.
+pub fn read(app_id: &str) -> AppVersionConfig {
+    let obj = read_raw(app_id);
+    AppVersionConfig {
+        pinned_version: obj
+            .get("pinnedVersion")
+            .and_then(Value::as_str)
+            .filter(|s| !s.is_empty())
+            .map(String::from),
+        auto_update: obj
+            .get("autoUpdate")
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+    }
+}
+
+/// The pinned version for `app_id`, if set. Convenience for the hot path.
+pub fn pinned_version(app_id: &str) -> Option<String> {
+    read(app_id).pinned_version
+}
+
+/// Parse the leading numeric `major.minor.patch` of a version string into a
+/// comparable tuple. Stops at the first non-numeric component, so Bitcoin Core
+/// (`31.0`, `28.4`) and the Knots date-suffixed form (`29.3.knots20260508` →
+/// `(29, 3, 0)`) both compare on their consensus-relevant major/minor. The
+/// Knots build-date suffix is intentionally ignored — a same-major.minor Knots
+/// rebuild is not a chainstate downgrade.
+fn version_key(version: &str) -> (u64, u64, u64) {
+    let mut it = version.split('.').map(|c| {
+        // Take the leading digit run of each dotted component (`knots20260508`
+        // yields no leading digits → 0; `3` → 3).
+        c.chars()
+            .take_while(|ch| ch.is_ascii_digit())
+            .collect::<String>()
+            .parse::<u64>()
+            .unwrap_or(0)
+    });
+    (
+        it.next().unwrap_or(0),
+        it.next().unwrap_or(0),
+        it.next().unwrap_or(0),
+    )
+}
+
+/// True when installing `candidate` over `current` is a DOWNGRADE — an older
+/// Bitcoin release over a chainstate written by a newer one. This is the
+/// highest-risk operation (Core refuses to start on a newer chainstate without
+/// an expensive reindex; pruned nodes can lose data), so the UI must warn and
+/// the switch must be explicitly confirmed (design §4). Equal or newer → false.
+pub fn is_downgrade(current: &str, candidate: &str) -> bool {
+    version_key(candidate) < version_key(current)
+}
+
+/// Merge `cfg` into the on-disk config, preserving every other key. A
+/// `pinned_version` of `None` removes the `pinnedVersion` key (un-pins / "track
+/// latest"). Creates the directory and file on first write.
+pub fn write(app_id: &str, cfg: &AppVersionConfig) -> std::io::Result<()> {
+    let path = config_path(app_id);
+    let mut obj = read_raw(app_id);
+    match &cfg.pinned_version {
+        Some(v) => {
+            obj.insert("pinnedVersion".to_string(), Value::String(v.clone()));
+        }
+        None => {
+            obj.remove("pinnedVersion");
+        }
+    }
+    obj.insert("autoUpdate".to_string(), Value::Bool(cfg.auto_update));
+
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+    let serialized = serde_json::to_string_pretty(&Value::Object(obj))
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+    // Atomic-ish write: temp + rename so a crash mid-write can't truncate config.
+    let tmp = path.with_extension("json.tmp");
+    std::fs::write(&tmp, serialized.as_bytes())?;
+    std::fs::rename(&tmp, &path)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // `ARCHIPELAGO_DATA_DIR` is process-global, so the write/read tests must not
+    // run concurrently — serialize them and give each a unique dir. Without this
+    // lock, parallel `cargo test` races on the env var (poisoning is fine: a
+    // panicking test still releases a usable guard).
+    static ENV_LOCK: std::sync::Mutex<u64> = std::sync::Mutex::new(0);
+
+    fn with_tmp_data_dir<F: FnOnce()>(f: F) {
+        let mut counter = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        *counter += 1;
+        let dir =
+            std::env::temp_dir().join(format!("archy-vc-test-{}-{}", std::process::id(), *counter));
+        let _ = std::fs::remove_dir_all(&dir);
+        std::fs::create_dir_all(&dir).unwrap();
+        std::env::set_var("ARCHIPELAGO_DATA_DIR", &dir);
+        f();
+        std::env::remove_var("ARCHIPELAGO_DATA_DIR");
+        let _ = std::fs::remove_dir_all(&dir);
+        // `counter` guard drops here, releasing the lock for the next test.
+    }
+
+    #[test]
+    fn defaults_when_absent() {
+        with_tmp_data_dir(|| {
+            let cfg = read("bitcoin-core");
+            assert_eq!(cfg.pinned_version, None);
+            assert!(!cfg.auto_update);
+        });
+    }
+
+    #[test]
+    fn write_then_read_roundtrips() {
+        with_tmp_data_dir(|| {
+            write(
+                "bitcoin-knots",
+                &AppVersionConfig {
+                    pinned_version: Some("29.3.knots20260508".into()),
+                    auto_update: false,
+                },
+            )
+            .unwrap();
+            let cfg = read("bitcoin-knots");
+            assert_eq!(cfg.pinned_version.as_deref(), Some("29.3.knots20260508"));
+            assert!(!cfg.auto_update);
+        });
+    }
+
+    #[test]
+    fn write_preserves_existing_keys() {
+        with_tmp_data_dir(|| {
+            // Simulate a generic app's containerConfig already on disk.
+            let path = config_path("someapp");
+            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+            std::fs::write(&path, r#"{"ports":["80:80"],"autoUpdate":false}"#).unwrap();
+            write(
+                "someapp",
+                &AppVersionConfig {
+                    pinned_version: Some("1.2.3".into()),
+                    auto_update: true,
+                },
+            )
+            .unwrap();
+            let raw = read_raw("someapp");
+            assert!(raw.contains_key("ports"), "ports key must survive");
+            assert_eq!(raw.get("pinnedVersion").unwrap(), "1.2.3");
+            assert_eq!(raw.get("autoUpdate").unwrap(), &Value::Bool(true));
+        });
+    }
+
+    #[test]
+    fn downgrade_detection() {
+        // Older over newer = downgrade.
+        assert!(is_downgrade("31.0", "30.0"));
+        assert!(is_downgrade("28.4", "27.2"));
+        // Same or newer = not a downgrade.
+        assert!(!is_downgrade("30.0", "31.0"));
+        assert!(!is_downgrade("28.4", "28.4"));
+        // Knots date-suffixed strings compare on major.minor only.
+        assert!(is_downgrade("29.3.knots20260508", "28.1.knots20251010"));
+        assert!(!is_downgrade("29.3.knots20260101", "29.3.knots20260508"));
+    }
+
+    #[test]
+    fn unpin_removes_key() {
+        with_tmp_data_dir(|| {
+            write(
+                "bitcoin-core",
+                &AppVersionConfig {
+                    pinned_version: Some("31.0".into()),
+                    auto_update: true,
+                },
+            )
+            .unwrap();
+            write(
+                "bitcoin-core",
+                &AppVersionConfig {
+                    pinned_version: None,
+                    auto_update: true,
+                },
+            )
+            .unwrap();
+            let raw = read_raw("bitcoin-core");
+            assert!(!raw.contains_key("pinnedVersion"));
+            assert_eq!(read("bitcoin-core").pinned_version, None);
+            assert!(read("bitcoin-core").auto_update);
+        });
+    }
+}