From 140b7038387bb23b0b0e52f3bc42cb9ca225e3db Mon Sep 17 00:00:00 2001
From: archipelago <archipelago@localhost>
Date: Fri, 10 Jul 2026 12:39:24 -0400
Subject: [PATCH] feat(orchestrator): surface apps running with failed
 dependencies
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

btcpay-server started and reported healthy while its declared
dependencies (archy-btcpay-db, archy-nbxplorer) failed every boot
reconcile (live-testing report 2026-07-10, Fix 3). After each
reconcile pass, flag every app the pass left up whose manifest
dependency landed in the failure list — a loud DEGRADED journal error
instead of silence. True start-gating on dependency health is a
larger design change (boot-ordering/deadlock risk) and stays open.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../src/container/prod_orchestrator.rs        | 101 ++++++++++++++++++
 1 file changed, 101 insertions(+)

diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs
index 78a36479..a04ff1ab 100644
--- a/core/archipelago/src/container/prod_orchestrator.rs
+++ b/core/archipelago/src/container/prod_orchestrator.rs
@@ -1032,6 +1032,44 @@ fn resolve_catalog_image(resolved_manifest: &mut AppManifest) {
     }
 }
 
+/// (running app, failed dependency) pairs after a reconcile pass: apps the
+/// pass left up whose declared manifest dependency FAILED to reconcile.
+/// btcpay-server came up "healthy" while archy-btcpay-db/archy-nbxplorer
+/// never made it (live-testing 2026-07-10) — until real start-gating exists,
+/// surface the degraded state loudly instead of silently.
+fn degraded_running_apps<'m>(
+    report: &ReconcileReport,
+    manifests: impl Iterator<Item = &'m AppManifest>,
+) -> Vec<(String, String)> {
+    let failed: HashSet<&str> = report.failures.iter().map(|(id, _)| id.as_str()).collect();
+    if failed.is_empty() {
+        return Vec::new();
+    }
+    let up: HashSet<&str> = report
+        .actions
+        .iter()
+        .filter(|(_, action)| {
+            matches!(
+                action,
+                ReconcileAction::NoOp | ReconcileAction::Started | ReconcileAction::Installed
+            )
+        })
+        .map(|(id, _)| id.as_str())
+        .collect();
+    let mut out = Vec::new();
+    for m in manifests {
+        if !up.contains(m.app.id.as_str()) {
+            continue;
+        }
+        for dep in manifest_dependency_app_ids(m) {
+            if failed.contains(dep.as_str()) {
+                out.push((m.app.id.clone(), dep));
+            }
+        }
+    }
+    out
+}
+
 /// Host-port collisions across a set of manifests, excluding pairs that are
 /// mutually exclusive by design (the bitcoin-core/knots variants; the legacy
 /// mempool umbrella vs its split frontend, which render the same container).
@@ -1721,6 +1759,20 @@ impl ProdContainerOrchestrator {
             }
         }
 
+        {
+            let state = self.state.read().await;
+            for (app, dep) in degraded_running_apps(
+                &report,
+                state.manifests.values().map(|lm| &lm.manifest),
+            ) {
+                tracing::error!(
+                    app_id = %app,
+                    dependency = %dep,
+                    "app is up but its declared dependency failed to reconcile — running DEGRADED"
+                );
+            }
+        }
+
         report
     }
 
@@ -5310,6 +5362,55 @@ app:
         assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
     }
 
+    #[test]
+    fn degraded_running_apps_flags_up_app_with_failed_dependency() {
+        let btcpay = AppManifest::parse(
+            "app:\n  id: btcpay-server\n  name: btcpay\n  version: 1.0.0\n  container:\n    image: btcpay:1\n  dependencies:\n    - app_id: archy-nbxplorer\n      version: \">=1\"\n    - app_id: archy-btcpay-db\n      version: \">=15\"\n",
+        )
+        .unwrap();
+        let nbx = pull_manifest("archy-nbxplorer", "nbx:1");
+        let db = pull_manifest("archy-btcpay-db", "pg:15");
+
+        // btcpay up, both deps failed → two degraded pairs.
+        let report = ReconcileReport {
+            actions: vec![("btcpay-server".to_string(), ReconcileAction::NoOp)],
+            failures: vec![
+                ("archy-nbxplorer".to_string(), "pulling ...".to_string()),
+                ("archy-btcpay-db".to_string(), "pulling ...".to_string()),
+            ],
+        };
+        let found = degraded_running_apps(&report, [&btcpay, &nbx, &db].into_iter());
+        assert_eq!(
+            found,
+            vec![
+                ("btcpay-server".to_string(), "archy-nbxplorer".to_string()),
+                ("btcpay-server".to_string(), "archy-btcpay-db".to_string()),
+            ]
+        );
+
+        // Deps fine → nothing flagged.
+        let clean = ReconcileReport {
+            actions: vec![
+                ("btcpay-server".to_string(), ReconcileAction::NoOp),
+                ("archy-nbxplorer".to_string(), ReconcileAction::NoOp),
+                ("archy-btcpay-db".to_string(), ReconcileAction::NoOp),
+            ],
+            failures: vec![],
+        };
+        assert!(degraded_running_apps(&clean, [&btcpay, &nbx, &db].into_iter()).is_empty());
+
+        // The app itself failed too → not "up", not flagged (its own failure
+        // is already reported).
+        let both_down = ReconcileReport {
+            actions: vec![],
+            failures: vec![
+                ("btcpay-server".to_string(), "x".to_string()),
+                ("archy-nbxplorer".to_string(), "x".to_string()),
+            ],
+        };
+        assert!(degraded_running_apps(&both_down, [&btcpay, &nbx, &db].into_iter()).is_empty());
+    }
+
     #[test]
     fn host_port_collisions_flags_pairs_and_skips_exclusive_variants() {
         let with_port = |id: &str, port: u16| {