feat(orchestrator): surface apps running with failed dependencies

btcpay-server started and reported healthy while its declared
dependencies (archy-btcpay-db, archy-nbxplorer) failed every boot
reconcile (live-testing report 2026-07-10, Fix 3). After each
reconcile pass, flag every app the pass left up whose manifest
dependency landed in the failure list — a loud DEGRADED journal error
instead of silence. True start-gating on dependency health is a
larger design change (boot-ordering/deadlock risk) and stays open.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-10 12:39:24 -04:00
parent ebd904262e
commit 140b703838

View File

@ -1032,6 +1032,44 @@ fn resolve_catalog_image(resolved_manifest: &mut AppManifest) {
} }
} }
/// (running app, failed dependency) pairs after a reconcile pass: apps the
/// pass left up whose declared manifest dependency FAILED to reconcile.
/// btcpay-server came up "healthy" while archy-btcpay-db/archy-nbxplorer
/// never made it (live-testing 2026-07-10) — until real start-gating exists,
/// surface the degraded state loudly instead of silently.
fn degraded_running_apps<'m>(
report: &ReconcileReport,
manifests: impl Iterator<Item = &'m AppManifest>,
) -> Vec<(String, String)> {
let failed: HashSet<&str> = report.failures.iter().map(|(id, _)| id.as_str()).collect();
if failed.is_empty() {
return Vec::new();
}
let up: HashSet<&str> = report
.actions
.iter()
.filter(|(_, action)| {
matches!(
action,
ReconcileAction::NoOp | ReconcileAction::Started | ReconcileAction::Installed
)
})
.map(|(id, _)| id.as_str())
.collect();
let mut out = Vec::new();
for m in manifests {
if !up.contains(m.app.id.as_str()) {
continue;
}
for dep in manifest_dependency_app_ids(m) {
if failed.contains(dep.as_str()) {
out.push((m.app.id.clone(), dep));
}
}
}
out
}
/// Host-port collisions across a set of manifests, excluding pairs that are /// Host-port collisions across a set of manifests, excluding pairs that are
/// mutually exclusive by design (the bitcoin-core/knots variants; the legacy /// mutually exclusive by design (the bitcoin-core/knots variants; the legacy
/// mempool umbrella vs its split frontend, which render the same container). /// mempool umbrella vs its split frontend, which render the same container).
@ -1721,6 +1759,20 @@ impl ProdContainerOrchestrator {
} }
} }
{
let state = self.state.read().await;
for (app, dep) in degraded_running_apps(
&report,
state.manifests.values().map(|lm| &lm.manifest),
) {
tracing::error!(
app_id = %app,
dependency = %dep,
"app is up but its declared dependency failed to reconcile — running DEGRADED"
);
}
}
report report
} }
@ -5310,6 +5362,55 @@ app:
assert!(!calls.iter().any(|c| c.starts_with("start_container:"))); assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
} }
#[test]
fn degraded_running_apps_flags_up_app_with_failed_dependency() {
let btcpay = AppManifest::parse(
"app:\n id: btcpay-server\n name: btcpay\n version: 1.0.0\n container:\n image: btcpay:1\n dependencies:\n - app_id: archy-nbxplorer\n version: \">=1\"\n - app_id: archy-btcpay-db\n version: \">=15\"\n",
)
.unwrap();
let nbx = pull_manifest("archy-nbxplorer", "nbx:1");
let db = pull_manifest("archy-btcpay-db", "pg:15");
// btcpay up, both deps failed → two degraded pairs.
let report = ReconcileReport {
actions: vec![("btcpay-server".to_string(), ReconcileAction::NoOp)],
failures: vec![
("archy-nbxplorer".to_string(), "pulling ...".to_string()),
("archy-btcpay-db".to_string(), "pulling ...".to_string()),
],
};
let found = degraded_running_apps(&report, [&btcpay, &nbx, &db].into_iter());
assert_eq!(
found,
vec![
("btcpay-server".to_string(), "archy-nbxplorer".to_string()),
("btcpay-server".to_string(), "archy-btcpay-db".to_string()),
]
);
// Deps fine → nothing flagged.
let clean = ReconcileReport {
actions: vec![
("btcpay-server".to_string(), ReconcileAction::NoOp),
("archy-nbxplorer".to_string(), ReconcileAction::NoOp),
("archy-btcpay-db".to_string(), ReconcileAction::NoOp),
],
failures: vec![],
};
assert!(degraded_running_apps(&clean, [&btcpay, &nbx, &db].into_iter()).is_empty());
// The app itself failed too → not "up", not flagged (its own failure
// is already reported).
let both_down = ReconcileReport {
actions: vec![],
failures: vec![
("btcpay-server".to_string(), "x".to_string()),
("archy-nbxplorer".to_string(), "x".to_string()),
],
};
assert!(degraded_running_apps(&both_down, [&btcpay, &nbx, &db].into_iter()).is_empty());
}
#[test] #[test]
fn host_port_collisions_flags_pairs_and_skips_exclusive_variants() { fn host_port_collisions_flags_pairs_and_skips_exclusive_variants() {
let with_port = |id: &str, port: u16| { let with_port = |id: &str, port: u16| {