feat(orchestrator): surface apps running with failed dependencies
btcpay-server started and reported healthy while its declared dependencies (archy-btcpay-db, archy-nbxplorer) failed every boot reconcile (live-testing report 2026-07-10, Fix 3). After each reconcile pass, flag every app the pass left up whose manifest dependency landed in the failure list — a loud DEGRADED journal error instead of silence. True start-gating on dependency health is a larger design change (boot-ordering/deadlock risk) and stays open. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
ebd904262e
commit
140b703838
@ -1032,6 +1032,44 @@ fn resolve_catalog_image(resolved_manifest: &mut AppManifest) {
|
||||
}
|
||||
}
|
||||
|
||||
/// (running app, failed dependency) pairs after a reconcile pass: apps the
|
||||
/// pass left up whose declared manifest dependency FAILED to reconcile.
|
||||
/// btcpay-server came up "healthy" while archy-btcpay-db/archy-nbxplorer
|
||||
/// never made it (live-testing 2026-07-10) — until real start-gating exists,
|
||||
/// surface the degraded state loudly instead of silently.
|
||||
fn degraded_running_apps<'m>(
|
||||
report: &ReconcileReport,
|
||||
manifests: impl Iterator<Item = &'m AppManifest>,
|
||||
) -> Vec<(String, String)> {
|
||||
let failed: HashSet<&str> = report.failures.iter().map(|(id, _)| id.as_str()).collect();
|
||||
if failed.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let up: HashSet<&str> = report
|
||||
.actions
|
||||
.iter()
|
||||
.filter(|(_, action)| {
|
||||
matches!(
|
||||
action,
|
||||
ReconcileAction::NoOp | ReconcileAction::Started | ReconcileAction::Installed
|
||||
)
|
||||
})
|
||||
.map(|(id, _)| id.as_str())
|
||||
.collect();
|
||||
let mut out = Vec::new();
|
||||
for m in manifests {
|
||||
if !up.contains(m.app.id.as_str()) {
|
||||
continue;
|
||||
}
|
||||
for dep in manifest_dependency_app_ids(m) {
|
||||
if failed.contains(dep.as_str()) {
|
||||
out.push((m.app.id.clone(), dep));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Host-port collisions across a set of manifests, excluding pairs that are
|
||||
/// mutually exclusive by design (the bitcoin-core/knots variants; the legacy
|
||||
/// mempool umbrella vs its split frontend, which render the same container).
|
||||
@ -1721,6 +1759,20 @@ impl ProdContainerOrchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let state = self.state.read().await;
|
||||
for (app, dep) in degraded_running_apps(
|
||||
&report,
|
||||
state.manifests.values().map(|lm| &lm.manifest),
|
||||
) {
|
||||
tracing::error!(
|
||||
app_id = %app,
|
||||
dependency = %dep,
|
||||
"app is up but its declared dependency failed to reconcile — running DEGRADED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
@ -5310,6 +5362,55 @@ app:
|
||||
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_running_apps_flags_up_app_with_failed_dependency() {
|
||||
let btcpay = AppManifest::parse(
|
||||
"app:\n id: btcpay-server\n name: btcpay\n version: 1.0.0\n container:\n image: btcpay:1\n dependencies:\n - app_id: archy-nbxplorer\n version: \">=1\"\n - app_id: archy-btcpay-db\n version: \">=15\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
let nbx = pull_manifest("archy-nbxplorer", "nbx:1");
|
||||
let db = pull_manifest("archy-btcpay-db", "pg:15");
|
||||
|
||||
// btcpay up, both deps failed → two degraded pairs.
|
||||
let report = ReconcileReport {
|
||||
actions: vec![("btcpay-server".to_string(), ReconcileAction::NoOp)],
|
||||
failures: vec![
|
||||
("archy-nbxplorer".to_string(), "pulling ...".to_string()),
|
||||
("archy-btcpay-db".to_string(), "pulling ...".to_string()),
|
||||
],
|
||||
};
|
||||
let found = degraded_running_apps(&report, [&btcpay, &nbx, &db].into_iter());
|
||||
assert_eq!(
|
||||
found,
|
||||
vec![
|
||||
("btcpay-server".to_string(), "archy-nbxplorer".to_string()),
|
||||
("btcpay-server".to_string(), "archy-btcpay-db".to_string()),
|
||||
]
|
||||
);
|
||||
|
||||
// Deps fine → nothing flagged.
|
||||
let clean = ReconcileReport {
|
||||
actions: vec![
|
||||
("btcpay-server".to_string(), ReconcileAction::NoOp),
|
||||
("archy-nbxplorer".to_string(), ReconcileAction::NoOp),
|
||||
("archy-btcpay-db".to_string(), ReconcileAction::NoOp),
|
||||
],
|
||||
failures: vec![],
|
||||
};
|
||||
assert!(degraded_running_apps(&clean, [&btcpay, &nbx, &db].into_iter()).is_empty());
|
||||
|
||||
// The app itself failed too → not "up", not flagged (its own failure
|
||||
// is already reported).
|
||||
let both_down = ReconcileReport {
|
||||
actions: vec![],
|
||||
failures: vec![
|
||||
("btcpay-server".to_string(), "x".to_string()),
|
||||
("archy-nbxplorer".to_string(), "x".to_string()),
|
||||
],
|
||||
};
|
||||
assert!(degraded_running_apps(&both_down, [&btcpay, &nbx, &db].into_iter()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_port_collisions_flags_pairs_and_skips_exclusive_variants() {
|
||||
let with_port = |id: &str, port: u16| {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user