From 161a6e4dbdaffc0f156f551b2fbdbd03c5a0fee9 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 8 Jul 2026 19:34:46 -0400 Subject: [PATCH] fix(orchestrator): make legacy `mempool` id start/stop/restart the split stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by the .228 lifecycle gate (2026-07-08, first quadlet-mode run): stop→start on the legacy `mempool` umbrella app id destroyed the mempool deployment. Under quadlet, package.stop removes the containers; package.start then failed with `unknown app_id: mempool` because load_manifests drops the umbrella manifest whenever the three split-stack members are loaded — leaving nothing to recreate from and the stack down. Two fixes: - prod_orchestrator start/stop/restart now resolve the historical `mempool`/`mempool-web` id to the split members (archy-mempool-db, mempool-api, archy-mempool-web) whenever the umbrella manifest was dropped — the same alias install already implements ("installing mempool assembles the split stack"). Restart falls back to start for members whose container/unit is gone. Legacy umbrella-only nodes are unaffected (alias inactive while the umbrella manifest is live). - is_missing_container_error (3 sites) now recognizes podman 5.x's `no such object` inspect phrasing, so a genuinely absent container is classified as missing instead of surfacing as a hard inspect error. Tests: 2 new alias tests + 2 classifier tests; prod_orchestrator suite 61/61 green. Co-Authored-By: Claude Fable 5 --- .../src/api/rpc/package/runtime.rs | 18 +++ .../archipelago/src/api/rpc/package/stacks.rs | 2 + .../src/container/prod_orchestrator.rs | 115 ++++++++++++++++++ core/container/src/runtime.rs | 12 ++ 4 files changed, 147 insertions(+) diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index e28d8382..6b0add47 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -926,6 +926,8 @@ async fn inspect_runtime_container_state(container_name: &str) -> Result bool { stderr.contains("no such container") || stderr.contains("no container with name") + // podman 5.x `inspect` phrasing: `Error: no such object: "name"` + || stderr.contains("no such object") || stderr.contains("does not exist") || stderr.contains("not found") } @@ -1966,6 +1968,22 @@ pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec { mod tests { use super::*; + #[test] + fn missing_container_classifier_covers_podman5_phrasings() { + // Regression (.228 gate 2026-07-08): podman 5.x `inspect` on a missing + // container says `Error: no such object: "mempool"` — the classifier + // missed it, so a plain missing container surfaced as a hard inspect + // error and package.start aborted instead of proceeding to recreate. + assert!(is_missing_container_error( + "Error: no such object: \"mempool\"" + )); + assert!(is_missing_container_error("Error: no such container mempool")); + assert!(is_missing_container_error( + "Error: no container with name or id \"x\" found" + )); + assert!(!is_missing_container_error("Error: OCI runtime error")); + } + #[test] fn runtime_host_ports_are_manifest_derived_for_public_apps() { assert_eq!(runtime_host_ports("photoprism"), vec![2342]); diff --git a/core/archipelago/src/api/rpc/package/stacks.rs b/core/archipelago/src/api/rpc/package/stacks.rs index 505f8700..7899edf8 100644 --- a/core/archipelago/src/api/rpc/package/stacks.rs +++ b/core/archipelago/src/api/rpc/package/stacks.rs @@ -130,6 +130,8 @@ async fn wait_for_stack_container_absent(container_name: &str, timeout: Duration fn is_missing_container_error(stderr: &str) -> bool { stderr.contains("no such container") || stderr.contains("no container with name") + // podman 5.x `inspect` phrasing: `Error: no such object: "name"` + || stderr.contains("no such object") || stderr.contains("does not exist") || stderr.contains("not found") } diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 49a3644a..41dbfb47 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -1352,6 +1352,28 @@ impl ProdContainerOrchestrator { .ok_or_else(|| anyhow::anyhow!("unknown app_id: {app_id}")) } + /// Lifecycle alias for the legacy `mempool` umbrella id. `load_manifests` + /// drops the umbrella manifest whenever all three split-stack members are + /// loaded (they render the same frontend container), and install already + /// assembles the split stack under the historical id — so start/stop/ + /// restart addressed to `mempool` must fan out to the members too, or a + /// stopped stack becomes unstartable ("unknown app_id: mempool" while no + /// member container is live). Returns members in dependency-start order. + async fn mempool_umbrella_members(&self, app_id: &str) -> Option> { + if !matches!(app_id, "mempool" | "mempool-web") { + return None; + } + let state = self.state.read().await; + if state.manifests.contains_key("mempool") { + return None; // legacy node: umbrella manifest still live, no alias + } + let members = ["archy-mempool-db", "mempool-api", "archy-mempool-web"]; + members + .iter() + .all(|id| state.manifests.contains_key(*id)) + .then(|| members.iter().map(|s| s.to_string()).collect()) + } + /// Snapshot of the app IDs currently in the in-memory manifest map. /// Used by the boot reconciler to drive companion-unit reconciliation. pub async fn manifest_ids(&self) -> Vec { @@ -3615,6 +3637,18 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { } async fn start(&self, app_id: &str) -> Result<()> { + if let Some(members) = self.mempool_umbrella_members(app_id).await { + tracing::info!(app_id, "starting legacy umbrella id via split-stack members"); + for (i, member) in members.iter().enumerate() { + if i > 0 { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + Box::pin(self.start(member)) + .await + .with_context(|| format!("start split-stack member {member}"))?; + } + return Ok(()); + } { let mut state = self.state.write().await; state.disabled.remove(app_id); @@ -3646,6 +3680,15 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { } async fn stop(&self, app_id: &str) -> Result<()> { + if let Some(members) = self.mempool_umbrella_members(app_id).await { + tracing::info!(app_id, "stopping legacy umbrella id via split-stack members"); + for member in members.iter().rev() { + Box::pin(self.stop(member)) + .await + .with_context(|| format!("stop split-stack member {member}"))?; + } + return Ok(()); + } { let mut state = self.state.write().await; state.disabled.insert(app_id.to_string()); @@ -3704,6 +3747,24 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { } async fn restart(&self, app_id: &str) -> Result<()> { + if let Some(members) = self.mempool_umbrella_members(app_id).await { + tracing::info!(app_id, "restarting legacy umbrella id via split-stack members"); + for (i, member) in members.iter().enumerate() { + if i > 0 { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if let Err(err) = Box::pin(self.restart(member)).await { + // A member whose container/unit is gone (e.g. removed by a + // quadlet stop) can't be "restarted" — fall back to start, + // which recreates it via ensure_running. + tracing::warn!(member, error = %err, "member restart failed; falling back to start"); + Box::pin(self.start(member)) + .await + .with_context(|| format!("restart split-stack member {member}"))?; + } + } + return Ok(()); + } let lm = self.loaded(app_id).await?; let lock = self.app_lock(app_id).await; let _guard = lock.lock().await; @@ -5396,6 +5457,60 @@ app: assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots")); } + #[tokio::test] + async fn umbrella_mempool_id_fans_out_to_split_members() { + let rt = Arc::new(MockRuntime::default()); + let orch = orch_with(rt.clone()).await; + for (id, image) in [ + ("archy-mempool-db", "docker.io/library/mariadb:11.4"), + ("mempool-api", "docker.io/mempool/backend:v3.0"), + ("archy-mempool-web", "docker.io/mempool/frontend:v3.0"), + ] { + orch.insert_manifest_for_test(pull_manifest(id, image), PathBuf::from("/tmp/mp")) + .await; + } + // No umbrella `mempool` manifest is loaded (load_manifests drops it + // when the split members are present) and no member container is live + // — the exact state a quadlet stop leaves behind. start("mempool") + // must resurrect the members rather than fail with unknown app_id. + orch.start("mempool").await.unwrap(); + let calls = rt.calls(); + for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] { + assert!( + calls.iter().any(|c| c == &format!("start_container:{name}")), + "{name} not started: {calls:?}" + ); + } + + orch.stop("mempool").await.unwrap(); + let calls = rt.calls(); + for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] { + assert!( + calls.iter().any(|c| c == &format!("stop_container:{name}")), + "{name} not stopped: {calls:?}" + ); + } + + orch.restart("mempool").await.unwrap(); + } + + #[tokio::test] + async fn umbrella_alias_inactive_when_umbrella_manifest_live() { + let rt = Arc::new(MockRuntime::default()); + let orch = orch_with(rt.clone()).await; + // Legacy node: only the umbrella manifest exists — the alias must not + // kick in; `mempool` operates as its own single-container app. + orch.insert_manifest_for_test( + pull_manifest("mempool", "docker.io/mempool/frontend:v3.0"), + PathBuf::from("/tmp/mp"), + ) + .await; + orch.start("mempool").await.unwrap(); + let calls = rt.calls(); + assert!(calls.iter().any(|c| c == "start_container:mempool")); + assert!(!calls.iter().any(|c| c.contains("archy-mempool"))); + } + #[tokio::test] async fn reconcile_force_recreates_stopping_container() { let rt = Arc::new(MockRuntime::default()); diff --git a/core/container/src/runtime.rs b/core/container/src/runtime.rs index 4ee040b3..72c147e9 100644 --- a/core/container/src/runtime.rs +++ b/core/container/src/runtime.rs @@ -572,6 +572,8 @@ fn is_missing_container_error(stderr: &str) -> bool { let stderr = stderr.to_ascii_lowercase(); stderr.contains("no container with name or id") || stderr.contains("no such container") + // podman 5.x `inspect` phrasing: `Error: no such object: "name"` + || stderr.contains("no such object") || stderr.contains("does not exist") || stderr.contains("not found") } @@ -1035,6 +1037,16 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn missing_container_classifier_covers_podman5_phrasings() { + // podman 5.x `inspect` phrasing for a missing container. + assert!(is_missing_container_error( + "Error: no such object: \"mempool\"" + )); + assert!(is_missing_container_error("Error: no such container x")); + assert!(!is_missing_container_error("Error: OCI runtime error")); + } + fn cfg(context: &str, tag: &str, dockerfile: &str, args: &[(&str, &str)]) -> BuildConfig { BuildConfig { context: context.to_string(),