fix(orchestrator): make legacy mempool id start/stop/restart the split stack

Found by the .228 lifecycle gate (2026-07-08, first quadlet-mode run):
stop→start on the legacy `mempool` umbrella app id destroyed the mempool
deployment. Under quadlet, package.stop removes the containers; package.start
then failed with `unknown app_id: mempool` because load_manifests drops the
umbrella manifest whenever the three split-stack members are loaded — leaving
nothing to recreate from and the stack down.

Two fixes:
- prod_orchestrator start/stop/restart now resolve the historical
  `mempool`/`mempool-web` id to the split members (archy-mempool-db,
  mempool-api, archy-mempool-web) whenever the umbrella manifest was dropped —
  the same alias install already implements ("installing mempool assembles the
  split stack"). Restart falls back to start for members whose container/unit
  is gone. Legacy umbrella-only nodes are unaffected (alias inactive while the
  umbrella manifest is live).
- is_missing_container_error (3 sites) now recognizes podman 5.x's
  `no such object` inspect phrasing, so a genuinely absent container is
  classified as missing instead of surfacing as a hard inspect error.

Tests: 2 new alias tests + 2 classifier tests; prod_orchestrator suite 61/61
green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-08 19:34:46 -04:00
parent 380f4f1947
commit 161a6e4dbd
4 changed files with 147 additions and 0 deletions

View File

@ -926,6 +926,8 @@ async fn inspect_runtime_container_state(container_name: &str) -> Result<Option<
fn is_missing_container_error(stderr: &str) -> bool { fn is_missing_container_error(stderr: &str) -> bool {
stderr.contains("no such container") stderr.contains("no such container")
|| stderr.contains("no container with name") || stderr.contains("no container with name")
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|| stderr.contains("no such object")
|| stderr.contains("does not exist") || stderr.contains("does not exist")
|| stderr.contains("not found") || stderr.contains("not found")
} }
@ -1966,6 +1968,22 @@ pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec<String> {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn missing_container_classifier_covers_podman5_phrasings() {
// Regression (.228 gate 2026-07-08): podman 5.x `inspect` on a missing
// container says `Error: no such object: "mempool"` — the classifier
// missed it, so a plain missing container surfaced as a hard inspect
// error and package.start aborted instead of proceeding to recreate.
assert!(is_missing_container_error(
"Error: no such object: \"mempool\""
));
assert!(is_missing_container_error("Error: no such container mempool"));
assert!(is_missing_container_error(
"Error: no container with name or id \"x\" found"
));
assert!(!is_missing_container_error("Error: OCI runtime error"));
}
#[test] #[test]
fn runtime_host_ports_are_manifest_derived_for_public_apps() { fn runtime_host_ports_are_manifest_derived_for_public_apps() {
assert_eq!(runtime_host_ports("photoprism"), vec![2342]); assert_eq!(runtime_host_ports("photoprism"), vec![2342]);

View File

@ -130,6 +130,8 @@ async fn wait_for_stack_container_absent(container_name: &str, timeout: Duration
fn is_missing_container_error(stderr: &str) -> bool { fn is_missing_container_error(stderr: &str) -> bool {
stderr.contains("no such container") stderr.contains("no such container")
|| stderr.contains("no container with name") || stderr.contains("no container with name")
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|| stderr.contains("no such object")
|| stderr.contains("does not exist") || stderr.contains("does not exist")
|| stderr.contains("not found") || stderr.contains("not found")
} }

View File

@ -1352,6 +1352,28 @@ impl ProdContainerOrchestrator {
.ok_or_else(|| anyhow::anyhow!("unknown app_id: {app_id}")) .ok_or_else(|| anyhow::anyhow!("unknown app_id: {app_id}"))
} }
/// Lifecycle alias for the legacy `mempool` umbrella id. `load_manifests`
/// drops the umbrella manifest whenever all three split-stack members are
/// loaded (they render the same frontend container), and install already
/// assembles the split stack under the historical id — so start/stop/
/// restart addressed to `mempool` must fan out to the members too, or a
/// stopped stack becomes unstartable ("unknown app_id: mempool" while no
/// member container is live). Returns members in dependency-start order.
async fn mempool_umbrella_members(&self, app_id: &str) -> Option<Vec<String>> {
if !matches!(app_id, "mempool" | "mempool-web") {
return None;
}
let state = self.state.read().await;
if state.manifests.contains_key("mempool") {
return None; // legacy node: umbrella manifest still live, no alias
}
let members = ["archy-mempool-db", "mempool-api", "archy-mempool-web"];
members
.iter()
.all(|id| state.manifests.contains_key(*id))
.then(|| members.iter().map(|s| s.to_string()).collect())
}
/// Snapshot of the app IDs currently in the in-memory manifest map. /// Snapshot of the app IDs currently in the in-memory manifest map.
/// Used by the boot reconciler to drive companion-unit reconciliation. /// Used by the boot reconciler to drive companion-unit reconciliation.
pub async fn manifest_ids(&self) -> Vec<String> { pub async fn manifest_ids(&self) -> Vec<String> {
@ -3615,6 +3637,18 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
} }
async fn start(&self, app_id: &str) -> Result<()> { async fn start(&self, app_id: &str) -> Result<()> {
if let Some(members) = self.mempool_umbrella_members(app_id).await {
tracing::info!(app_id, "starting legacy umbrella id via split-stack members");
for (i, member) in members.iter().enumerate() {
if i > 0 {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
Box::pin(self.start(member))
.await
.with_context(|| format!("start split-stack member {member}"))?;
}
return Ok(());
}
{ {
let mut state = self.state.write().await; let mut state = self.state.write().await;
state.disabled.remove(app_id); state.disabled.remove(app_id);
@ -3646,6 +3680,15 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
} }
async fn stop(&self, app_id: &str) -> Result<()> { async fn stop(&self, app_id: &str) -> Result<()> {
if let Some(members) = self.mempool_umbrella_members(app_id).await {
tracing::info!(app_id, "stopping legacy umbrella id via split-stack members");
for member in members.iter().rev() {
Box::pin(self.stop(member))
.await
.with_context(|| format!("stop split-stack member {member}"))?;
}
return Ok(());
}
{ {
let mut state = self.state.write().await; let mut state = self.state.write().await;
state.disabled.insert(app_id.to_string()); state.disabled.insert(app_id.to_string());
@ -3704,6 +3747,24 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
} }
async fn restart(&self, app_id: &str) -> Result<()> { async fn restart(&self, app_id: &str) -> Result<()> {
if let Some(members) = self.mempool_umbrella_members(app_id).await {
tracing::info!(app_id, "restarting legacy umbrella id via split-stack members");
for (i, member) in members.iter().enumerate() {
if i > 0 {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if let Err(err) = Box::pin(self.restart(member)).await {
// A member whose container/unit is gone (e.g. removed by a
// quadlet stop) can't be "restarted" — fall back to start,
// which recreates it via ensure_running.
tracing::warn!(member, error = %err, "member restart failed; falling back to start");
Box::pin(self.start(member))
.await
.with_context(|| format!("restart split-stack member {member}"))?;
}
}
return Ok(());
}
let lm = self.loaded(app_id).await?; let lm = self.loaded(app_id).await?;
let lock = self.app_lock(app_id).await; let lock = self.app_lock(app_id).await;
let _guard = lock.lock().await; let _guard = lock.lock().await;
@ -5396,6 +5457,60 @@ app:
assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots")); assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots"));
} }
#[tokio::test]
async fn umbrella_mempool_id_fans_out_to_split_members() {
let rt = Arc::new(MockRuntime::default());
let orch = orch_with(rt.clone()).await;
for (id, image) in [
("archy-mempool-db", "docker.io/library/mariadb:11.4"),
("mempool-api", "docker.io/mempool/backend:v3.0"),
("archy-mempool-web", "docker.io/mempool/frontend:v3.0"),
] {
orch.insert_manifest_for_test(pull_manifest(id, image), PathBuf::from("/tmp/mp"))
.await;
}
// No umbrella `mempool` manifest is loaded (load_manifests drops it
// when the split members are present) and no member container is live
// — the exact state a quadlet stop leaves behind. start("mempool")
// must resurrect the members rather than fail with unknown app_id.
orch.start("mempool").await.unwrap();
let calls = rt.calls();
for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] {
assert!(
calls.iter().any(|c| c == &format!("start_container:{name}")),
"{name} not started: {calls:?}"
);
}
orch.stop("mempool").await.unwrap();
let calls = rt.calls();
for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] {
assert!(
calls.iter().any(|c| c == &format!("stop_container:{name}")),
"{name} not stopped: {calls:?}"
);
}
orch.restart("mempool").await.unwrap();
}
#[tokio::test]
async fn umbrella_alias_inactive_when_umbrella_manifest_live() {
let rt = Arc::new(MockRuntime::default());
let orch = orch_with(rt.clone()).await;
// Legacy node: only the umbrella manifest exists — the alias must not
// kick in; `mempool` operates as its own single-container app.
orch.insert_manifest_for_test(
pull_manifest("mempool", "docker.io/mempool/frontend:v3.0"),
PathBuf::from("/tmp/mp"),
)
.await;
orch.start("mempool").await.unwrap();
let calls = rt.calls();
assert!(calls.iter().any(|c| c == "start_container:mempool"));
assert!(!calls.iter().any(|c| c.contains("archy-mempool")));
}
#[tokio::test] #[tokio::test]
async fn reconcile_force_recreates_stopping_container() { async fn reconcile_force_recreates_stopping_container() {
let rt = Arc::new(MockRuntime::default()); let rt = Arc::new(MockRuntime::default());

View File

@ -572,6 +572,8 @@ fn is_missing_container_error(stderr: &str) -> bool {
let stderr = stderr.to_ascii_lowercase(); let stderr = stderr.to_ascii_lowercase();
stderr.contains("no container with name or id") stderr.contains("no container with name or id")
|| stderr.contains("no such container") || stderr.contains("no such container")
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|| stderr.contains("no such object")
|| stderr.contains("does not exist") || stderr.contains("does not exist")
|| stderr.contains("not found") || stderr.contains("not found")
} }
@ -1035,6 +1037,16 @@ mod tests {
use super::*; use super::*;
use std::collections::HashMap; use std::collections::HashMap;
#[test]
fn missing_container_classifier_covers_podman5_phrasings() {
// podman 5.x `inspect` phrasing for a missing container.
assert!(is_missing_container_error(
"Error: no such object: \"mempool\""
));
assert!(is_missing_container_error("Error: no such container x"));
assert!(!is_missing_container_error("Error: OCI runtime error"));
}
fn cfg(context: &str, tag: &str, dockerfile: &str, args: &[(&str, &str)]) -> BuildConfig { fn cfg(context: &str, tag: &str, dockerfile: &str, args: &[(&str, &str)]) -> BuildConfig {
BuildConfig { BuildConfig {
context: context.to_string(), context: context.to_string(),