fix(orchestrator): make legacy mempool id start/stop/restart the split stack
Found by the .228 lifecycle gate (2026-07-08, first quadlet-mode run):
stop→start on the legacy `mempool` umbrella app id destroyed the mempool
deployment. Under quadlet, package.stop removes the containers; package.start
then failed with `unknown app_id: mempool` because load_manifests drops the
umbrella manifest whenever the three split-stack members are loaded — leaving
nothing to recreate from and the stack down.
Two fixes:
- prod_orchestrator start/stop/restart now resolve the historical
`mempool`/`mempool-web` id to the split members (archy-mempool-db,
mempool-api, archy-mempool-web) whenever the umbrella manifest was dropped —
the same alias install already implements ("installing mempool assembles the
split stack"). Restart falls back to start for members whose container/unit
is gone. Legacy umbrella-only nodes are unaffected (alias inactive while the
umbrella manifest is live).
- is_missing_container_error (3 sites) now recognizes podman 5.x's
`no such object` inspect phrasing, so a genuinely absent container is
classified as missing instead of surfacing as a hard inspect error.
Tests: 2 new alias tests + 2 classifier tests; prod_orchestrator suite 61/61
green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
380f4f1947
commit
161a6e4dbd
@ -926,6 +926,8 @@ async fn inspect_runtime_container_state(container_name: &str) -> Result<Option<
|
||||
fn is_missing_container_error(stderr: &str) -> bool {
|
||||
stderr.contains("no such container")
|
||||
|| stderr.contains("no container with name")
|
||||
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|
||||
|| stderr.contains("no such object")
|
||||
|| stderr.contains("does not exist")
|
||||
|| stderr.contains("not found")
|
||||
}
|
||||
@ -1966,6 +1968,22 @@ pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn missing_container_classifier_covers_podman5_phrasings() {
|
||||
// Regression (.228 gate 2026-07-08): podman 5.x `inspect` on a missing
|
||||
// container says `Error: no such object: "mempool"` — the classifier
|
||||
// missed it, so a plain missing container surfaced as a hard inspect
|
||||
// error and package.start aborted instead of proceeding to recreate.
|
||||
assert!(is_missing_container_error(
|
||||
"Error: no such object: \"mempool\""
|
||||
));
|
||||
assert!(is_missing_container_error("Error: no such container mempool"));
|
||||
assert!(is_missing_container_error(
|
||||
"Error: no container with name or id \"x\" found"
|
||||
));
|
||||
assert!(!is_missing_container_error("Error: OCI runtime error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_host_ports_are_manifest_derived_for_public_apps() {
|
||||
assert_eq!(runtime_host_ports("photoprism"), vec![2342]);
|
||||
|
||||
@ -130,6 +130,8 @@ async fn wait_for_stack_container_absent(container_name: &str, timeout: Duration
|
||||
fn is_missing_container_error(stderr: &str) -> bool {
|
||||
stderr.contains("no such container")
|
||||
|| stderr.contains("no container with name")
|
||||
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|
||||
|| stderr.contains("no such object")
|
||||
|| stderr.contains("does not exist")
|
||||
|| stderr.contains("not found")
|
||||
}
|
||||
|
||||
@ -1352,6 +1352,28 @@ impl ProdContainerOrchestrator {
|
||||
.ok_or_else(|| anyhow::anyhow!("unknown app_id: {app_id}"))
|
||||
}
|
||||
|
||||
/// Lifecycle alias for the legacy `mempool` umbrella id. `load_manifests`
|
||||
/// drops the umbrella manifest whenever all three split-stack members are
|
||||
/// loaded (they render the same frontend container), and install already
|
||||
/// assembles the split stack under the historical id — so start/stop/
|
||||
/// restart addressed to `mempool` must fan out to the members too, or a
|
||||
/// stopped stack becomes unstartable ("unknown app_id: mempool" while no
|
||||
/// member container is live). Returns members in dependency-start order.
|
||||
async fn mempool_umbrella_members(&self, app_id: &str) -> Option<Vec<String>> {
|
||||
if !matches!(app_id, "mempool" | "mempool-web") {
|
||||
return None;
|
||||
}
|
||||
let state = self.state.read().await;
|
||||
if state.manifests.contains_key("mempool") {
|
||||
return None; // legacy node: umbrella manifest still live, no alias
|
||||
}
|
||||
let members = ["archy-mempool-db", "mempool-api", "archy-mempool-web"];
|
||||
members
|
||||
.iter()
|
||||
.all(|id| state.manifests.contains_key(*id))
|
||||
.then(|| members.iter().map(|s| s.to_string()).collect())
|
||||
}
|
||||
|
||||
/// Snapshot of the app IDs currently in the in-memory manifest map.
|
||||
/// Used by the boot reconciler to drive companion-unit reconciliation.
|
||||
pub async fn manifest_ids(&self) -> Vec<String> {
|
||||
@ -3615,6 +3637,18 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
}
|
||||
|
||||
async fn start(&self, app_id: &str) -> Result<()> {
|
||||
if let Some(members) = self.mempool_umbrella_members(app_id).await {
|
||||
tracing::info!(app_id, "starting legacy umbrella id via split-stack members");
|
||||
for (i, member) in members.iter().enumerate() {
|
||||
if i > 0 {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||
}
|
||||
Box::pin(self.start(member))
|
||||
.await
|
||||
.with_context(|| format!("start split-stack member {member}"))?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
{
|
||||
let mut state = self.state.write().await;
|
||||
state.disabled.remove(app_id);
|
||||
@ -3646,6 +3680,15 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
}
|
||||
|
||||
async fn stop(&self, app_id: &str) -> Result<()> {
|
||||
if let Some(members) = self.mempool_umbrella_members(app_id).await {
|
||||
tracing::info!(app_id, "stopping legacy umbrella id via split-stack members");
|
||||
for member in members.iter().rev() {
|
||||
Box::pin(self.stop(member))
|
||||
.await
|
||||
.with_context(|| format!("stop split-stack member {member}"))?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
{
|
||||
let mut state = self.state.write().await;
|
||||
state.disabled.insert(app_id.to_string());
|
||||
@ -3704,6 +3747,24 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
}
|
||||
|
||||
async fn restart(&self, app_id: &str) -> Result<()> {
|
||||
if let Some(members) = self.mempool_umbrella_members(app_id).await {
|
||||
tracing::info!(app_id, "restarting legacy umbrella id via split-stack members");
|
||||
for (i, member) in members.iter().enumerate() {
|
||||
if i > 0 {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||
}
|
||||
if let Err(err) = Box::pin(self.restart(member)).await {
|
||||
// A member whose container/unit is gone (e.g. removed by a
|
||||
// quadlet stop) can't be "restarted" — fall back to start,
|
||||
// which recreates it via ensure_running.
|
||||
tracing::warn!(member, error = %err, "member restart failed; falling back to start");
|
||||
Box::pin(self.start(member))
|
||||
.await
|
||||
.with_context(|| format!("restart split-stack member {member}"))?;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let lm = self.loaded(app_id).await?;
|
||||
let lock = self.app_lock(app_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
@ -5396,6 +5457,60 @@ app:
|
||||
assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn umbrella_mempool_id_fans_out_to_split_members() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let orch = orch_with(rt.clone()).await;
|
||||
for (id, image) in [
|
||||
("archy-mempool-db", "docker.io/library/mariadb:11.4"),
|
||||
("mempool-api", "docker.io/mempool/backend:v3.0"),
|
||||
("archy-mempool-web", "docker.io/mempool/frontend:v3.0"),
|
||||
] {
|
||||
orch.insert_manifest_for_test(pull_manifest(id, image), PathBuf::from("/tmp/mp"))
|
||||
.await;
|
||||
}
|
||||
// No umbrella `mempool` manifest is loaded (load_manifests drops it
|
||||
// when the split members are present) and no member container is live
|
||||
// — the exact state a quadlet stop leaves behind. start("mempool")
|
||||
// must resurrect the members rather than fail with unknown app_id.
|
||||
orch.start("mempool").await.unwrap();
|
||||
let calls = rt.calls();
|
||||
for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] {
|
||||
assert!(
|
||||
calls.iter().any(|c| c == &format!("start_container:{name}")),
|
||||
"{name} not started: {calls:?}"
|
||||
);
|
||||
}
|
||||
|
||||
orch.stop("mempool").await.unwrap();
|
||||
let calls = rt.calls();
|
||||
for name in ["archy-mempool-db", "mempool-api", "archy-mempool-web"] {
|
||||
assert!(
|
||||
calls.iter().any(|c| c == &format!("stop_container:{name}")),
|
||||
"{name} not stopped: {calls:?}"
|
||||
);
|
||||
}
|
||||
|
||||
orch.restart("mempool").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn umbrella_alias_inactive_when_umbrella_manifest_live() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let orch = orch_with(rt.clone()).await;
|
||||
// Legacy node: only the umbrella manifest exists — the alias must not
|
||||
// kick in; `mempool` operates as its own single-container app.
|
||||
orch.insert_manifest_for_test(
|
||||
pull_manifest("mempool", "docker.io/mempool/frontend:v3.0"),
|
||||
PathBuf::from("/tmp/mp"),
|
||||
)
|
||||
.await;
|
||||
orch.start("mempool").await.unwrap();
|
||||
let calls = rt.calls();
|
||||
assert!(calls.iter().any(|c| c == "start_container:mempool"));
|
||||
assert!(!calls.iter().any(|c| c.contains("archy-mempool")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_force_recreates_stopping_container() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
|
||||
@ -572,6 +572,8 @@ fn is_missing_container_error(stderr: &str) -> bool {
|
||||
let stderr = stderr.to_ascii_lowercase();
|
||||
stderr.contains("no container with name or id")
|
||||
|| stderr.contains("no such container")
|
||||
// podman 5.x `inspect` phrasing: `Error: no such object: "name"`
|
||||
|| stderr.contains("no such object")
|
||||
|| stderr.contains("does not exist")
|
||||
|| stderr.contains("not found")
|
||||
}
|
||||
@ -1035,6 +1037,16 @@ mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[test]
|
||||
fn missing_container_classifier_covers_podman5_phrasings() {
|
||||
// podman 5.x `inspect` phrasing for a missing container.
|
||||
assert!(is_missing_container_error(
|
||||
"Error: no such object: \"mempool\""
|
||||
));
|
||||
assert!(is_missing_container_error("Error: no such container x"));
|
||||
assert!(!is_missing_container_error("Error: OCI runtime error"));
|
||||
}
|
||||
|
||||
fn cfg(context: &str, tag: &str, dockerfile: &str, args: &[(&str, &str)]) -> BuildConfig {
|
||||
BuildConfig {
|
||||
context: context.to_string(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user