From 6d60082997407f319be5633f063fc512f5def1f9 Mon Sep 17 00:00:00 2001 From: archipelago Date: Thu, 9 Jul 2026 04:18:55 -0400 Subject: [PATCH] fix(lifecycle): route stack stop/restart via member app ids, not container names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit package.stop/restart on orchestrator-managed stacks (immich, indeedhub, btcpay, netbird, mempool) addressed live CONTAINER names; unknown app ids fell through to raw podman stop/start, racing systemd's --rm cleanup of the quadlet unit — the container vanished on stop and the start half found nothing (immich RESTART FAIL + indeedhub stuck restarting, gate 2026-07-09). Now stack ops resolve member APP ids (reverse start order for stop) so the orchestrator drives the quadlet .service; user-stop markers are written for member app ids too (covers absent containers), and the orchestrator stop fan-out treats an already-missing container as stopped. Co-Authored-By: Claude Fable 5 --- .../src/api/rpc/package/runtime.rs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index 781e809c..411ceef7 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -63,6 +63,8 @@ impl RpcHandler { let to_start = if self.orchestrator.is_some() && uses_single_orchestrator_app(package_id) { vec![orchestrator_app_id(package_id).to_string()] + } else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) { + members } else { ordered_containers_for_start(package_id).await? }; @@ -153,6 +155,22 @@ impl RpcHandler { return Err(anyhow::anyhow!("No containers found for {}", package_id)); } + // Orchestrator-managed stacks are stopped via member APP ids (reverse + // start order), never live container names: orchestrator.stop(app_id) + // stops the member's quadlet .service (systemd removes the --rm + // container), while a container NAME falls through the unknown-app-id + // fallback to a raw `podman stop` that races systemd over the unit + // (immich, gate 2026-07-09). + let to_stop_ids = if !single_orchestrator_app { + orchestrator_stack_members(self.orchestrator.is_some(), package_id) + .map(|mut members| { + members.reverse(); + members + }) + } else { + None + }; + // Mark as user-stopped BEFORE the spawn so health monitor and // crash recovery don't auto-restart mid-flight. Ordering is // load-bearing — see runtime.rs:145-148 original note. @@ -160,9 +178,17 @@ impl RpcHandler { for name in &containers { crate::crash_recovery::mark_user_stopped(&self.config.data_dir, name).await; } + // Stack members are marked under their app ids too, so the reconcile + // guard holds for members whose container is currently absent (a live + // container list can't name those). + if let Some(ids) = &to_stop_ids { + for id in ids { + crate::crash_recovery::mark_user_stopped(&self.config.data_dir, id).await; + } + } let package_id_owned = package_id.to_string(); - let to_stop = containers.clone(); + let to_stop = to_stop_ids.unwrap_or_else(|| containers.clone()); let orchestrator = self.orchestrator.clone(); let state_manager = Arc::clone(&self.state_manager); let pre_state = @@ -246,6 +272,12 @@ impl RpcHandler { let companion_app_id = package_id_owned.clone(); let to_restart = if single_orchestrator_app { vec![orchestrator_app_id(package_id).to_string()] + } else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) { + // Restart stacks via member APP ids: restarting by live container + // name podman-stops the quadlet container (systemd --rm removes + // it) and the start half then finds no such container — a 5-min + // outage + RESTART FAIL on immich (gate 2026-07-09). + members } else { ordered_containers_for_start(package_id).await? }; @@ -1036,6 +1068,13 @@ async fn do_orchestrator_package_stop( errors.push(format!("{}: {:#}", name, e)); } } + // A member whose container is already gone IS stopped: quadlet + // stop removes the --rm container before the orchestrator's + // defensive `podman stop` fallback probes it, and a stack member + // can be absent outright (stopped earlier, or never revived). + Err(e) if is_missing_container_error(&format!("{:#}", e)) => { + tracing::debug!(container = %name, error = %e, "stop: container already absent — treating as stopped"); + } Err(e) => { tracing::error!(container = %name, error = %e, "orchestrator stop failed"); errors.push(format!("{}: {:#}", name, e)); @@ -1057,6 +1096,25 @@ fn orchestrator_app_id(package_id: &str) -> &str { } } +/// Member APP ids (start order) for an orchestrator-managed stack, or None +/// when there's no orchestrator / the id isn't a known stack. Lifecycle ops +/// on stacks must address members by app id so the orchestrator drives the +/// quadlet .service; addressing live container names falls through the +/// unknown-app-id fallback to raw podman stop/start, which races systemd's +/// --rm cleanup (the container is removed on stop, the start half then finds +/// nothing — immich restart, gate 2026-07-09). +fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Option> { + if !has_orchestrator { + return None; + } + let members = super::dependencies::stack_member_app_ids(package_id); + if members.is_empty() { + None + } else { + Some(members.iter().map(|s| s.to_string()).collect()) + } +} + /// Per-app lifecycle-operation locks. package.start/stop/restart reply /// immediately and run their multi-container sequences in spawned tasks; /// unserialized, back-to-back RPCs interleave those sequences (gate run D,