lfg2025/archy
1
0
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 44 Wiki Activity

fix(lifecycle): route stack stop/restart via member app ids, not container names

Browse Source 
package.stop/restart on orchestrator-managed stacks (immich, indeedhub,
btcpay, netbird, mempool) addressed live CONTAINER names; unknown app ids
fell through to raw podman stop/start, racing systemd's --rm cleanup of the
quadlet unit — the container vanished on stop and the start half found
nothing (immich RESTART FAIL + indeedhub stuck restarting, gate 2026-07-09).

Now stack ops resolve member APP ids (reverse start order for stop) so the
orchestrator drives the quadlet .service; user-stop markers are written for
member app ids too (covers absent containers), and the orchestrator stop
fan-out treats an already-missing container as stopped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-09 04:18:55 -04:00
parent 75cb9f9aff
commit 6d60082997
1 changed files with 59 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
core/archipelago/src/api/rpc/package/runtime.rs
View File

@ -63,6 +63,8 @@ impl RpcHandler {
let to_start = if self.orchestrator.is_some() && uses_single_orchestrator_app(package_id) {
vec![orchestrator_app_id(package_id).to_string()]
} else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) {
members
} else {
ordered_containers_for_start(package_id).await?
};
@ -153,6 +155,22 @@ impl RpcHandler {
return Err(anyhow::anyhow!("No containers found for {}", package_id));
}
// Orchestrator-managed stacks are stopped via member APP ids (reverse
// start order), never live container names: orchestrator.stop(app_id)
// stops the member's quadlet .service (systemd removes the --rm
// container), while a container NAME falls through the unknown-app-id
// fallback to a raw `podman stop` that races systemd over the unit
// (immich, gate 2026-07-09).
let to_stop_ids = if !single_orchestrator_app {
orchestrator_stack_members(self.orchestrator.is_some(), package_id)
.map(|mut members| {
members.reverse();
members
})
} else {
None
};
// Mark as user-stopped BEFORE the spawn so health monitor and
// crash recovery don't auto-restart mid-flight. Ordering is
// load-bearing — see runtime.rs:145-148 original note.
@ -160,9 +178,17 @@ impl RpcHandler {
for name in &containers {
crate::crash_recovery::mark_user_stopped(&self.config.data_dir, name).await;
}
// Stack members are marked under their app ids too, so the reconcile
// guard holds for members whose container is currently absent (a live
// container list can't name those).
if let Some(ids) = &to_stop_ids {
for id in ids {
crate::crash_recovery::mark_user_stopped(&self.config.data_dir, id).await;
}
}
let package_id_owned = package_id.to_string();
let to_stop = containers.clone();
let to_stop = to_stop_ids.unwrap_or_else(|| containers.clone());
let orchestrator = self.orchestrator.clone();
let state_manager = Arc::clone(&self.state_manager);
let pre_state =
@ -246,6 +272,12 @@ impl RpcHandler {
let companion_app_id = package_id_owned.clone();
let to_restart = if single_orchestrator_app {
vec![orchestrator_app_id(package_id).to_string()]
} else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) {
// Restart stacks via member APP ids: restarting by live container
// name podman-stops the quadlet container (systemd --rm removes
// it) and the start half then finds no such container — a 5-min
// outage + RESTART FAIL on immich (gate 2026-07-09).
members
} else {
ordered_containers_for_start(package_id).await?
};
@ -1036,6 +1068,13 @@ async fn do_orchestrator_package_stop(
errors.push(format!("{}: {:#}", name, e));
}
}
// A member whose container is already gone IS stopped: quadlet
// stop removes the --rm container before the orchestrator's
// defensive `podman stop` fallback probes it, and a stack member
// can be absent outright (stopped earlier, or never revived).
Err(e) if is_missing_container_error(&format!("{:#}", e)) => {
tracing::debug!(container = %name, error = %e, "stop: container already absent — treating as stopped");
}
Err(e) => {
tracing::error!(container = %name, error = %e, "orchestrator stop failed");
errors.push(format!("{}: {:#}", name, e));
@ -1057,6 +1096,25 @@ fn orchestrator_app_id(package_id: &str) -> &str {
}
}
/// Member APP ids (start order) for an orchestrator-managed stack, or None
/// when there's no orchestrator / the id isn't a known stack. Lifecycle ops
/// on stacks must address members by app id so the orchestrator drives the
/// quadlet .service; addressing live container names falls through the
/// unknown-app-id fallback to raw podman stop/start, which races systemd's
/// --rm cleanup (the container is removed on stop, the start half then finds
/// nothing — immich restart, gate 2026-07-09).
fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Option<Vec<String>> {
if !has_orchestrator {
return None;
}
let members = super::dependencies::stack_member_app_ids(package_id);
if members.is_empty() {
None
} else {
Some(members.iter().map(|s| s.to_string()).collect())
}
}
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
/// immediately and run their multi-container sequences in spawned tasks;
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,