fix(lifecycle): route stack stop/restart via member app ids, not container names
package.stop/restart on orchestrator-managed stacks (immich, indeedhub, btcpay, netbird, mempool) addressed live CONTAINER names; unknown app ids fell through to raw podman stop/start, racing systemd's --rm cleanup of the quadlet unit — the container vanished on stop and the start half found nothing (immich RESTART FAIL + indeedhub stuck restarting, gate 2026-07-09). Now stack ops resolve member APP ids (reverse start order for stop) so the orchestrator drives the quadlet .service; user-stop markers are written for member app ids too (covers absent containers), and the orchestrator stop fan-out treats an already-missing container as stopped. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
75cb9f9aff
commit
6d60082997
@ -63,6 +63,8 @@ impl RpcHandler {
|
|||||||
|
|
||||||
let to_start = if self.orchestrator.is_some() && uses_single_orchestrator_app(package_id) {
|
let to_start = if self.orchestrator.is_some() && uses_single_orchestrator_app(package_id) {
|
||||||
vec![orchestrator_app_id(package_id).to_string()]
|
vec![orchestrator_app_id(package_id).to_string()]
|
||||||
|
} else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) {
|
||||||
|
members
|
||||||
} else {
|
} else {
|
||||||
ordered_containers_for_start(package_id).await?
|
ordered_containers_for_start(package_id).await?
|
||||||
};
|
};
|
||||||
@ -153,6 +155,22 @@ impl RpcHandler {
|
|||||||
return Err(anyhow::anyhow!("No containers found for {}", package_id));
|
return Err(anyhow::anyhow!("No containers found for {}", package_id));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Orchestrator-managed stacks are stopped via member APP ids (reverse
|
||||||
|
// start order), never live container names: orchestrator.stop(app_id)
|
||||||
|
// stops the member's quadlet .service (systemd removes the --rm
|
||||||
|
// container), while a container NAME falls through the unknown-app-id
|
||||||
|
// fallback to a raw `podman stop` that races systemd over the unit
|
||||||
|
// (immich, gate 2026-07-09).
|
||||||
|
let to_stop_ids = if !single_orchestrator_app {
|
||||||
|
orchestrator_stack_members(self.orchestrator.is_some(), package_id)
|
||||||
|
.map(|mut members| {
|
||||||
|
members.reverse();
|
||||||
|
members
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
// Mark as user-stopped BEFORE the spawn so health monitor and
|
// Mark as user-stopped BEFORE the spawn so health monitor and
|
||||||
// crash recovery don't auto-restart mid-flight. Ordering is
|
// crash recovery don't auto-restart mid-flight. Ordering is
|
||||||
// load-bearing — see runtime.rs:145-148 original note.
|
// load-bearing — see runtime.rs:145-148 original note.
|
||||||
@ -160,9 +178,17 @@ impl RpcHandler {
|
|||||||
for name in &containers {
|
for name in &containers {
|
||||||
crate::crash_recovery::mark_user_stopped(&self.config.data_dir, name).await;
|
crate::crash_recovery::mark_user_stopped(&self.config.data_dir, name).await;
|
||||||
}
|
}
|
||||||
|
// Stack members are marked under their app ids too, so the reconcile
|
||||||
|
// guard holds for members whose container is currently absent (a live
|
||||||
|
// container list can't name those).
|
||||||
|
if let Some(ids) = &to_stop_ids {
|
||||||
|
for id in ids {
|
||||||
|
crate::crash_recovery::mark_user_stopped(&self.config.data_dir, id).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let package_id_owned = package_id.to_string();
|
let package_id_owned = package_id.to_string();
|
||||||
let to_stop = containers.clone();
|
let to_stop = to_stop_ids.unwrap_or_else(|| containers.clone());
|
||||||
let orchestrator = self.orchestrator.clone();
|
let orchestrator = self.orchestrator.clone();
|
||||||
let state_manager = Arc::clone(&self.state_manager);
|
let state_manager = Arc::clone(&self.state_manager);
|
||||||
let pre_state =
|
let pre_state =
|
||||||
@ -246,6 +272,12 @@ impl RpcHandler {
|
|||||||
let companion_app_id = package_id_owned.clone();
|
let companion_app_id = package_id_owned.clone();
|
||||||
let to_restart = if single_orchestrator_app {
|
let to_restart = if single_orchestrator_app {
|
||||||
vec![orchestrator_app_id(package_id).to_string()]
|
vec![orchestrator_app_id(package_id).to_string()]
|
||||||
|
} else if let Some(members) = orchestrator_stack_members(self.orchestrator.is_some(), package_id) {
|
||||||
|
// Restart stacks via member APP ids: restarting by live container
|
||||||
|
// name podman-stops the quadlet container (systemd --rm removes
|
||||||
|
// it) and the start half then finds no such container — a 5-min
|
||||||
|
// outage + RESTART FAIL on immich (gate 2026-07-09).
|
||||||
|
members
|
||||||
} else {
|
} else {
|
||||||
ordered_containers_for_start(package_id).await?
|
ordered_containers_for_start(package_id).await?
|
||||||
};
|
};
|
||||||
@ -1036,6 +1068,13 @@ async fn do_orchestrator_package_stop(
|
|||||||
errors.push(format!("{}: {:#}", name, e));
|
errors.push(format!("{}: {:#}", name, e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A member whose container is already gone IS stopped: quadlet
|
||||||
|
// stop removes the --rm container before the orchestrator's
|
||||||
|
// defensive `podman stop` fallback probes it, and a stack member
|
||||||
|
// can be absent outright (stopped earlier, or never revived).
|
||||||
|
Err(e) if is_missing_container_error(&format!("{:#}", e)) => {
|
||||||
|
tracing::debug!(container = %name, error = %e, "stop: container already absent — treating as stopped");
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!(container = %name, error = %e, "orchestrator stop failed");
|
tracing::error!(container = %name, error = %e, "orchestrator stop failed");
|
||||||
errors.push(format!("{}: {:#}", name, e));
|
errors.push(format!("{}: {:#}", name, e));
|
||||||
@ -1057,6 +1096,25 @@ fn orchestrator_app_id(package_id: &str) -> &str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Member APP ids (start order) for an orchestrator-managed stack, or None
|
||||||
|
/// when there's no orchestrator / the id isn't a known stack. Lifecycle ops
|
||||||
|
/// on stacks must address members by app id so the orchestrator drives the
|
||||||
|
/// quadlet .service; addressing live container names falls through the
|
||||||
|
/// unknown-app-id fallback to raw podman stop/start, which races systemd's
|
||||||
|
/// --rm cleanup (the container is removed on stop, the start half then finds
|
||||||
|
/// nothing — immich restart, gate 2026-07-09).
|
||||||
|
fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Option<Vec<String>> {
|
||||||
|
if !has_orchestrator {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let members = super::dependencies::stack_member_app_ids(package_id);
|
||||||
|
if members.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(members.iter().map(|s| s.to_string()).collect())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
|
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
|
||||||
/// immediately and run their multi-container sequences in spawned tasks;
|
/// immediately and run their multi-container sequences in spawned tasks;
|
||||||
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,
|
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user