fix(health): monitor skips apps with a lifecycle op in flight

The health monitor restarted containers that were legitimately down
between a package.start/stop/restart worker's stop and start halves —
the same interleave class as the reconciler's mempool repair-recreate
(e275494a). Before restarting, probe the app_ops op-lock registry via
the container name, the derived app id, and a '_'->'-' normalization
(legacy stack containers are underscore-named while lock keys use
hyphenated app ids) and skip the cycle when a worker holds the lock.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-10 08:48:36 -04:00
parent 6fe623556c
commit cfdf2da9a5

View File

@ -147,6 +147,21 @@ fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> b
}) })
} }
/// True when a lifecycle op lock covering this container is held. Probes the
/// container name, the derived app id, and a '_'→'-' normalization of the
/// name — legacy stack containers are named with underscores
/// (immich_postgres) while op-lock keys and the stack-member table use the
/// hyphenated app ids.
fn lifecycle_op_covers_container(container_name: &str, app_id: &str) -> bool {
let hyphenated = container_name.replace('_', "-");
let mut keys = vec![container_name, app_id];
if hyphenated != container_name {
keys.push(hyphenated.as_str());
}
keys.iter()
.any(|key| crate::app_ops::lifecycle_op_in_flight(key))
}
/// Track restart attempts per container with exponential backoff and stability reset. /// Track restart attempts per container with exponential backoff and stability reset.
struct RestartTracker { struct RestartTracker {
attempts: HashMap<String, u32>, attempts: HashMap<String, u32>,
@ -1057,6 +1072,19 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
continue; continue;
} }
// A package.start/stop/restart worker is mid-sequence on this
// app (or its owning stack): the container is legitimately
// down between the worker's stop and start halves. Restarting
// it here races the worker the same way the reconciler's
// repair-recreate did (e275494a) — skip for this cycle.
if lifecycle_op_covers_container(&container.name, &container.app_id) {
debug!(
"Skipping auto-restart for {}: lifecycle operation in flight",
container.name
);
continue;
}
// When transitioning to a higher tier, wait briefly for previous tier to stabilize // When transitioning to a higher tier, wait briefly for previous tier to stabilize
if let Some(prev) = prev_tier { if let Some(prev) = prev_tier {
if tier > prev { if tier > prev {
@ -1456,6 +1484,30 @@ mod tests {
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra); assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
} }
#[tokio::test]
async fn lifecycle_guard_covers_name_app_id_and_legacy_underscores() {
assert!(!lifecycle_op_covers_container("hm-guard-app", "hm-guard-app"));
// Held package lock covers the container directly and via stack
// membership; the '_'→'-' probe covers legacy underscore names.
let lock = crate::app_ops::op_lock("immich");
let _guard = lock.lock().await;
assert!(lifecycle_op_covers_container(
"immich-postgres",
"immich-postgres"
));
assert!(lifecycle_op_covers_container(
"immich_postgres",
"immich_postgres"
));
assert!(!lifecycle_op_covers_container("grafana", "grafana"));
// archy- prefixed containers probe via the stripped app id too.
let lnd = crate::app_ops::op_lock("lnd");
let _lnd_guard = lnd.lock().await;
assert!(lifecycle_op_covers_container("archy-lnd", "lnd"));
}
#[test] #[test]
fn test_container_tier_dependent() { fn test_container_tier_dependent() {
assert_eq!(container_tier("lnd"), StartupTier::DependentService); assert_eq!(container_tier("lnd"), StartupTier::DependentService);