diff --git a/core/archipelago/src/health_monitor.rs b/core/archipelago/src/health_monitor.rs index 55b03c59..0f1489c7 100644 --- a/core/archipelago/src/health_monitor.rs +++ b/core/archipelago/src/health_monitor.rs @@ -147,6 +147,21 @@ fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> b }) } +/// True when a lifecycle op lock covering this container is held. Probes the +/// container name, the derived app id, and a '_'→'-' normalization of the +/// name — legacy stack containers are named with underscores +/// (immich_postgres) while op-lock keys and the stack-member table use the +/// hyphenated app ids. +fn lifecycle_op_covers_container(container_name: &str, app_id: &str) -> bool { + let hyphenated = container_name.replace('_', "-"); + let mut keys = vec![container_name, app_id]; + if hyphenated != container_name { + keys.push(hyphenated.as_str()); + } + keys.iter() + .any(|key| crate::app_ops::lifecycle_op_in_flight(key)) +} + /// Track restart attempts per container with exponential backoff and stability reset. struct RestartTracker { attempts: HashMap, @@ -1057,6 +1072,19 @@ pub fn spawn_health_monitor(state: Arc, data_dir: PathBuf) { continue; } + // A package.start/stop/restart worker is mid-sequence on this + // app (or its owning stack): the container is legitimately + // down between the worker's stop and start halves. Restarting + // it here races the worker the same way the reconciler's + // repair-recreate did (e275494a) — skip for this cycle. + if lifecycle_op_covers_container(&container.name, &container.app_id) { + debug!( + "Skipping auto-restart for {}: lifecycle operation in flight", + container.name + ); + continue; + } + // When transitioning to a higher tier, wait briefly for previous tier to stabilize if let Some(prev) = prev_tier { if tier > prev { @@ -1456,6 +1484,30 @@ mod tests { assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra); } + #[tokio::test] + async fn lifecycle_guard_covers_name_app_id_and_legacy_underscores() { + assert!(!lifecycle_op_covers_container("hm-guard-app", "hm-guard-app")); + + // Held package lock covers the container directly and via stack + // membership; the '_'→'-' probe covers legacy underscore names. + let lock = crate::app_ops::op_lock("immich"); + let _guard = lock.lock().await; + assert!(lifecycle_op_covers_container( + "immich-postgres", + "immich-postgres" + )); + assert!(lifecycle_op_covers_container( + "immich_postgres", + "immich_postgres" + )); + assert!(!lifecycle_op_covers_container("grafana", "grafana")); + + // archy- prefixed containers probe via the stripped app id too. + let lnd = crate::app_ops::op_lock("lnd"); + let _lnd_guard = lnd.lock().await; + assert!(lifecycle_op_covers_container("archy-lnd", "lnd")); + } + #[test] fn test_container_tier_dependent() { assert_eq!(container_tier("lnd"), StartupTier::DependentService);