From cfdf2da9a597c465f97603e6ac19d86b51d6b603 Mon Sep 17 00:00:00 2001
From: archipelago <archipelago@localhost>
Date: Fri, 10 Jul 2026 08:48:36 -0400
Subject: [PATCH] fix(health): monitor skips apps with a lifecycle op in flight
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The health monitor restarted containers that were legitimately down
between a package.start/stop/restart worker's stop and start halves —
the same interleave class as the reconciler's mempool repair-recreate
(e275494a). Before restarting, probe the app_ops op-lock registry via
the container name, the derived app id, and a '_'->'-' normalization
(legacy stack containers are underscore-named while lock keys use
hyphenated app ids) and skip the cycle when a worker holds the lock.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 core/archipelago/src/health_monitor.rs | 52 ++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/core/archipelago/src/health_monitor.rs b/core/archipelago/src/health_monitor.rs
index 55b03c59..0f1489c7 100644
--- a/core/archipelago/src/health_monitor.rs
+++ b/core/archipelago/src/health_monitor.rs
@@ -147,6 +147,21 @@ fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> b
     })
 }
 
+/// True when a lifecycle op lock covering this container is held. Probes the
+/// container name, the derived app id, and a '_'→'-' normalization of the
+/// name — legacy stack containers are named with underscores
+/// (immich_postgres) while op-lock keys and the stack-member table use the
+/// hyphenated app ids.
+fn lifecycle_op_covers_container(container_name: &str, app_id: &str) -> bool {
+    let hyphenated = container_name.replace('_', "-");
+    let mut keys = vec![container_name, app_id];
+    if hyphenated != container_name {
+        keys.push(hyphenated.as_str());
+    }
+    keys.iter()
+        .any(|key| crate::app_ops::lifecycle_op_in_flight(key))
+}
+
 /// Track restart attempts per container with exponential backoff and stability reset.
 struct RestartTracker {
     attempts: HashMap<String, u32>,
@@ -1057,6 +1072,19 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
                     continue;
                 }
 
+                // A package.start/stop/restart worker is mid-sequence on this
+                // app (or its owning stack): the container is legitimately
+                // down between the worker's stop and start halves. Restarting
+                // it here races the worker the same way the reconciler's
+                // repair-recreate did (e275494a) — skip for this cycle.
+                if lifecycle_op_covers_container(&container.name, &container.app_id) {
+                    debug!(
+                        "Skipping auto-restart for {}: lifecycle operation in flight",
+                        container.name
+                    );
+                    continue;
+                }
+
                 // When transitioning to a higher tier, wait briefly for previous tier to stabilize
                 if let Some(prev) = prev_tier {
                     if tier > prev {
@@ -1456,6 +1484,30 @@ mod tests {
         assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
     }
 
+    #[tokio::test]
+    async fn lifecycle_guard_covers_name_app_id_and_legacy_underscores() {
+        assert!(!lifecycle_op_covers_container("hm-guard-app", "hm-guard-app"));
+
+        // Held package lock covers the container directly and via stack
+        // membership; the '_'→'-' probe covers legacy underscore names.
+        let lock = crate::app_ops::op_lock("immich");
+        let _guard = lock.lock().await;
+        assert!(lifecycle_op_covers_container(
+            "immich-postgres",
+            "immich-postgres"
+        ));
+        assert!(lifecycle_op_covers_container(
+            "immich_postgres",
+            "immich_postgres"
+        ));
+        assert!(!lifecycle_op_covers_container("grafana", "grafana"));
+
+        // archy- prefixed containers probe via the stripped app id too.
+        let lnd = crate::app_ops::op_lock("lnd");
+        let _lnd_guard = lnd.lock().await;
+        assert!(lifecycle_op_covers_container("archy-lnd", "lnd"));
+    }
+
     #[test]
     fn test_container_tier_dependent() {
         assert_eq!(container_tier("lnd"), StartupTier::DependentService);