fix(lifecycle): reconciler skips apps with an in-flight lifecycle op

The reconciler doesn't take the RPC layer's per-app FIFO op lock (known
limit of 891cbba4): between a restart worker's stop and start halves it saw
the mempool frontend "missing", repair-recreated it behind systemd's back,
killed the worker's fresh container 11s after start, and left the unit down
for ~3.5 min until the next heal — gate test 123 measured exactly that
window (.228 iteration 3, 2026-07-09).

New crate::app_ops module owns the op-lock registry + stack member table
(runtime.rs and dependencies.rs now delegate) so the reconciler can probe
lifecycle_op_in_flight(app_id) — covering both the app's own key and its
owning stack package — and skip that app for the cycle. The ownership-sweep
podman restart gets the same guard. Health monitor is a follow-up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/archipelago/src/api/rpc/package/dependencies.rs

@@ -623,29 +623,9 @@ pub(super) async fn ordered_containers_for_start(package_id: &str) -> Result<Vec
 /// other members absent AND user-stopped (.228 indeedhub, 2026-07-09 — the
 /// reconciler then correctly refused to revive them).
 pub(super) fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] {
-    match package_id {
-        "immich" => &["immich-postgres", "immich-redis", "immich"],
-        "indeedhub" => &[
-            "indeedhub-postgres",
-            "indeedhub-redis",
-            "indeedhub-minio",
-            "indeedhub-relay",
-            "indeedhub-api",
-            "indeedhub-ffmpeg",
-            "indeedhub",
-        ],
-        "btcpay-server" | "btcpayserver" | "btcpay" => {
-            &["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"]
-        }
-        "netbird" => &["netbird-server", "netbird-dashboard", "netbird"],
-        // The legacy umbrella id maps to the split stack (the orchestrator's
-        // umbrella alias handles this too; listing it here keeps the RPC
-        // layer's fan-out explicit).
-        "mempool" | "mempool-web" => {
-            &["archy-mempool-db", "mempool-api", "archy-mempool-web"]
-        }
-        _ => &[],
-    }
+    // Canonical table moved to crate::app_ops (shared with the reconciler's
+    // in-flight-op guard).
+    crate::app_ops::stack_member_app_ids(package_id)
 }
 
 fn order_present_containers(package_id: &str, containers: Vec<String>) -> Vec<String> {

core/archipelago/src/api/rpc/package/runtime.rs

@@ -1210,17 +1210,10 @@ async fn cascade_restart_address_caching_dependents(
 /// Workers take the app's lock as their first await; tokio's Mutex is fair
 /// (FIFO), so queued operations run in RPC arrival order and the final
 /// state matches the last request.
-static APP_OP_LOCKS: std::sync::LazyLock<
-    std::sync::Mutex<std::collections::HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
-> = std::sync::LazyLock::new(Default::default);
-
+/// Registry lives in crate::app_ops so background actors (reconciler) can
+/// probe in-flight ops without an api ↔ container dependency cycle.
 fn app_op_lock(package_id: &str) -> Arc<tokio::sync::Mutex<()>> {
-    APP_OP_LOCKS
-        .lock()
-        .expect("APP_OP_LOCKS poisoned")
-        .entry(orchestrator_app_id(package_id).to_string())
-        .or_default()
-        .clone()
+    crate::app_ops::op_lock(orchestrator_app_id(package_id))
 }
 
 fn uses_single_orchestrator_app(package_id: &str) -> bool {

core/archipelago/src/app_ops.rs

@@ -0,0 +1,115 @@
+//! Cross-layer registry of per-app lifecycle-operation locks and stack
+//! membership.
+//!
+//! The RPC layer's package.start/stop/restart workers serialize through
+//! these locks (FIFO, see api::rpc::package::runtime). Background actors
+//! (the reconciler; eventually the health monitor) must NOT act on an app
+//! while a lifecycle op is mid-sequence: the reconciler once saw a stack
+//! member "missing" between a restart worker's stop and start halves and
+//! repair-recreated it behind systemd's back, killing the worker's fresh
+//! container and leaving the unit down for minutes (.228 mempool frontend,
+//! gate 2026-07-09). This module lives outside both layers so each can
+//! consult the same state without an api ↔ container dependency cycle.
+
+use std::sync::Arc;
+
+static APP_OP_LOCKS: std::sync::LazyLock<
+    std::sync::Mutex<std::collections::HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+> = std::sync::LazyLock::new(Default::default);
+
+/// The per-app lifecycle-operation lock for a (normalized) app key. Workers
+/// take this as their first await; tokio's Mutex is fair (FIFO), so queued
+/// operations run in RPC arrival order and the final state matches the last
+/// request.
+pub fn op_lock(app_key: &str) -> Arc<tokio::sync::Mutex<()>> {
+    APP_OP_LOCKS
+        .lock()
+        .expect("APP_OP_LOCKS poisoned")
+        .entry(app_key.to_string())
+        .or_default()
+        .clone()
+}
+
+/// Member APP ids (start order) for orchestrator-managed stacks. Every entry
+/// is a real manifest app id the orchestrator can `start()`/`stop()` so the
+/// quadlet .service is driven instead of raw podman racing systemd's --rm
+/// cleanup. Single source of truth — the RPC layer re-exports this.
+pub fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] {
+    match package_id {
+        "immich" => &["immich-postgres", "immich-redis", "immich"],
+        "indeedhub" => &[
+            "indeedhub-postgres",
+            "indeedhub-redis",
+            "indeedhub-minio",
+            "indeedhub-relay",
+            "indeedhub-api",
+            "indeedhub-ffmpeg",
+            "indeedhub",
+        ],
+        "btcpay-server" | "btcpayserver" | "btcpay" => {
+            &["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"]
+        }
+        "netbird" => &["netbird-server", "netbird-dashboard", "netbird"],
+        // The legacy umbrella id maps to the split stack (the orchestrator's
+        // umbrella alias handles this too; listing it here keeps the RPC
+        // layer's fan-out explicit).
+        "mempool" | "mempool-web" => {
+            &["archy-mempool-db", "mempool-api", "archy-mempool-web"]
+        }
+        _ => &[],
+    }
+}
+
+/// The package whose lifecycle lock covers `app_id`: the stack package when
+/// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while
+/// they drive archy-mempool-web), otherwise the app itself.
+fn owning_package(app_id: &str) -> &str {
+    const STACKS: &[&str] = &["immich", "indeedhub", "btcpay-server", "netbird", "mempool"];
+    for stack in STACKS {
+        if stack_member_app_ids(stack).contains(&app_id) {
+            return stack;
+        }
+    }
+    app_id
+}
+
+/// True when a package.start/stop/restart worker currently holds the
+/// lifecycle lock covering `app_id` (under its own key or its owning stack
+/// package's key). Background actors use this to skip the app for a cycle
+/// instead of interleaving with the worker's multi-step sequence. try_lock
+/// on a fair tokio Mutex is non-blocking and does not queue.
+pub fn lifecycle_op_in_flight(app_id: &str) -> bool {
+    let keys = [app_id, owning_package(app_id)];
+    for key in keys {
+        let lock = op_lock(key);
+        let held = lock.try_lock().is_err();
+        if held {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn owning_package_maps_members_to_stack() {
+        assert_eq!(owning_package("archy-mempool-web"), "mempool");
+        assert_eq!(owning_package("immich-postgres"), "immich");
+        assert_eq!(owning_package("indeedhub-relay"), "indeedhub");
+        assert_eq!(owning_package("archy-nbxplorer"), "btcpay-server");
+        assert_eq!(owning_package("lnd"), "lnd");
+    }
+
+    #[tokio::test]
+    async fn in_flight_reflects_held_package_lock() {
+        assert!(!lifecycle_op_in_flight("archy-mempool-web"));
+        let lock = op_lock("mempool");
+        let _guard = lock.lock().await;
+        assert!(lifecycle_op_in_flight("archy-mempool-web"));
+        assert!(lifecycle_op_in_flight("mempool"));
+        assert!(!lifecycle_op_in_flight("jellyfin"));
+    }
+}

core/archipelago/src/container/prod_orchestrator.rs

@@ -1482,6 +1482,18 @@ impl ProdContainerOrchestrator {
         for lm in manifests {
             let app_id = lm.manifest.app.id.clone();
             let container_name = compute_container_name(&lm.manifest);
+            // A package.start/stop/restart worker is mid-sequence on this app
+            // (or its stack): between its stop and start halves the container
+            // is legitimately absent, and repair-recreating it here races the
+            // worker — the reconciler once killed a restart's fresh container
+            // and left the unit down for minutes (.228 mempool frontend, gate
+            // 2026-07-09). Skip this cycle; the worker owns the outcome.
+            if crate::app_ops::lifecycle_op_in_flight(&app_id) {
+                report.record(&app_id, ReconcileAction::Left("lifecycle-op-in-flight".into()));
+                crate::crash_recovery::pending_boot_start_done(&app_id);
+                crate::crash_recovery::pending_boot_start_done(&container_name);
+                continue;
+            }
             if mode == ReconcileMode::ExistingOnly
                 && requires_archival_bitcoin(&app_id)
                 && disk_gb < ARCHIVAL_BITCOIN_DISK_GB
@@ -1544,6 +1556,11 @@ impl ProdContainerOrchestrator {
                 .iter()
                 .filter(|c| matches!(c.state, ContainerState::Running))
             {
+                // Same in-flight guard as the main loop: don't podman-restart
+                // a container a lifecycle worker is mid-way through cycling.
+                if crate::app_ops::lifecycle_op_in_flight(&c.name) {
+                    continue;
+                }
                 if ensure_running_container_ownership(&c.name).await {
                     tracing::info!(container = %c.name, "volume ownership repaired during reconcile — restarting to recover");
                     let _ = tokio::process::Command::new("podman")

core/archipelago/src/main.rs

@@ -26,6 +26,7 @@ use tokio::sync::Notify;
 use tracing::info;
 
 mod api;
+mod app_ops;
 mod auth;
 mod avatar;
 mod backup;