From 891cbba46977848d1b943b6aa1050394c4d1e575 Mon Sep 17 00:00:00 2001
From: archipelago <archipelago@localhost>
Date: Thu, 9 Jul 2026 01:10:47 -0400
Subject: [PATCH] fix(lifecycle): serialize per-app start/stop/restart workers
 (FIFO app-op lock)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

package.start/stop/restart reply immediately and run their multi-container
sequences in spawned tasks with no coordination. Back-to-back RPCs on the
same app interleave those sequences: gate run D (.228, 2026-07-09 04:28)
fired stop→start→restart on mempool; the start's member bring-up raced the
still-running stop's member shutdown — archy-mempool-db was stopped 2s
after it started and the stack finished with ZERO containers (tests 95/123/
124). A user double-clicking restart in the UI can do the same.

Workers now take a per-app tokio Mutex (fair/FIFO, keyed by the normalized
orchestrator app id) as their first await, so queued ops run in RPC arrival
order and the final state matches the last request. The RPC reply stays
immediate.

Known limit: container lists are still computed at RPC time (pre-lock), so
a stop landing mid-start can still return 'No containers found' — it can no
longer interleave/corrupt, only error. Reconciler/health-monitor restarts
don't take the lock yet (separate paths, same candidate follow-up).

Also from run D, filed separately: jellyfin/vaultwarden (legacy containers
on a quadlet node) stop fine but the package state sticks at 'stopping' —
'Failed to stop jellyfin.service: Unit not loaded' error path suspected.

Tests: runtime 9/9; archipelago builds clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../src/api/rpc/package/runtime.rs            | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs
index efa8f699..781e809c 100644
--- a/core/archipelago/src/api/rpc/package/runtime.rs
+++ b/core/archipelago/src/api/rpc/package/runtime.rs
@@ -91,7 +91,9 @@ impl RpcHandler {
         ))
         .await;
 
+        let op_lock = app_op_lock(package_id);
         tokio::spawn(async move {
+            let _op_guard = op_lock.lock().await;
             let result = if let Some(orchestrator) = orchestrator.as_ref() {
                 do_orchestrator_package_start(orchestrator.as_ref(), &to_start).await
             } else {
@@ -172,7 +174,9 @@ impl RpcHandler {
         ))
         .await;
 
+        let op_lock = app_op_lock(package_id);
         tokio::spawn(async move {
+            let _op_guard = op_lock.lock().await;
             let result = if let Some(orchestrator) = orchestrator.as_ref() {
                 do_orchestrator_package_stop(orchestrator.as_ref(), &to_stop).await
             } else {
@@ -256,7 +260,9 @@ impl RpcHandler {
         ))
         .await;
 
+        let op_lock = app_op_lock(package_id);
         tokio::spawn(async move {
+            let _op_guard = op_lock.lock().await;
             let result = if let Some(orchestrator) = orchestrator.as_ref() {
                 do_orchestrator_package_restart(orchestrator.as_ref(), &to_restart).await
             } else {
@@ -1051,6 +1057,28 @@ fn orchestrator_app_id(package_id: &str) -> &str {
     }
 }
 
+/// Per-app lifecycle-operation locks. package.start/stop/restart reply
+/// immediately and run their multi-container sequences in spawned tasks;
+/// unserialized, back-to-back RPCs interleave those sequences (gate run D,
+/// 2026-07-09: package.start's member bring-up raced the still-running
+/// package.stop's member shutdown — archy-mempool-db was stopped 2 seconds
+/// after it started and the mempool stack finished with zero containers).
+/// Workers take the app's lock as their first await; tokio's Mutex is fair
+/// (FIFO), so queued operations run in RPC arrival order and the final
+/// state matches the last request.
+static APP_OP_LOCKS: std::sync::LazyLock<
+    std::sync::Mutex<std::collections::HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+> = std::sync::LazyLock::new(Default::default);
+
+fn app_op_lock(package_id: &str) -> Arc<tokio::sync::Mutex<()>> {
+    APP_OP_LOCKS
+        .lock()
+        .expect("APP_OP_LOCKS poisoned")
+        .entry(orchestrator_app_id(package_id).to_string())
+        .or_default()
+        .clone()
+}
+
 fn uses_single_orchestrator_app(package_id: &str) -> bool {
     startup_order(package_id).is_empty()
         && matches!(