diff --git a/core/archipelago/src/api/rpc/package/dependencies.rs b/core/archipelago/src/api/rpc/package/dependencies.rs index 5725248c..5d4a2575 100644 --- a/core/archipelago/src/api/rpc/package/dependencies.rs +++ b/core/archipelago/src/api/rpc/package/dependencies.rs @@ -623,29 +623,9 @@ pub(super) async fn ordered_containers_for_start(package_id: &str) -> Result &'static [&'static str] { - match package_id { - "immich" => &["immich-postgres", "immich-redis", "immich"], - "indeedhub" => &[ - "indeedhub-postgres", - "indeedhub-redis", - "indeedhub-minio", - "indeedhub-relay", - "indeedhub-api", - "indeedhub-ffmpeg", - "indeedhub", - ], - "btcpay-server" | "btcpayserver" | "btcpay" => { - &["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"] - } - "netbird" => &["netbird-server", "netbird-dashboard", "netbird"], - // The legacy umbrella id maps to the split stack (the orchestrator's - // umbrella alias handles this too; listing it here keeps the RPC - // layer's fan-out explicit). - "mempool" | "mempool-web" => { - &["archy-mempool-db", "mempool-api", "archy-mempool-web"] - } - _ => &[], - } + // Canonical table moved to crate::app_ops (shared with the reconciler's + // in-flight-op guard). + crate::app_ops::stack_member_app_ids(package_id) } fn order_present_containers(package_id: &str, containers: Vec) -> Vec { diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index 704966c9..6bef7bcf 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -1210,17 +1210,10 @@ async fn cascade_restart_address_caching_dependents( /// Workers take the app's lock as their first await; tokio's Mutex is fair /// (FIFO), so queued operations run in RPC arrival order and the final /// state matches the last request. -static APP_OP_LOCKS: std::sync::LazyLock< - std::sync::Mutex>>>, -> = std::sync::LazyLock::new(Default::default); - +/// Registry lives in crate::app_ops so background actors (reconciler) can +/// probe in-flight ops without an api ↔ container dependency cycle. fn app_op_lock(package_id: &str) -> Arc> { - APP_OP_LOCKS - .lock() - .expect("APP_OP_LOCKS poisoned") - .entry(orchestrator_app_id(package_id).to_string()) - .or_default() - .clone() + crate::app_ops::op_lock(orchestrator_app_id(package_id)) } fn uses_single_orchestrator_app(package_id: &str) -> bool { diff --git a/core/archipelago/src/app_ops.rs b/core/archipelago/src/app_ops.rs new file mode 100644 index 00000000..e01dd9c3 --- /dev/null +++ b/core/archipelago/src/app_ops.rs @@ -0,0 +1,115 @@ +//! Cross-layer registry of per-app lifecycle-operation locks and stack +//! membership. +//! +//! The RPC layer's package.start/stop/restart workers serialize through +//! these locks (FIFO, see api::rpc::package::runtime). Background actors +//! (the reconciler; eventually the health monitor) must NOT act on an app +//! while a lifecycle op is mid-sequence: the reconciler once saw a stack +//! member "missing" between a restart worker's stop and start halves and +//! repair-recreated it behind systemd's back, killing the worker's fresh +//! container and leaving the unit down for minutes (.228 mempool frontend, +//! gate 2026-07-09). This module lives outside both layers so each can +//! consult the same state without an api ↔ container dependency cycle. + +use std::sync::Arc; + +static APP_OP_LOCKS: std::sync::LazyLock< + std::sync::Mutex>>>, +> = std::sync::LazyLock::new(Default::default); + +/// The per-app lifecycle-operation lock for a (normalized) app key. Workers +/// take this as their first await; tokio's Mutex is fair (FIFO), so queued +/// operations run in RPC arrival order and the final state matches the last +/// request. +pub fn op_lock(app_key: &str) -> Arc> { + APP_OP_LOCKS + .lock() + .expect("APP_OP_LOCKS poisoned") + .entry(app_key.to_string()) + .or_default() + .clone() +} + +/// Member APP ids (start order) for orchestrator-managed stacks. Every entry +/// is a real manifest app id the orchestrator can `start()`/`stop()` so the +/// quadlet .service is driven instead of raw podman racing systemd's --rm +/// cleanup. Single source of truth — the RPC layer re-exports this. +pub fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] { + match package_id { + "immich" => &["immich-postgres", "immich-redis", "immich"], + "indeedhub" => &[ + "indeedhub-postgres", + "indeedhub-redis", + "indeedhub-minio", + "indeedhub-relay", + "indeedhub-api", + "indeedhub-ffmpeg", + "indeedhub", + ], + "btcpay-server" | "btcpayserver" | "btcpay" => { + &["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"] + } + "netbird" => &["netbird-server", "netbird-dashboard", "netbird"], + // The legacy umbrella id maps to the split stack (the orchestrator's + // umbrella alias handles this too; listing it here keeps the RPC + // layer's fan-out explicit). + "mempool" | "mempool-web" => { + &["archy-mempool-db", "mempool-api", "archy-mempool-web"] + } + _ => &[], + } +} + +/// The package whose lifecycle lock covers `app_id`: the stack package when +/// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while +/// they drive archy-mempool-web), otherwise the app itself. +fn owning_package(app_id: &str) -> &str { + const STACKS: &[&str] = &["immich", "indeedhub", "btcpay-server", "netbird", "mempool"]; + for stack in STACKS { + if stack_member_app_ids(stack).contains(&app_id) { + return stack; + } + } + app_id +} + +/// True when a package.start/stop/restart worker currently holds the +/// lifecycle lock covering `app_id` (under its own key or its owning stack +/// package's key). Background actors use this to skip the app for a cycle +/// instead of interleaving with the worker's multi-step sequence. try_lock +/// on a fair tokio Mutex is non-blocking and does not queue. +pub fn lifecycle_op_in_flight(app_id: &str) -> bool { + let keys = [app_id, owning_package(app_id)]; + for key in keys { + let lock = op_lock(key); + let held = lock.try_lock().is_err(); + if held { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owning_package_maps_members_to_stack() { + assert_eq!(owning_package("archy-mempool-web"), "mempool"); + assert_eq!(owning_package("immich-postgres"), "immich"); + assert_eq!(owning_package("indeedhub-relay"), "indeedhub"); + assert_eq!(owning_package("archy-nbxplorer"), "btcpay-server"); + assert_eq!(owning_package("lnd"), "lnd"); + } + + #[tokio::test] + async fn in_flight_reflects_held_package_lock() { + assert!(!lifecycle_op_in_flight("archy-mempool-web")); + let lock = op_lock("mempool"); + let _guard = lock.lock().await; + assert!(lifecycle_op_in_flight("archy-mempool-web")); + assert!(lifecycle_op_in_flight("mempool")); + assert!(!lifecycle_op_in_flight("jellyfin")); + } +} diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 9af0d518..df4e6b8c 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -1482,6 +1482,18 @@ impl ProdContainerOrchestrator { for lm in manifests { let app_id = lm.manifest.app.id.clone(); let container_name = compute_container_name(&lm.manifest); + // A package.start/stop/restart worker is mid-sequence on this app + // (or its stack): between its stop and start halves the container + // is legitimately absent, and repair-recreating it here races the + // worker — the reconciler once killed a restart's fresh container + // and left the unit down for minutes (.228 mempool frontend, gate + // 2026-07-09). Skip this cycle; the worker owns the outcome. + if crate::app_ops::lifecycle_op_in_flight(&app_id) { + report.record(&app_id, ReconcileAction::Left("lifecycle-op-in-flight".into())); + crate::crash_recovery::pending_boot_start_done(&app_id); + crate::crash_recovery::pending_boot_start_done(&container_name); + continue; + } if mode == ReconcileMode::ExistingOnly && requires_archival_bitcoin(&app_id) && disk_gb < ARCHIVAL_BITCOIN_DISK_GB @@ -1544,6 +1556,11 @@ impl ProdContainerOrchestrator { .iter() .filter(|c| matches!(c.state, ContainerState::Running)) { + // Same in-flight guard as the main loop: don't podman-restart + // a container a lifecycle worker is mid-way through cycling. + if crate::app_ops::lifecycle_op_in_flight(&c.name) { + continue; + } if ensure_running_container_ownership(&c.name).await { tracing::info!(container = %c.name, "volume ownership repaired during reconcile — restarting to recover"); let _ = tokio::process::Command::new("podman") diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index 23805fb9..0db5dca4 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -26,6 +26,7 @@ use tokio::sync::Notify; use tracing::info; mod api; +mod app_ops; mod auth; mod avatar; mod backup;