fix(health): monitor skips apps with a lifecycle op in flight
The health monitor restarted containers that were legitimately down between a package.start/stop/restart worker's stop and start halves — the same interleave class as the reconciler's mempool repair-recreate (e275494a). Before restarting, probe the app_ops op-lock registry via the container name, the derived app id, and a '_'->'-' normalization (legacy stack containers are underscore-named while lock keys use hyphenated app ids) and skip the cycle when a worker holds the lock. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
6fe623556c
commit
cfdf2da9a5
@ -147,6 +147,21 @@ fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> b
|
||||
})
|
||||
}
|
||||
|
||||
/// True when a lifecycle op lock covering this container is held. Probes the
|
||||
/// container name, the derived app id, and a '_'→'-' normalization of the
|
||||
/// name — legacy stack containers are named with underscores
|
||||
/// (immich_postgres) while op-lock keys and the stack-member table use the
|
||||
/// hyphenated app ids.
|
||||
fn lifecycle_op_covers_container(container_name: &str, app_id: &str) -> bool {
|
||||
let hyphenated = container_name.replace('_', "-");
|
||||
let mut keys = vec![container_name, app_id];
|
||||
if hyphenated != container_name {
|
||||
keys.push(hyphenated.as_str());
|
||||
}
|
||||
keys.iter()
|
||||
.any(|key| crate::app_ops::lifecycle_op_in_flight(key))
|
||||
}
|
||||
|
||||
/// Track restart attempts per container with exponential backoff and stability reset.
|
||||
struct RestartTracker {
|
||||
attempts: HashMap<String, u32>,
|
||||
@ -1057,6 +1072,19 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// A package.start/stop/restart worker is mid-sequence on this
|
||||
// app (or its owning stack): the container is legitimately
|
||||
// down between the worker's stop and start halves. Restarting
|
||||
// it here races the worker the same way the reconciler's
|
||||
// repair-recreate did (e275494a) — skip for this cycle.
|
||||
if lifecycle_op_covers_container(&container.name, &container.app_id) {
|
||||
debug!(
|
||||
"Skipping auto-restart for {}: lifecycle operation in flight",
|
||||
container.name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
|
||||
if let Some(prev) = prev_tier {
|
||||
if tier > prev {
|
||||
@ -1456,6 +1484,30 @@ mod tests {
|
||||
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_guard_covers_name_app_id_and_legacy_underscores() {
|
||||
assert!(!lifecycle_op_covers_container("hm-guard-app", "hm-guard-app"));
|
||||
|
||||
// Held package lock covers the container directly and via stack
|
||||
// membership; the '_'→'-' probe covers legacy underscore names.
|
||||
let lock = crate::app_ops::op_lock("immich");
|
||||
let _guard = lock.lock().await;
|
||||
assert!(lifecycle_op_covers_container(
|
||||
"immich-postgres",
|
||||
"immich-postgres"
|
||||
));
|
||||
assert!(lifecycle_op_covers_container(
|
||||
"immich_postgres",
|
||||
"immich_postgres"
|
||||
));
|
||||
assert!(!lifecycle_op_covers_container("grafana", "grafana"));
|
||||
|
||||
// archy- prefixed containers probe via the stripped app id too.
|
||||
let lnd = crate::app_ops::op_lock("lnd");
|
||||
let _lnd_guard = lnd.lock().await;
|
||||
assert!(lifecycle_op_covers_container("archy-lnd", "lnd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_container_tier_dependent() {
|
||||
assert_eq!(container_tier("lnd"), StartupTier::DependentService);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user