fix(health): monitor skips apps with a lifecycle op in flight
The health monitor restarted containers that were legitimately down between a package.start/stop/restart worker's stop and start halves — the same interleave class as the reconciler's mempool repair-recreate (e275494a). Before restarting, probe the app_ops op-lock registry via the container name, the derived app id, and a '_'->'-' normalization (legacy stack containers are underscore-named while lock keys use hyphenated app ids) and skip the cycle when a worker holds the lock. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
6fe623556c
commit
cfdf2da9a5
@ -147,6 +147,21 @@ fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> b
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// True when a lifecycle op lock covering this container is held. Probes the
|
||||||
|
/// container name, the derived app id, and a '_'→'-' normalization of the
|
||||||
|
/// name — legacy stack containers are named with underscores
|
||||||
|
/// (immich_postgres) while op-lock keys and the stack-member table use the
|
||||||
|
/// hyphenated app ids.
|
||||||
|
fn lifecycle_op_covers_container(container_name: &str, app_id: &str) -> bool {
|
||||||
|
let hyphenated = container_name.replace('_', "-");
|
||||||
|
let mut keys = vec![container_name, app_id];
|
||||||
|
if hyphenated != container_name {
|
||||||
|
keys.push(hyphenated.as_str());
|
||||||
|
}
|
||||||
|
keys.iter()
|
||||||
|
.any(|key| crate::app_ops::lifecycle_op_in_flight(key))
|
||||||
|
}
|
||||||
|
|
||||||
/// Track restart attempts per container with exponential backoff and stability reset.
|
/// Track restart attempts per container with exponential backoff and stability reset.
|
||||||
struct RestartTracker {
|
struct RestartTracker {
|
||||||
attempts: HashMap<String, u32>,
|
attempts: HashMap<String, u32>,
|
||||||
@ -1057,6 +1072,19 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A package.start/stop/restart worker is mid-sequence on this
|
||||||
|
// app (or its owning stack): the container is legitimately
|
||||||
|
// down between the worker's stop and start halves. Restarting
|
||||||
|
// it here races the worker the same way the reconciler's
|
||||||
|
// repair-recreate did (e275494a) — skip for this cycle.
|
||||||
|
if lifecycle_op_covers_container(&container.name, &container.app_id) {
|
||||||
|
debug!(
|
||||||
|
"Skipping auto-restart for {}: lifecycle operation in flight",
|
||||||
|
container.name
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
|
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
|
||||||
if let Some(prev) = prev_tier {
|
if let Some(prev) = prev_tier {
|
||||||
if tier > prev {
|
if tier > prev {
|
||||||
@ -1456,6 +1484,30 @@ mod tests {
|
|||||||
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
|
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lifecycle_guard_covers_name_app_id_and_legacy_underscores() {
|
||||||
|
assert!(!lifecycle_op_covers_container("hm-guard-app", "hm-guard-app"));
|
||||||
|
|
||||||
|
// Held package lock covers the container directly and via stack
|
||||||
|
// membership; the '_'→'-' probe covers legacy underscore names.
|
||||||
|
let lock = crate::app_ops::op_lock("immich");
|
||||||
|
let _guard = lock.lock().await;
|
||||||
|
assert!(lifecycle_op_covers_container(
|
||||||
|
"immich-postgres",
|
||||||
|
"immich-postgres"
|
||||||
|
));
|
||||||
|
assert!(lifecycle_op_covers_container(
|
||||||
|
"immich_postgres",
|
||||||
|
"immich_postgres"
|
||||||
|
));
|
||||||
|
assert!(!lifecycle_op_covers_container("grafana", "grafana"));
|
||||||
|
|
||||||
|
// archy- prefixed containers probe via the stripped app id too.
|
||||||
|
let lnd = crate::app_ops::op_lock("lnd");
|
||||||
|
let _lnd_guard = lnd.lock().await;
|
||||||
|
assert!(lifecycle_op_covers_container("archy-lnd", "lnd"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_container_tier_dependent() {
|
fn test_container_tier_dependent() {
|
||||||
assert_eq!(container_tier("lnd"), StartupTier::DependentService);
|
assert_eq!(container_tier("lnd"), StartupTier::DependentService);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user