46350f48b6
Pure formatter output — no semantic changes. Sweeping these into their own commit so the FIPS integration diff that follows stays scoped to the actual feature. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1012 lines
37 KiB
Rust
1012 lines
37 KiB
Rust
// Container Health Monitor
|
|
// Checks container health every 120s, auto-restarts unhealthy containers (max 10 times)
|
|
// with exponential backoff (10s..120s), dependency-aware restart ordering (deps first),
|
|
// handles "created" state containers, resets dependent counters when deps recover,
|
|
// and sends WebSocket notifications to the UI on failure.
|
|
|
|
use crate::data_model::{Notification, NotificationLevel};
|
|
use crate::state::StateManager;
|
|
use crate::webhooks::{self, WebhookEvent};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
use tracing::{debug, info, warn};
|
|
|
|
const MAX_RESTART_ATTEMPTS: u32 = 10;
|
|
const CHECK_INTERVAL_SECS: u64 = 120;
|
|
/// Backoff delays per attempt — escalating from 10s to 120s
|
|
const BACKOFF_DELAYS_SECS: [u64; 10] = [10, 15, 20, 30, 30, 45, 60, 60, 90, 120];
|
|
/// Reset restart counter after 1 hour of stability
|
|
const STABILITY_RESET_SECS: u64 = 3600;
|
|
|
|
/// Container startup tier for dependency ordering.
|
|
/// Lower tiers start first. Containers in the same tier start together.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
enum StartupTier {
|
|
/// Databases: postgres, redis, mariadb, mysql
|
|
Database = 0,
|
|
/// Core infrastructure: bitcoin-knots, bitcoin-core
|
|
CoreInfra = 1,
|
|
/// Services depending on core: lnd, electrs, nbxplorer
|
|
DependentService = 2,
|
|
/// Application layer: mempool-api, btcpay-server, fedimint, nextcloud, etc.
|
|
Application = 3,
|
|
/// UI/frontend containers: mempool-web, bitcoin-ui, lnd-ui
|
|
Frontend = 4,
|
|
}
|
|
|
|
fn container_tier(name: &str) -> StartupTier {
|
|
let id = name.strip_prefix("archy-").unwrap_or(name);
|
|
match id {
|
|
// Tier 0: Databases and data stores
|
|
"btcpay-db" | "mempool-db" | "mysql-mempool" | "penpot-postgres" | "immich_postgres"
|
|
| "immich_redis" | "penpot-valkey" | "endurain-db" | "nextcloud-db"
|
|
| "indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" => StartupTier::Database,
|
|
|
|
// Tier 1: Core infrastructure
|
|
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => StartupTier::CoreInfra,
|
|
|
|
// Tier 2: Dependent services (need databases or bitcoin)
|
|
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer" | "mempool-api"
|
|
| "indeedhub-api" => StartupTier::DependentService,
|
|
|
|
// Tier 4: Frontend/UI
|
|
"mempool-web" | "bitcoin-ui" | "lnd-ui" | "electrs-ui" | "penpot-frontend"
|
|
| "penpot-exporter" | "indeedhub" => StartupTier::Frontend,
|
|
|
|
// Tier 3: Application layer (everything else)
|
|
_ => StartupTier::Application,
|
|
}
|
|
}
|
|
|
|
/// Map containers to their required dependencies.
|
|
/// When a dependent fails, check and restart its dependencies first.
|
|
fn container_dependencies(name: &str) -> &'static [&'static str] {
|
|
let id = name.strip_prefix("archy-").unwrap_or(name);
|
|
match id {
|
|
// Bitcoin-dependent chain
|
|
"lnd" => &["bitcoin-knots"],
|
|
"electrumx" | "mempool-electrs" | "electrs" => &["bitcoin-knots"],
|
|
"nbxplorer" => &["bitcoin-knots"],
|
|
"btcpay-server" => &["btcpay-db", "nbxplorer"],
|
|
"mempool-api" => &["mempool-db", "electrumx"],
|
|
"mempool-web" => &["mempool-api"],
|
|
"fedimint" => &["bitcoin-knots"],
|
|
"fedimint-gateway" => &["bitcoin-knots", "fedimint"],
|
|
|
|
// IndeedHub stack
|
|
"indeedhub-api" => &["indeedhub-postgres", "indeedhub-redis"],
|
|
"indeedhub" => &["indeedhub-api"],
|
|
"indeedhub-relay" => &["indeedhub-postgres"],
|
|
"indeedhub-ffmpeg" => &["indeedhub-api"],
|
|
|
|
// Multi-container stacks
|
|
"immich_server" => &["immich_postgres", "immich_redis"],
|
|
"penpot-backend" => &["penpot-postgres", "penpot-valkey"],
|
|
"penpot-frontend" => &["penpot-backend"],
|
|
|
|
// UI containers
|
|
"bitcoin-ui" => &["bitcoin-knots"],
|
|
"lnd-ui" => &["lnd"],
|
|
"electrs-ui" => &["electrumx"],
|
|
|
|
_ => &[],
|
|
}
|
|
}
|
|
|
|
/// Check if all of a container's dependencies are currently running.
|
|
fn deps_are_running(name: &str, containers: &[ContainerHealth]) -> bool {
|
|
let deps = container_dependencies(name);
|
|
if deps.is_empty() {
|
|
return true;
|
|
}
|
|
for dep in deps {
|
|
// Check both plain name and archy- prefixed name
|
|
let dep_running = containers.iter().any(|c| {
|
|
let c_id = c.name.strip_prefix("archy-").unwrap_or(&c.name);
|
|
(c_id == *dep || c.name == *dep) && c.state == "running"
|
|
});
|
|
if !dep_running {
|
|
return false;
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
/// Track restart attempts per container with exponential backoff and stability reset.
|
|
struct RestartTracker {
|
|
attempts: HashMap<String, u32>,
|
|
last_failure: HashMap<String, Instant>,
|
|
last_healthy: HashMap<String, Instant>,
|
|
}
|
|
|
|
impl RestartTracker {
|
|
fn new() -> Self {
|
|
Self {
|
|
attempts: HashMap::new(),
|
|
last_failure: HashMap::new(),
|
|
last_healthy: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Record a restart attempt. Returns false if max attempts exceeded.
|
|
fn record_attempt(&mut self, name: &str) -> bool {
|
|
let count = self.attempts.entry(name.to_string()).or_insert(0);
|
|
*count += 1;
|
|
self.last_failure.insert(name.to_string(), Instant::now());
|
|
*count <= MAX_RESTART_ATTEMPTS
|
|
}
|
|
|
|
/// Clear restart count when a container is healthy again.
|
|
fn clear(&mut self, name: &str) {
|
|
self.attempts.remove(name);
|
|
self.last_failure.remove(name);
|
|
self.last_healthy.insert(name.to_string(), Instant::now());
|
|
}
|
|
|
|
fn attempt_count(&self, name: &str) -> u32 {
|
|
*self.attempts.get(name).unwrap_or(&0)
|
|
}
|
|
|
|
/// Get the backoff delay in seconds for the current attempt number.
|
|
fn backoff_delay_secs(&self, name: &str) -> u64 {
|
|
let attempts = self.attempt_count(name);
|
|
if attempts == 0 {
|
|
return BACKOFF_DELAYS_SECS[0];
|
|
}
|
|
let idx = (attempts as usize)
|
|
.saturating_sub(1)
|
|
.min(BACKOFF_DELAYS_SECS.len() - 1);
|
|
BACKOFF_DELAYS_SECS[idx]
|
|
}
|
|
|
|
/// Check if enough time has passed since last failure for the backoff delay.
|
|
fn backoff_elapsed(&self, name: &str) -> bool {
|
|
let delay = self.backoff_delay_secs(name);
|
|
match self.last_failure.get(name) {
|
|
Some(last) => last.elapsed().as_secs() >= delay,
|
|
None => true,
|
|
}
|
|
}
|
|
|
|
/// Check if a failed container should have its counter reset (1h stability window).
|
|
fn should_reset_failed(&self, name: &str) -> bool {
|
|
if self.attempt_count(name) < MAX_RESTART_ATTEMPTS {
|
|
return false;
|
|
}
|
|
match self.last_failure.get(name) {
|
|
Some(last) => last.elapsed().as_secs() >= STABILITY_RESET_SECS,
|
|
None => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ContainerHealth {
|
|
name: String,
|
|
app_id: String,
|
|
state: String,
|
|
healthy: bool,
|
|
}
|
|
|
|
/// Track container memory usage over time for leak detection.
|
|
struct MemoryTracker {
|
|
/// Per-container memory samples: (timestamp, rss_bytes)
|
|
samples: HashMap<String, Vec<(Instant, u64)>>,
|
|
}
|
|
|
|
impl MemoryTracker {
|
|
fn new() -> Self {
|
|
Self {
|
|
samples: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Record a memory sample for a container.
|
|
fn record(&mut self, name: &str, rss_bytes: u64) {
|
|
let entry = self.samples.entry(name.to_string()).or_default();
|
|
entry.push((Instant::now(), rss_bytes));
|
|
// Keep only last 288 samples (24h at 5min intervals)
|
|
if entry.len() > 288 {
|
|
entry.remove(0);
|
|
}
|
|
}
|
|
|
|
/// Check if a container's memory has grown by more than 50% over the tracking period.
|
|
/// Returns Some(growth_percent) if a leak is detected, None otherwise.
|
|
fn check_leak(&self, name: &str) -> Option<f64> {
|
|
let samples = self.samples.get(name)?;
|
|
if samples.len() < 12 {
|
|
return None; // Need at least 1 hour of data
|
|
}
|
|
let (oldest_time, oldest_rss) = samples.first()?;
|
|
let (_, latest_rss) = samples.last()?;
|
|
let elapsed_hours = oldest_time.elapsed().as_secs() as f64 / 3600.0;
|
|
if elapsed_hours < 1.0 || *oldest_rss == 0 {
|
|
return None;
|
|
}
|
|
let growth = (*latest_rss as f64 - *oldest_rss as f64) / *oldest_rss as f64 * 100.0;
|
|
if growth > 50.0 {
|
|
Some(growth)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Persistent restart tracking ────────────────────────────────────────
|
|
// Survives process restarts so a container can't loop infinitely by
|
|
// crashing 3 times → triggering process restart → resetting counter → repeat.
|
|
|
|
const RESTART_HISTORY_FILE: &str = "restart-tracker.json";
|
|
|
|
#[derive(Serialize, Deserialize, Default)]
|
|
struct RestartHistory {
|
|
containers: HashMap<String, ContainerRestartRecord>,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Clone)]
|
|
struct ContainerRestartRecord {
|
|
attempts: u32,
|
|
last_failure_epoch: i64,
|
|
}
|
|
|
|
impl RestartHistory {
|
|
async fn load(data_dir: &Path) -> Self {
|
|
let path = data_dir.join(RESTART_HISTORY_FILE);
|
|
match tokio::fs::read_to_string(&path).await {
|
|
Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
|
|
Err(_) => Self::default(),
|
|
}
|
|
}
|
|
|
|
async fn save(&self, data_dir: &Path) {
|
|
let path = data_dir.join(RESTART_HISTORY_FILE);
|
|
if let Ok(json) = serde_json::to_string(self) {
|
|
let _ = tokio::fs::write(&path, json).await;
|
|
}
|
|
}
|
|
|
|
/// Seed the in-memory RestartTracker from persisted history.
|
|
fn seed_tracker(&self, tracker: &mut RestartTracker) {
|
|
let now_epoch = chrono::Utc::now().timestamp();
|
|
for (name, record) in &self.containers {
|
|
// Only seed if last failure was within the stability window
|
|
let secs_since_failure = now_epoch - record.last_failure_epoch;
|
|
if secs_since_failure < STABILITY_RESET_SECS as i64 && record.attempts > 0 {
|
|
tracker.attempts.insert(name.clone(), record.attempts);
|
|
info!(
|
|
"Restored restart counter for {}: {} attempts ({}s ago)",
|
|
name, record.attempts, secs_since_failure
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn record_attempt(&mut self, name: &str) {
|
|
let entry = self
|
|
.containers
|
|
.entry(name.to_string())
|
|
.or_insert(ContainerRestartRecord {
|
|
attempts: 0,
|
|
last_failure_epoch: 0,
|
|
});
|
|
entry.attempts += 1;
|
|
entry.last_failure_epoch = chrono::Utc::now().timestamp();
|
|
}
|
|
|
|
fn clear(&mut self, name: &str) {
|
|
self.containers.remove(name);
|
|
}
|
|
}
|
|
|
|
/// Query container memory stats from podman.
|
|
async fn check_container_memory() -> HashMap<String, u64> {
|
|
let output = match tokio::time::timeout(
|
|
std::time::Duration::from_secs(30),
|
|
tokio::process::Command::new("podman")
|
|
.args([
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{.Name}} {{.MemUsage}}",
|
|
])
|
|
.output(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(Ok(o)) if o.status.success() => o,
|
|
Ok(Err(e)) => {
|
|
debug!("podman stats failed: {}", e);
|
|
return HashMap::new();
|
|
}
|
|
Err(_) => {
|
|
debug!("podman stats timed out (30s)");
|
|
return HashMap::new();
|
|
}
|
|
_ => return HashMap::new(),
|
|
};
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let mut result = HashMap::new();
|
|
for line in stdout.lines() {
|
|
let parts: Vec<&str> = line.split_whitespace().collect();
|
|
if parts.len() >= 2 {
|
|
let name = parts[0].to_string();
|
|
// Parse memory like "123.4MiB", "1.2GiB", "45.6kB"
|
|
let mem_str = parts[1];
|
|
if let Some(bytes) = parse_memory_string(mem_str) {
|
|
result.insert(name, bytes);
|
|
}
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Parse memory string like "123.4MiB" or "1.2GiB" to bytes.
|
|
fn parse_memory_string(s: &str) -> Option<u64> {
|
|
let s = s.trim();
|
|
if s.ends_with("GiB") {
|
|
let num: f64 = s.strip_suffix("GiB")?.parse().ok()?;
|
|
Some((num * 1_073_741_824.0) as u64)
|
|
} else if s.ends_with("MiB") {
|
|
let num: f64 = s.strip_suffix("MiB")?.parse().ok()?;
|
|
Some((num * 1_048_576.0) as u64)
|
|
} else if s.ends_with("KiB") || s.ends_with("kB") {
|
|
let suffix = if s.ends_with("KiB") { "KiB" } else { "kB" };
|
|
let num: f64 = s.strip_suffix(suffix)?.parse().ok()?;
|
|
Some((num * 1024.0) as u64)
|
|
} else if s.ends_with("B") {
|
|
let num: f64 = s.strip_suffix('B')?.parse().ok()?;
|
|
Some(num as u64)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Query all containers and their health status.
|
|
async fn check_containers() -> Vec<ContainerHealth> {
|
|
let output = match tokio::time::timeout(
|
|
std::time::Duration::from_secs(60),
|
|
tokio::process::Command::new("podman")
|
|
.args(["ps", "-a", "--format", "json"])
|
|
.output(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(Ok(o)) if o.status.success() => o,
|
|
Ok(Err(e)) => {
|
|
debug!("podman ps failed: {}", e);
|
|
return Vec::new();
|
|
}
|
|
Err(_) => {
|
|
debug!("podman ps timed out (60s)");
|
|
return Vec::new();
|
|
}
|
|
_ => return Vec::new(),
|
|
};
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let containers: Vec<serde_json::Value> = serde_json::from_str(&stdout).unwrap_or_default();
|
|
|
|
// Monitor ALL long-running containers for health — backend services (databases,
|
|
// nbxplorer, mempool-api) and UI containers need auto-restart too.
|
|
// Only skip ephemeral containers (build infrastructure, init one-shots).
|
|
|
|
containers
|
|
.iter()
|
|
.filter_map(|c| {
|
|
let name = c.get("Names").and_then(|v| {
|
|
if let Some(arr) = v.as_array() {
|
|
arr.first().and_then(|n| n.as_str()).map(|s| s.to_string())
|
|
} else {
|
|
v.as_str().map(|s| s.to_string())
|
|
}
|
|
})?;
|
|
|
|
// Skip podman-compose infrastructure and one-shot init containers
|
|
if name.starts_with("indeedhub-build_") || name.contains("-init") {
|
|
return None;
|
|
}
|
|
|
|
let app_id = name.strip_prefix("archy-").unwrap_or(&name).to_string();
|
|
|
|
let state = c
|
|
.get("State")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("unknown")
|
|
.to_lowercase();
|
|
|
|
let healthy = state == "running";
|
|
|
|
Some(ContainerHealth {
|
|
name,
|
|
app_id,
|
|
state,
|
|
healthy,
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Try to restart a container.
|
|
async fn restart_container(name: &str) -> bool {
|
|
info!("Auto-restarting unhealthy container: {}", name);
|
|
let result = tokio::time::timeout(
|
|
std::time::Duration::from_secs(120),
|
|
tokio::process::Command::new("podman")
|
|
.args(["start", name])
|
|
.output(),
|
|
)
|
|
.await;
|
|
|
|
match result {
|
|
Ok(Ok(output)) if output.status.success() => {
|
|
info!("Successfully restarted container: {}", name);
|
|
true
|
|
}
|
|
Ok(Ok(output)) => {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
warn!("Failed to restart container {}: {}", name, stderr.trim());
|
|
false
|
|
}
|
|
Ok(Err(e)) => {
|
|
warn!("Failed to execute podman start for {}: {}", name, e);
|
|
false
|
|
}
|
|
Err(_) => {
|
|
warn!("Timeout starting container {} (120s)", name);
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Spawn the health monitor background task.
|
|
pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
|
tokio::spawn(async move {
|
|
// Wait for boot recovery to complete before starting health checks.
|
|
// This prevents the health monitor from fighting with crash_recovery
|
|
// which is starting containers in tier order.
|
|
info!("Health monitor: waiting for boot recovery to complete...");
|
|
let wait_start = std::time::Instant::now();
|
|
loop {
|
|
if crate::crash_recovery::is_recovery_complete() {
|
|
break;
|
|
}
|
|
// Safety timeout: start anyway after 5 minutes even if recovery hangs
|
|
if wait_start.elapsed().as_secs() > 300 {
|
|
warn!("Health monitor: boot recovery did not complete within 5 minutes, starting anyway");
|
|
break;
|
|
}
|
|
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
|
}
|
|
// Additional cooldown after recovery to let containers stabilize
|
|
info!("Health monitor: recovery done, waiting 60s for containers to stabilize...");
|
|
tokio::time::sleep(std::time::Duration::from_secs(60)).await;
|
|
info!("Health monitor: starting health checks");
|
|
|
|
let mut tracker = RestartTracker::new();
|
|
let mut mem_tracker = MemoryTracker::new();
|
|
let mut mem_check_counter: u32 = 0;
|
|
let mut interval =
|
|
tokio::time::interval(std::time::Duration::from_secs(CHECK_INTERVAL_SECS));
|
|
// Skip missed ticks — prevents burst of health checks after slow podman response
|
|
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
|
|
|
// Load persistent restart history and seed the in-memory tracker
|
|
let mut restart_history = RestartHistory::load(&data_dir).await;
|
|
restart_history.seed_tracker(&mut tracker);
|
|
#[allow(unused_assignments)]
|
|
let mut history_dirty = false;
|
|
|
|
loop {
|
|
interval.tick().await;
|
|
mem_check_counter += 1;
|
|
|
|
// Check container memory every 5 minutes (every 5th health check)
|
|
if mem_check_counter.is_multiple_of(5) {
|
|
let mem_stats = check_container_memory().await;
|
|
for (name, rss) in &mem_stats {
|
|
mem_tracker.record(name, *rss);
|
|
if let Some(growth) = mem_tracker.check_leak(name) {
|
|
warn!(
|
|
"Potential memory leak in {}: {:.0}% growth over tracking period",
|
|
name, growth
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
let containers = check_containers().await;
|
|
if containers.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Load user-stopped list to skip intentionally stopped containers
|
|
let user_stopped = crate::crash_recovery::load_user_stopped(&data_dir).await;
|
|
|
|
// Sort containers by startup tier so databases restart before dependent services
|
|
let mut unhealthy: Vec<&ContainerHealth> = Vec::new();
|
|
let mut state_changed = false;
|
|
let (mut data, _) = state.get_snapshot().await;
|
|
|
|
for container in &containers {
|
|
// Skip optional/marketplace containers that aren't installed
|
|
if let Some(pkg) = data.package_data.get(&container.app_id) {
|
|
if pkg.installed.is_none() {
|
|
debug!("Skipping uninstalled container: {}", container.name);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if container.healthy {
|
|
if tracker.attempt_count(&container.name) > 0 {
|
|
info!(
|
|
"Container {} is healthy again after restart",
|
|
container.name
|
|
);
|
|
// Reset attempt counters for containers that depend on this one,
|
|
// since their previous failures may have been caused by this
|
|
// dependency being down
|
|
let recovered_id = container
|
|
.name
|
|
.strip_prefix("archy-")
|
|
.unwrap_or(&container.name)
|
|
.to_string();
|
|
for other in &containers {
|
|
let deps = container_dependencies(&other.name);
|
|
if deps
|
|
.iter()
|
|
.any(|d| *d == recovered_id || *d == container.name)
|
|
&& tracker.attempt_count(&other.name) > 0
|
|
{
|
|
info!(
|
|
"Resetting restart counter for {} (dependency {} recovered)",
|
|
other.name, container.name
|
|
);
|
|
tracker.clear(&other.name);
|
|
restart_history.clear(&other.name);
|
|
}
|
|
}
|
|
tracker.clear(&container.name);
|
|
restart_history.clear(&container.name);
|
|
history_dirty = true;
|
|
}
|
|
continue;
|
|
}
|
|
// Handle exited, stopped, AND created state containers
|
|
if container.state == "exited"
|
|
|| container.state == "stopped"
|
|
|| container.state == "created"
|
|
{
|
|
// Skip user-stopped containers
|
|
if user_stopped.contains(&container.name) {
|
|
debug!("Skipping user-stopped container: {}", container.name);
|
|
continue;
|
|
}
|
|
unhealthy.push(container);
|
|
}
|
|
}
|
|
|
|
// Sort by startup tier: databases first, then core, then dependent, then apps, then UIs
|
|
unhealthy.sort_by_key(|c| container_tier(&c.name));
|
|
|
|
let mut prev_tier: Option<StartupTier> = None;
|
|
for container in &unhealthy {
|
|
let tier = container_tier(&container.name);
|
|
|
|
// Reset counter after 1 hour for permanently failed containers
|
|
if tracker.should_reset_failed(&container.name) {
|
|
info!(
|
|
"Resetting restart counter for {} after {}s stability window",
|
|
container.name, STABILITY_RESET_SECS
|
|
);
|
|
tracker.clear(&container.name);
|
|
restart_history.clear(&container.name);
|
|
history_dirty = true;
|
|
}
|
|
|
|
if tracker.attempt_count(&container.name) >= MAX_RESTART_ATTEMPTS {
|
|
debug!(
|
|
"Container {} exceeded max restart attempts ({})",
|
|
container.name, MAX_RESTART_ATTEMPTS
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Wait for backoff delay before retrying
|
|
if !tracker.backoff_elapsed(&container.name) {
|
|
let delay = tracker.backoff_delay_secs(&container.name);
|
|
debug!(
|
|
"Container {} waiting for backoff ({}s)",
|
|
container.name, delay
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Skip if dependencies aren't running — they need to start first
|
|
if !deps_are_running(&container.name, &containers) {
|
|
let deps = container_dependencies(&container.name);
|
|
debug!(
|
|
"Container {} waiting for dependencies {:?}",
|
|
container.name, deps
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
|
|
if let Some(prev) = prev_tier {
|
|
if tier > prev {
|
|
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
|
}
|
|
}
|
|
prev_tier = Some(tier);
|
|
|
|
if tracker.record_attempt(&container.name) {
|
|
restart_history.record_attempt(&container.name);
|
|
history_dirty = true;
|
|
let attempt = tracker.attempt_count(&container.name);
|
|
info!(
|
|
"Restarting {} (tier {:?}, attempt {}/{}, backoff {}s)",
|
|
container.name,
|
|
tier,
|
|
attempt,
|
|
MAX_RESTART_ATTEMPTS,
|
|
BACKOFF_DELAYS_SECS
|
|
.get(attempt.saturating_sub(1) as usize)
|
|
.unwrap_or(&90)
|
|
);
|
|
|
|
let restarted = restart_container(&container.name).await;
|
|
|
|
if !restarted || attempt >= MAX_RESTART_ATTEMPTS {
|
|
let notification = Notification {
|
|
id: format!(
|
|
"health-{}-{}",
|
|
container.app_id,
|
|
chrono::Utc::now().timestamp()
|
|
),
|
|
level: NotificationLevel::Error,
|
|
title: format!("{} is unhealthy", container.app_id),
|
|
message: if restarted {
|
|
format!(
|
|
"Container restarted ({}/{} attempts). May need manual attention.",
|
|
attempt, MAX_RESTART_ATTEMPTS
|
|
)
|
|
} else {
|
|
format!(
|
|
"Auto-restart failed (attempt {}/{}). Container state: {}",
|
|
attempt, MAX_RESTART_ATTEMPTS, container.state
|
|
)
|
|
},
|
|
timestamp: chrono::Utc::now().to_rfc3339(),
|
|
app_id: Some(container.app_id.clone()),
|
|
};
|
|
|
|
data.notifications.push(notification.clone());
|
|
if data.notifications.len() > 20 {
|
|
data.notifications =
|
|
data.notifications.split_off(data.notifications.len() - 20);
|
|
}
|
|
state_changed = true;
|
|
|
|
let webhook_payload = webhooks::WebhookPayload {
|
|
event: WebhookEvent::ContainerCrash,
|
|
title: notification.title,
|
|
message: notification.message,
|
|
timestamp: notification.timestamp,
|
|
node_id: String::new(),
|
|
details: Some(serde_json::json!({
|
|
"container": container.name,
|
|
"app_id": container.app_id,
|
|
"state": container.state,
|
|
"attempt": attempt,
|
|
"tier": format!("{:?}", tier),
|
|
})),
|
|
};
|
|
webhooks::send_webhook(&data_dir, webhook_payload).await;
|
|
}
|
|
}
|
|
}
|
|
|
|
if state_changed {
|
|
state.update_data(data).await;
|
|
debug!("Health monitor: state updated with notifications");
|
|
}
|
|
|
|
// Persist restart history to disk (debounced: once per check cycle)
|
|
if history_dirty {
|
|
restart_history.save(&data_dir).await;
|
|
history_dirty = false;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_restart_tracker_new_is_empty() {
|
|
let tracker = RestartTracker::new();
|
|
assert_eq!(tracker.attempt_count("any-container"), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_record_attempt_increments() {
|
|
let mut tracker = RestartTracker::new();
|
|
assert!(tracker.record_attempt("test-container"));
|
|
assert_eq!(tracker.attempt_count("test-container"), 1);
|
|
|
|
assert!(tracker.record_attempt("test-container"));
|
|
assert_eq!(tracker.attempt_count("test-container"), 2);
|
|
|
|
assert!(tracker.record_attempt("test-container"));
|
|
assert_eq!(tracker.attempt_count("test-container"), 3);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_max_attempts_exceeded() {
|
|
let mut tracker = RestartTracker::new();
|
|
for i in 1..=MAX_RESTART_ATTEMPTS {
|
|
assert!(
|
|
tracker.record_attempt("container-a"),
|
|
"Attempt {} should be allowed",
|
|
i
|
|
);
|
|
}
|
|
assert!(!tracker.record_attempt("container-a"));
|
|
assert_eq!(
|
|
tracker.attempt_count("container-a"),
|
|
MAX_RESTART_ATTEMPTS + 1
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_independent_containers() {
|
|
let mut tracker = RestartTracker::new();
|
|
tracker.record_attempt("container-a");
|
|
tracker.record_attempt("container-a");
|
|
tracker.record_attempt("container-b");
|
|
|
|
assert_eq!(tracker.attempt_count("container-a"), 2);
|
|
assert_eq!(tracker.attempt_count("container-b"), 1);
|
|
assert_eq!(tracker.attempt_count("container-c"), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_clear_resets_count() {
|
|
let mut tracker = RestartTracker::new();
|
|
tracker.record_attempt("container-x");
|
|
tracker.record_attempt("container-x");
|
|
assert_eq!(tracker.attempt_count("container-x"), 2);
|
|
|
|
tracker.clear("container-x");
|
|
assert_eq!(tracker.attempt_count("container-x"), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_clear_allows_new_attempts() {
|
|
let mut tracker = RestartTracker::new();
|
|
for _ in 0..=MAX_RESTART_ATTEMPTS {
|
|
tracker.record_attempt("container-y");
|
|
}
|
|
assert!(!tracker.record_attempt("container-y"));
|
|
|
|
tracker.clear("container-y");
|
|
assert!(tracker.record_attempt("container-y"));
|
|
assert_eq!(tracker.attempt_count("container-y"), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_restart_tracker_clear_nonexistent_is_safe() {
|
|
let mut tracker = RestartTracker::new();
|
|
tracker.clear("nonexistent");
|
|
assert_eq!(tracker.attempt_count("nonexistent"), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_health_struct() {
|
|
let health = ContainerHealth {
|
|
name: "archy-bitcoin-knots".to_string(),
|
|
app_id: "bitcoin-knots".to_string(),
|
|
state: "running".to_string(),
|
|
healthy: true,
|
|
};
|
|
assert!(health.healthy);
|
|
assert_eq!(health.name, "archy-bitcoin-knots");
|
|
assert_eq!(health.app_id, "bitcoin-knots");
|
|
assert_eq!(health.state, "running");
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_health_unhealthy() {
|
|
let health = ContainerHealth {
|
|
name: "archy-mempool-web".to_string(),
|
|
app_id: "mempool-web".to_string(),
|
|
state: "exited".to_string(),
|
|
healthy: false,
|
|
};
|
|
assert!(!health.healthy);
|
|
assert_eq!(health.state, "exited");
|
|
}
|
|
|
|
#[test]
|
|
fn test_max_restart_attempts_constant() {
|
|
assert!(MAX_RESTART_ATTEMPTS >= 1);
|
|
assert!(MAX_RESTART_ATTEMPTS <= 20);
|
|
assert_eq!(MAX_RESTART_ATTEMPTS, 10);
|
|
}
|
|
|
|
#[test]
|
|
fn test_check_interval_constant() {
|
|
assert_eq!(CHECK_INTERVAL_SECS, 120);
|
|
}
|
|
|
|
#[test]
|
|
fn test_backoff_delays() {
|
|
let tracker = RestartTracker::new();
|
|
// Before any attempts, delay is first backoff
|
|
assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[0]);
|
|
}
|
|
|
|
#[test]
|
|
fn test_backoff_delays_escalate() {
|
|
let mut tracker = RestartTracker::new();
|
|
tracker.record_attempt("test");
|
|
assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[0]); // 10s
|
|
tracker.record_attempt("test");
|
|
assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[1]); // 30s
|
|
tracker.record_attempt("test");
|
|
assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[2]); // 90s
|
|
}
|
|
|
|
#[test]
|
|
fn test_backoff_elapsed_true_for_new() {
|
|
let tracker = RestartTracker::new();
|
|
assert!(tracker.backoff_elapsed("test"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_stability_reset_not_triggered_early() {
|
|
let mut tracker = RestartTracker::new();
|
|
tracker.record_attempt("test");
|
|
assert!(!tracker.should_reset_failed("test"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_database() {
|
|
assert_eq!(container_tier("archy-btcpay-db"), StartupTier::Database);
|
|
assert_eq!(container_tier("immich_postgres"), StartupTier::Database);
|
|
assert_eq!(container_tier("penpot-valkey"), StartupTier::Database);
|
|
assert_eq!(container_tier("indeedhub-postgres"), StartupTier::Database);
|
|
assert_eq!(container_tier("indeedhub-redis"), StartupTier::Database);
|
|
assert_eq!(container_tier("indeedhub-minio"), StartupTier::Database);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_indeedhub_api() {
|
|
assert_eq!(
|
|
container_tier("indeedhub-api"),
|
|
StartupTier::DependentService
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_mempool_api() {
|
|
assert_eq!(container_tier("mempool-api"), StartupTier::DependentService);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_dependencies() {
|
|
assert!(container_dependencies("lnd").contains(&"bitcoin-knots"));
|
|
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-postgres"));
|
|
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-redis"));
|
|
assert!(container_dependencies("mempool-api").contains(&"mempool-db"));
|
|
assert!(container_dependencies("mempool-api").contains(&"electrumx"));
|
|
assert!(container_dependencies("nextcloud").is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_deps_are_running() {
|
|
let containers = vec![
|
|
ContainerHealth {
|
|
name: "indeedhub-postgres".into(),
|
|
app_id: "indeedhub-postgres".into(),
|
|
state: "running".into(),
|
|
healthy: true,
|
|
},
|
|
ContainerHealth {
|
|
name: "indeedhub-redis".into(),
|
|
app_id: "indeedhub-redis".into(),
|
|
state: "running".into(),
|
|
healthy: true,
|
|
},
|
|
ContainerHealth {
|
|
name: "indeedhub-api".into(),
|
|
app_id: "indeedhub-api".into(),
|
|
state: "exited".into(),
|
|
healthy: false,
|
|
},
|
|
];
|
|
assert!(deps_are_running("indeedhub-api", &containers));
|
|
// Missing postgres
|
|
let partial = vec![ContainerHealth {
|
|
name: "indeedhub-redis".into(),
|
|
app_id: "indeedhub-redis".into(),
|
|
state: "running".into(),
|
|
healthy: true,
|
|
}];
|
|
assert!(!deps_are_running("indeedhub-api", &partial));
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_core() {
|
|
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_dependent() {
|
|
assert_eq!(container_tier("lnd"), StartupTier::DependentService);
|
|
assert_eq!(
|
|
container_tier("mempool-electrs"),
|
|
StartupTier::DependentService
|
|
);
|
|
assert_eq!(
|
|
container_tier("archy-nbxplorer"),
|
|
StartupTier::DependentService
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_frontend() {
|
|
assert_eq!(container_tier("archy-mempool-web"), StartupTier::Frontend);
|
|
assert_eq!(container_tier("archy-bitcoin-ui"), StartupTier::Frontend);
|
|
}
|
|
|
|
#[test]
|
|
fn test_container_tier_application_default() {
|
|
assert_eq!(container_tier("nextcloud"), StartupTier::Application);
|
|
assert_eq!(container_tier("grafana"), StartupTier::Application);
|
|
assert_eq!(container_tier("btcpay-server"), StartupTier::Application);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tier_ordering() {
|
|
assert!(StartupTier::Database < StartupTier::CoreInfra);
|
|
assert!(StartupTier::CoreInfra < StartupTier::DependentService);
|
|
assert!(StartupTier::DependentService < StartupTier::Application);
|
|
assert!(StartupTier::Application < StartupTier::Frontend);
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_memory_gib() {
|
|
assert_eq!(parse_memory_string("1.5GiB"), Some(1_610_612_736));
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_memory_mib() {
|
|
assert_eq!(parse_memory_string("256MiB"), Some(268_435_456));
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_memory_kib() {
|
|
assert_eq!(parse_memory_string("512KiB"), Some(524_288));
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_memory_invalid() {
|
|
assert_eq!(parse_memory_string("abc"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_memory_tracker_no_leak_few_samples() {
|
|
let mut tracker = MemoryTracker::new();
|
|
tracker.record("test", 100_000_000);
|
|
assert!(tracker.check_leak("test").is_none());
|
|
}
|
|
}
|