2026-03-11 11:11:02 +00:00
|
|
|
pub mod collector;
|
2026-03-22 03:30:21 +00:00
|
|
|
pub(crate) mod alerts;
|
|
|
|
|
mod notifications;
|
|
|
|
|
pub mod store;
|
|
|
|
|
mod telemetry;
|
|
|
|
|
pub mod types;
|
|
|
|
|
|
|
|
|
|
// Re-export public types for external consumers
|
|
|
|
|
pub use store::MetricsStore;
|
|
|
|
|
pub use telemetry::spawn_telemetry_reporter;
|
|
|
|
|
pub use types::*;
|
2026-03-11 11:11:02 +00:00
|
|
|
|
2026-03-12 22:19:04 +00:00
|
|
|
use std::path::PathBuf;
|
2026-03-11 11:11:02 +00:00
|
|
|
use std::sync::Arc;
|
2026-03-22 03:30:21 +00:00
|
|
|
use tracing::{debug, warn};
|
2026-03-12 22:19:04 +00:00
|
|
|
|
2026-03-11 11:11:02 +00:00
|
|
|
/// Spawn the background metrics collector (runs every 60 seconds).
|
2026-03-22 03:30:21 +00:00
|
|
|
/// Evaluates alert rules on each snapshot and dispatches notifications.
|
2026-03-11 12:28:44 +00:00
|
|
|
pub fn spawn_metrics_collector(
|
|
|
|
|
store: Arc<MetricsStore>,
|
|
|
|
|
state: Option<Arc<crate::state::StateManager>>,
|
2026-03-12 22:32:19 +00:00
|
|
|
data_dir: Option<PathBuf>,
|
2026-03-11 12:28:44 +00:00
|
|
|
) {
|
2026-03-11 11:11:02 +00:00
|
|
|
tokio::spawn(async move {
|
|
|
|
|
// Wait 30s for system to stabilize after boot
|
|
|
|
|
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
|
|
|
|
|
|
|
|
|
|
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
|
|
|
|
|
|
|
|
|
|
loop {
|
|
|
|
|
interval.tick().await;
|
|
|
|
|
|
|
|
|
|
match collector::collect_snapshot().await {
|
|
|
|
|
Ok(snapshot) => {
|
2026-03-11 12:28:44 +00:00
|
|
|
let alerts = store.check_alerts(&snapshot).await;
|
2026-03-11 11:11:02 +00:00
|
|
|
store.push(snapshot).await;
|
|
|
|
|
debug!("Metrics snapshot collected");
|
2026-03-11 12:28:44 +00:00
|
|
|
|
|
|
|
|
if !alerts.is_empty() {
|
|
|
|
|
if let Some(ref state_mgr) = state {
|
2026-03-22 03:30:21 +00:00
|
|
|
notifications::push_alert_notifications(state_mgr, &alerts).await;
|
2026-03-11 12:28:44 +00:00
|
|
|
}
|
2026-03-12 22:32:19 +00:00
|
|
|
if let Some(ref dir) = data_dir {
|
2026-03-22 03:30:21 +00:00
|
|
|
notifications::deliver_alert_webhooks(dir, &alerts).await;
|
2026-03-12 22:32:19 +00:00
|
|
|
}
|
2026-03-11 12:28:44 +00:00
|
|
|
}
|
2026-03-11 11:11:02 +00:00
|
|
|
}
|
|
|
|
|
Err(e) => {
|
|
|
|
|
warn!("Failed to collect metrics: {}", e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|