Files
Mangalord/backend/src/api/admin/system.rs
MechaCat02 314fc8738b feat(admin): operational health checks page with editable thresholds
New /admin/health rolls up signals already collected — disk/memory sensors,
dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata
cron freshness, and the live crawler session/browser flags — into a flat
checks list (ok/warn/critical) with an overall status and remediation links.
The overview gains a compact Health summary banner linking in.

GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via
try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists
the thresholds to app_settings (merged over compiled defaults, forward-compat
via serde(default)), validates ranges (400 on bad input), and audit-logs the
change in one transaction. New repo::crawler::last_metadata_tick_at getter and
a lightweight system::memory_percent_used (no CPU settle delay).

Pure status helpers (over_limit for gauges, exceeds for backlog counts so a
0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape,
gating, persistence+audit, and validation.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-19 11:38:18 +02:00

250 lines
7.5 KiB
Rust

//! System metrics for the admin dashboard.
//!
//! Disk is `statvfs(storage_dir)` so the number reflects the volume the
//! app actually writes to (not the root filesystem of the host). When the
//! storage backend doesn't expose a local path (e.g. a future S3 impl)
//! the disk fields are `null` rather than fabricated.
//!
//! Memory and CPU come from `sysinfo`. CPU requires two refreshes with
//! at least 200ms between them to compute a meaningful delta; the
//! handler eats the 250ms wall-clock cost on each request. Admin
//! traffic is low-volume so a background cache isn't worth the moving
//! parts yet — revisit if polling becomes frequent.
use std::path::Path;
use std::time::Duration;
use axum::extract::State;
use axum::routing::get;
use axum::{Json, Router};
use serde::Serialize;
use sysinfo::{Components, CpuRefreshKind, MemoryRefreshKind, RefreshKind, System};
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::error::AppResult;
const ALERT_THRESHOLD_PERCENT: f64 = 90.0;
pub fn routes() -> Router<AppState> {
Router::new().route("/admin/system", get(system))
}
#[derive(Debug, Serialize)]
pub struct SystemStats {
pub disk: Option<DiskStats>,
pub memory: MemoryStats,
pub cpu: CpuStats,
/// Hardware temperature sensors. Empty when the platform doesn't
/// expose any (common inside containers without `/sys` access) —
/// the frontend renders an "unavailable" state, not an error.
pub temperatures: Vec<TempStat>,
pub alerts: Vec<Alert>,
}
#[derive(Debug, Serialize)]
pub struct DiskStats {
pub total_bytes: u64,
pub used_bytes: u64,
pub free_bytes: u64,
pub percent_used: f64,
}
#[derive(Debug, Serialize)]
pub struct MemoryStats {
pub total_bytes: u64,
pub used_bytes: u64,
pub percent_used: f64,
}
#[derive(Debug, Serialize)]
pub struct CpuStats {
pub percent_used: f64,
pub load_avg: LoadAvg,
/// Per-core usage percentages, one entry per logical core.
pub per_core: Vec<f64>,
}
#[derive(Debug, Serialize)]
pub struct LoadAvg {
pub one: f64,
pub five: f64,
pub fifteen: f64,
}
#[derive(Debug, Serialize)]
pub struct TempStat {
pub label: String,
pub celsius: f64,
/// Highest reading seen since boot, when the kernel exposes it.
pub max_celsius: Option<f64>,
/// Vendor critical/shutdown threshold, when available.
pub critical_celsius: Option<f64>,
}
#[derive(Debug, Serialize)]
pub struct Alert {
pub level: AlertLevel,
pub message: String,
}
#[derive(Debug, Serialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum AlertLevel {
Warning,
}
async fn system(
State(state): State<AppState>,
_admin: RequireAdmin,
) -> AppResult<Json<SystemStats>> {
let disk = state.storage.local_root().and_then(disk_stats_for);
let (memory, cpu) = memory_and_cpu().await;
let temperatures = temperatures();
let mut alerts = Vec::new();
if let Some(d) = &disk {
if d.percent_used >= ALERT_THRESHOLD_PERCENT {
alerts.push(Alert {
level: AlertLevel::Warning,
message: format!(
"disk near full ({:.0}% used)",
d.percent_used
),
});
}
}
if memory.percent_used >= ALERT_THRESHOLD_PERCENT {
alerts.push(Alert {
level: AlertLevel::Warning,
message: format!(
"memory near full ({:.0}% used)",
memory.percent_used
),
});
}
for t in &temperatures {
if let Some(crit) = t.critical_celsius {
if t.celsius >= crit {
alerts.push(Alert {
level: AlertLevel::Warning,
message: format!(
"{} temperature critical ({:.0}°C / {:.0}°C)",
t.label, t.celsius, crit
),
});
}
}
}
Ok(Json(SystemStats {
disk,
memory,
cpu,
temperatures,
alerts,
}))
}
/// Hardware temperature sensors via `sysinfo`'s `Components`. Components
/// whose temperature can't be read report `f32::NAN` on Linux — we skip
/// those so the dashboard never shows a bogus "NaN°C" row. Returns an
/// empty Vec when no sensors are exposed at all.
fn temperatures() -> Vec<TempStat> {
let components = Components::new_with_refreshed_list();
components
.list()
.iter()
.filter_map(|c| {
let celsius = c.temperature();
if celsius.is_nan() {
return None;
}
let max = c.max();
Some(TempStat {
label: c.label().to_string(),
celsius: celsius as f64,
max_celsius: (!max.is_nan()).then_some(max as f64),
critical_celsius: c.critical().map(|v| v as f64),
})
})
.collect()
}
/// Lightweight memory-utilization percentage for the health check. Unlike
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
/// health is polled more often and only needs the memory figure.
pub(crate) fn memory_percent_used() -> f64 {
let mut sys =
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
sys.refresh_memory();
let total = sys.total_memory();
if total == 0 {
0.0
} else {
(sys.used_memory() as f64) * 100.0 / (total as f64)
}
}
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
let s = nix::sys::statvfs::statvfs(root).ok()?;
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
// is "free to non-root callers" which is what an operator actually
// cares about — `f_bfree` includes blocks reserved for root.
let block = s.fragment_size();
let total = block * s.blocks();
let avail = block * s.blocks_available();
let used = total.saturating_sub(avail);
let percent_used = if total > 0 {
(used as f64) * 100.0 / (total as f64)
} else {
0.0
};
Some(DiskStats {
total_bytes: total,
used_bytes: used,
free_bytes: avail,
percent_used,
})
}
async fn memory_and_cpu() -> (MemoryStats, CpuStats) {
// sysinfo's CPU sampling needs two refreshes with a delay between
// them — the first seeds the delta counters, the second measures.
// We do this once per request; admin traffic is low enough that the
// 250ms cost is invisible.
let mut sys = System::new_with_specifics(
RefreshKind::new()
.with_cpu(CpuRefreshKind::everything())
.with_memory(MemoryRefreshKind::everything()),
);
sys.refresh_cpu_all();
// Yield the runtime instead of blocking it for the gap.
tokio::time::sleep(Duration::from_millis(250)).await;
sys.refresh_cpu_all();
sys.refresh_memory();
let total = sys.total_memory();
let used = sys.used_memory();
let mem_pct = if total > 0 {
(used as f64) * 100.0 / (total as f64)
} else {
0.0
};
let memory = MemoryStats {
total_bytes: total,
used_bytes: used,
percent_used: mem_pct,
};
let load = System::load_average();
let cpu = CpuStats {
percent_used: sys.global_cpu_usage() as f64,
load_avg: LoadAvg {
one: load.one,
five: load.five,
fifteen: load.fifteen,
},
per_core: sys.cpus().iter().map(|c| c.cpu_usage() as f64).collect(),
};
(memory, cpu)
}