New /admin/health rolls up signals already collected — disk/memory sensors, dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata cron freshness, and the live crawler session/browser flags — into a flat checks list (ok/warn/critical) with an overall status and remediation links. The overview gains a compact Health summary banner linking in. GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists the thresholds to app_settings (merged over compiled defaults, forward-compat via serde(default)), validates ranges (400 on bad input), and audit-logs the change in one transaction. New repo::crawler::last_metadata_tick_at getter and a lightweight system::memory_percent_used (no CPU settle delay). Pure status helpers (over_limit for gauges, exceeds for backlog counts so a 0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape, gating, persistence+audit, and validation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
250 lines
7.5 KiB
Rust
250 lines
7.5 KiB
Rust
//! System metrics for the admin dashboard.
|
|
//!
|
|
//! Disk is `statvfs(storage_dir)` so the number reflects the volume the
|
|
//! app actually writes to (not the root filesystem of the host). When the
|
|
//! storage backend doesn't expose a local path (e.g. a future S3 impl)
|
|
//! the disk fields are `null` rather than fabricated.
|
|
//!
|
|
//! Memory and CPU come from `sysinfo`. CPU requires two refreshes with
|
|
//! at least 200ms between them to compute a meaningful delta; the
|
|
//! handler eats the 250ms wall-clock cost on each request. Admin
|
|
//! traffic is low-volume so a background cache isn't worth the moving
|
|
//! parts yet — revisit if polling becomes frequent.
|
|
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
use axum::extract::State;
|
|
use axum::routing::get;
|
|
use axum::{Json, Router};
|
|
use serde::Serialize;
|
|
use sysinfo::{Components, CpuRefreshKind, MemoryRefreshKind, RefreshKind, System};
|
|
|
|
use crate::app::AppState;
|
|
use crate::auth::extractor::RequireAdmin;
|
|
use crate::error::AppResult;
|
|
|
|
const ALERT_THRESHOLD_PERCENT: f64 = 90.0;
|
|
|
|
pub fn routes() -> Router<AppState> {
|
|
Router::new().route("/admin/system", get(system))
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct SystemStats {
|
|
pub disk: Option<DiskStats>,
|
|
pub memory: MemoryStats,
|
|
pub cpu: CpuStats,
|
|
/// Hardware temperature sensors. Empty when the platform doesn't
|
|
/// expose any (common inside containers without `/sys` access) —
|
|
/// the frontend renders an "unavailable" state, not an error.
|
|
pub temperatures: Vec<TempStat>,
|
|
pub alerts: Vec<Alert>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct DiskStats {
|
|
pub total_bytes: u64,
|
|
pub used_bytes: u64,
|
|
pub free_bytes: u64,
|
|
pub percent_used: f64,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct MemoryStats {
|
|
pub total_bytes: u64,
|
|
pub used_bytes: u64,
|
|
pub percent_used: f64,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct CpuStats {
|
|
pub percent_used: f64,
|
|
pub load_avg: LoadAvg,
|
|
/// Per-core usage percentages, one entry per logical core.
|
|
pub per_core: Vec<f64>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct LoadAvg {
|
|
pub one: f64,
|
|
pub five: f64,
|
|
pub fifteen: f64,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct TempStat {
|
|
pub label: String,
|
|
pub celsius: f64,
|
|
/// Highest reading seen since boot, when the kernel exposes it.
|
|
pub max_celsius: Option<f64>,
|
|
/// Vendor critical/shutdown threshold, when available.
|
|
pub critical_celsius: Option<f64>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub struct Alert {
|
|
pub level: AlertLevel,
|
|
pub message: String,
|
|
}
|
|
|
|
#[derive(Debug, Serialize, Clone, Copy)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum AlertLevel {
|
|
Warning,
|
|
}
|
|
|
|
async fn system(
|
|
State(state): State<AppState>,
|
|
_admin: RequireAdmin,
|
|
) -> AppResult<Json<SystemStats>> {
|
|
let disk = state.storage.local_root().and_then(disk_stats_for);
|
|
let (memory, cpu) = memory_and_cpu().await;
|
|
let temperatures = temperatures();
|
|
let mut alerts = Vec::new();
|
|
if let Some(d) = &disk {
|
|
if d.percent_used >= ALERT_THRESHOLD_PERCENT {
|
|
alerts.push(Alert {
|
|
level: AlertLevel::Warning,
|
|
message: format!(
|
|
"disk near full ({:.0}% used)",
|
|
d.percent_used
|
|
),
|
|
});
|
|
}
|
|
}
|
|
if memory.percent_used >= ALERT_THRESHOLD_PERCENT {
|
|
alerts.push(Alert {
|
|
level: AlertLevel::Warning,
|
|
message: format!(
|
|
"memory near full ({:.0}% used)",
|
|
memory.percent_used
|
|
),
|
|
});
|
|
}
|
|
for t in &temperatures {
|
|
if let Some(crit) = t.critical_celsius {
|
|
if t.celsius >= crit {
|
|
alerts.push(Alert {
|
|
level: AlertLevel::Warning,
|
|
message: format!(
|
|
"{} temperature critical ({:.0}°C / {:.0}°C)",
|
|
t.label, t.celsius, crit
|
|
),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
Ok(Json(SystemStats {
|
|
disk,
|
|
memory,
|
|
cpu,
|
|
temperatures,
|
|
alerts,
|
|
}))
|
|
}
|
|
|
|
/// Hardware temperature sensors via `sysinfo`'s `Components`. Components
|
|
/// whose temperature can't be read report `f32::NAN` on Linux — we skip
|
|
/// those so the dashboard never shows a bogus "NaN°C" row. Returns an
|
|
/// empty Vec when no sensors are exposed at all.
|
|
fn temperatures() -> Vec<TempStat> {
|
|
let components = Components::new_with_refreshed_list();
|
|
components
|
|
.list()
|
|
.iter()
|
|
.filter_map(|c| {
|
|
let celsius = c.temperature();
|
|
if celsius.is_nan() {
|
|
return None;
|
|
}
|
|
let max = c.max();
|
|
Some(TempStat {
|
|
label: c.label().to_string(),
|
|
celsius: celsius as f64,
|
|
max_celsius: (!max.is_nan()).then_some(max as f64),
|
|
critical_celsius: c.critical().map(|v| v as f64),
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Lightweight memory-utilization percentage for the health check. Unlike
|
|
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
|
|
/// health is polled more often and only needs the memory figure.
|
|
pub(crate) fn memory_percent_used() -> f64 {
|
|
let mut sys =
|
|
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
|
|
sys.refresh_memory();
|
|
let total = sys.total_memory();
|
|
if total == 0 {
|
|
0.0
|
|
} else {
|
|
(sys.used_memory() as f64) * 100.0 / (total as f64)
|
|
}
|
|
}
|
|
|
|
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
|
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
|
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
|
// is "free to non-root callers" which is what an operator actually
|
|
// cares about — `f_bfree` includes blocks reserved for root.
|
|
let block = s.fragment_size();
|
|
let total = block * s.blocks();
|
|
let avail = block * s.blocks_available();
|
|
let used = total.saturating_sub(avail);
|
|
let percent_used = if total > 0 {
|
|
(used as f64) * 100.0 / (total as f64)
|
|
} else {
|
|
0.0
|
|
};
|
|
Some(DiskStats {
|
|
total_bytes: total,
|
|
used_bytes: used,
|
|
free_bytes: avail,
|
|
percent_used,
|
|
})
|
|
}
|
|
|
|
async fn memory_and_cpu() -> (MemoryStats, CpuStats) {
|
|
// sysinfo's CPU sampling needs two refreshes with a delay between
|
|
// them — the first seeds the delta counters, the second measures.
|
|
// We do this once per request; admin traffic is low enough that the
|
|
// 250ms cost is invisible.
|
|
let mut sys = System::new_with_specifics(
|
|
RefreshKind::new()
|
|
.with_cpu(CpuRefreshKind::everything())
|
|
.with_memory(MemoryRefreshKind::everything()),
|
|
);
|
|
sys.refresh_cpu_all();
|
|
// Yield the runtime instead of blocking it for the gap.
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
sys.refresh_cpu_all();
|
|
sys.refresh_memory();
|
|
|
|
let total = sys.total_memory();
|
|
let used = sys.used_memory();
|
|
let mem_pct = if total > 0 {
|
|
(used as f64) * 100.0 / (total as f64)
|
|
} else {
|
|
0.0
|
|
};
|
|
let memory = MemoryStats {
|
|
total_bytes: total,
|
|
used_bytes: used,
|
|
percent_used: mem_pct,
|
|
};
|
|
|
|
let load = System::load_average();
|
|
let cpu = CpuStats {
|
|
percent_used: sys.global_cpu_usage() as f64,
|
|
load_avg: LoadAvg {
|
|
one: load.one,
|
|
five: load.five,
|
|
fifteen: load.fifteen,
|
|
},
|
|
per_core: sys.cpus().iter().map(|c| c.cpu_usage() as f64).collect(),
|
|
};
|
|
(memory, cpu)
|
|
}
|