feat(admin): operational health checks page with editable thresholds
New /admin/health rolls up signals already collected — disk/memory sensors, dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata cron freshness, and the live crawler session/browser flags — into a flat checks list (ok/warn/critical) with an overall status and remediation links. The overview gains a compact Health summary banner linking in. GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists the thresholds to app_settings (merged over compiled defaults, forward-compat via serde(default)), validates ranges (400 on bad input), and audit-logs the change in one transaction. New repo::crawler::last_metadata_tick_at getter and a lightweight system::memory_percent_used (no CPU settle delay). Pure status helpers (over_limit for gauges, exceeds for backlog counts so a 0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape, gating, persistence+audit, and validation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
444
backend/src/api/admin/health.rs
Normal file
444
backend/src/api/admin/health.rs
Normal file
@@ -0,0 +1,444 @@
|
||||
//! GET /admin/health — aggregated operational health checks
|
||||
//! PUT /admin/health/thresholds — edit the alerting thresholds
|
||||
//!
|
||||
//! A single read that rolls up signals already collected elsewhere (system
|
||||
//! sensors, queue depths, recent failure rates, cron freshness, the live
|
||||
//! crawler session/browser flags) into a flat list of checks with an
|
||||
//! overall status. Thresholds live in `app_settings['health_thresholds']`
|
||||
//! (JSONB, merged over compiled defaults) and are admin-editable + audited.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::{get, put};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{Duration, Utc};
|
||||
use futures_util::TryFutureExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::crawler::browser_manager::RestartPhase;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
const KEY: &str = "health_thresholds";
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/health", get(health))
|
||||
.route("/admin/health/thresholds", put(update_thresholds))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Thresholds (persisted, editable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn d_disk() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_mem() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_dead() -> i64 {
|
||||
100
|
||||
}
|
||||
fn d_crawl_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_analysis_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_cron() -> i64 {
|
||||
26
|
||||
}
|
||||
fn d_covers() -> i64 {
|
||||
500
|
||||
}
|
||||
|
||||
/// Alerting thresholds. Each field has a `serde(default)` so a stored row
|
||||
/// that predates a newly-added field still deserializes (the missing field
|
||||
/// takes its compiled default — forward-compatible).
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct HealthThresholds {
|
||||
#[serde(default = "d_disk")]
|
||||
pub disk_pct: f64,
|
||||
#[serde(default = "d_mem")]
|
||||
pub mem_pct: f64,
|
||||
#[serde(default = "d_dead")]
|
||||
pub dead_jobs_max: i64,
|
||||
#[serde(default = "d_crawl_fail")]
|
||||
pub crawl_fail_pct: f64,
|
||||
#[serde(default = "d_analysis_fail")]
|
||||
pub analysis_fail_pct: f64,
|
||||
#[serde(default = "d_cron")]
|
||||
pub cron_freshness_hours: i64,
|
||||
#[serde(default = "d_covers")]
|
||||
pub missing_covers_max: i64,
|
||||
}
|
||||
|
||||
impl Default for HealthThresholds {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
disk_pct: d_disk(),
|
||||
mem_pct: d_mem(),
|
||||
dead_jobs_max: d_dead(),
|
||||
crawl_fail_pct: d_crawl_fail(),
|
||||
analysis_fail_pct: d_analysis_fail(),
|
||||
cron_freshness_hours: d_cron(),
|
||||
missing_covers_max: d_covers(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthThresholds {
|
||||
/// Reject nonsensical values so a check can't be silently disabled by a
|
||||
/// fat-fingered edit. Percentages 1..=100; counts/hours must be positive.
|
||||
fn validate(&self) -> Result<(), String> {
|
||||
let pct = |v: f64, name: &str| {
|
||||
if (1.0..=100.0).contains(&v) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{name} must be between 1 and 100"))
|
||||
}
|
||||
};
|
||||
pct(self.disk_pct, "disk_pct")?;
|
||||
pct(self.mem_pct, "mem_pct")?;
|
||||
pct(self.crawl_fail_pct, "crawl_fail_pct")?;
|
||||
pct(self.analysis_fail_pct, "analysis_fail_pct")?;
|
||||
if self.dead_jobs_max < 0 {
|
||||
return Err("dead_jobs_max must be >= 0".into());
|
||||
}
|
||||
if self.missing_covers_max < 0 {
|
||||
return Err("missing_covers_max must be >= 0".into());
|
||||
}
|
||||
if self.cron_freshness_hours < 1 {
|
||||
return Err("cron_freshness_hours must be >= 1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_thresholds(state: &AppState) -> AppResult<HealthThresholds> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY).await? {
|
||||
// serde(default) on each field fills anything the stored object omits.
|
||||
Some(v) => serde_json::from_value(v).unwrap_or_default(),
|
||||
None => HealthThresholds::default(),
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CheckStatus {
|
||||
Ok,
|
||||
Warn,
|
||||
Critical,
|
||||
}
|
||||
|
||||
impl CheckStatus {
|
||||
fn rank(self) -> u8 {
|
||||
match self {
|
||||
CheckStatus::Ok => 0,
|
||||
CheckStatus::Warn => 1,
|
||||
CheckStatus::Critical => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` while `value` is strictly below `limit`, else `Warn`. Used for
|
||||
/// "near a ceiling" gauges (disk/memory/fail-rate percentages, cron-age
|
||||
/// hours) where hitting the threshold is itself the alert. Pure so the
|
||||
/// rule is unit-testable without the DB/sensors.
|
||||
pub(crate) fn over_limit(value: f64, limit: f64) -> CheckStatus {
|
||||
if value < limit {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
CheckStatus::Warn
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` until `value` strictly exceeds `limit`. Used for "max allowed"
|
||||
/// backlog counts so a limit of `0` warns on the first item — and `0` items
|
||||
/// against a `0` limit stays `Ok` (no false alarm at exactly the boundary).
|
||||
pub(crate) fn exceeds(value: i64, limit: i64) -> CheckStatus {
|
||||
if value > limit {
|
||||
CheckStatus::Warn
|
||||
} else {
|
||||
CheckStatus::Ok
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthCheck {
|
||||
pub id: &'static str,
|
||||
pub label: &'static str,
|
||||
pub status: CheckStatus,
|
||||
pub value: String,
|
||||
pub threshold: Option<String>,
|
||||
pub hint: Option<String>,
|
||||
/// Admin route that remediates this check, when one applies.
|
||||
pub action_href: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthReport {
|
||||
/// Worst status across all checks (drives the page/summary badge).
|
||||
pub status: CheckStatus,
|
||||
pub checks: Vec<HealthCheck>,
|
||||
pub thresholds: HealthThresholds,
|
||||
}
|
||||
|
||||
async fn health(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
let thresholds = load_thresholds(&state).await?;
|
||||
let since_24h = Utc::now() - Duration::hours(24);
|
||||
|
||||
// DB-derived signals run concurrently — all hit existing indexes.
|
||||
let ((pending, running, dead), missing_covers, crawl, analysis, last_tick) = tokio::try_join!(
|
||||
repo::crawler::job_state_counts(&state.db).map_err(AppError::from),
|
||||
repo::crawler::count_missing_covers(&state.db).map_err(AppError::from),
|
||||
repo::crawl_metrics::summary(&state.db, Some(since_24h)).map_err(AppError::from),
|
||||
repo::page_analysis::analysis_metrics(&state.db, Some(since_24h)),
|
||||
repo::crawler::last_metadata_tick_at(&state.db).map_err(AppError::from),
|
||||
)?;
|
||||
let _ = (pending, running); // queue depth shown elsewhere; not a check yet
|
||||
|
||||
let mut checks: Vec<HealthCheck> = Vec::new();
|
||||
|
||||
// --- system sensors ---
|
||||
if let Some(d) = state.storage.local_root().and_then(super::system::disk_stats_for) {
|
||||
checks.push(HealthCheck {
|
||||
id: "disk",
|
||||
label: "Disk usage",
|
||||
status: over_limit(d.percent_used, thresholds.disk_pct),
|
||||
value: format!("{:.0}%", d.percent_used),
|
||||
threshold: Some(format!("{:.0}%", thresholds.disk_pct)),
|
||||
hint: Some("free space on the storage volume".into()),
|
||||
action_href: None,
|
||||
});
|
||||
}
|
||||
let mem = super::system::memory_percent_used();
|
||||
checks.push(HealthCheck {
|
||||
id: "memory",
|
||||
label: "Memory usage",
|
||||
status: over_limit(mem, thresholds.mem_pct),
|
||||
value: format!("{mem:.0}%"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.mem_pct)),
|
||||
hint: None,
|
||||
action_href: None,
|
||||
});
|
||||
|
||||
// --- queue / backlog ---
|
||||
checks.push(HealthCheck {
|
||||
id: "dead_jobs",
|
||||
label: "Dead jobs",
|
||||
status: exceeds(dead, thresholds.dead_jobs_max),
|
||||
value: dead.to_string(),
|
||||
threshold: Some(thresholds.dead_jobs_max.to_string()),
|
||||
hint: Some("jobs that exhausted their retries".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
checks.push(HealthCheck {
|
||||
id: "missing_covers",
|
||||
label: "Missing covers",
|
||||
status: exceeds(missing_covers, thresholds.missing_covers_max),
|
||||
value: missing_covers.to_string(),
|
||||
threshold: Some(thresholds.missing_covers_max.to_string()),
|
||||
hint: Some("mangas still awaiting a cover download".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
// --- failure rates (last 24h) ---
|
||||
let crawl_total: i64 = crawl.iter().map(|s| s.n).sum();
|
||||
let crawl_failed: i64 = crawl.iter().map(|s| s.failed).sum();
|
||||
let crawl_pct = pct(crawl_failed, crawl_total);
|
||||
checks.push(HealthCheck {
|
||||
id: "crawl_fail_rate",
|
||||
label: "Crawl failure rate (24h)",
|
||||
status: if crawl_total == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(crawl_pct, thresholds.crawl_fail_pct)
|
||||
},
|
||||
value: format!("{crawl_pct:.0}% ({crawl_failed}/{crawl_total})"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.crawl_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
let an_pct = pct(analysis.failed, analysis.n);
|
||||
checks.push(HealthCheck {
|
||||
id: "analysis_fail_rate",
|
||||
label: "Analysis failure rate (24h)",
|
||||
status: if analysis.n == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(an_pct, thresholds.analysis_fail_pct)
|
||||
},
|
||||
value: format!("{an_pct:.0}% ({}/{})", analysis.failed, analysis.n),
|
||||
threshold: Some(format!("{:.0}%", thresholds.analysis_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/analysis"),
|
||||
});
|
||||
|
||||
// --- crawler daemon liveness (only when the daemon is configured) ---
|
||||
if let Some(c) = state.crawler().as_ref() {
|
||||
// Cron freshness: stale tick = the daily pass hasn't run on time.
|
||||
match last_tick {
|
||||
Some(t) => {
|
||||
let age_h = (Utc::now() - t).num_hours();
|
||||
checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: over_limit(age_h as f64, thresholds.cron_freshness_hours as f64),
|
||||
value: format!("{age_h}h ago"),
|
||||
threshold: Some(format!("{}h", thresholds.cron_freshness_hours)),
|
||||
hint: Some("time since the last metadata pass tick".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
None => checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: CheckStatus::Ok,
|
||||
value: "no tick yet".into(),
|
||||
threshold: None,
|
||||
hint: Some("the daemon has not completed a pass yet".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
}),
|
||||
}
|
||||
|
||||
let expired = c.session.is_expired();
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_session",
|
||||
label: "Crawler session",
|
||||
status: if expired { CheckStatus::Critical } else { CheckStatus::Ok },
|
||||
value: if expired { "expired".into() } else { "ok".into() },
|
||||
threshold: None,
|
||||
hint: expired.then(|| "chapter workers are idle until the session is refreshed".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
let healthy = matches!(c.browser_manager.phase(), RestartPhase::Healthy);
|
||||
let phase = match c.browser_manager.phase() {
|
||||
RestartPhase::Healthy => "healthy",
|
||||
RestartPhase::Draining => "draining",
|
||||
RestartPhase::Restarting => "restarting",
|
||||
};
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_browser",
|
||||
label: "Crawler browser",
|
||||
status: if healthy { CheckStatus::Ok } else { CheckStatus::Warn },
|
||||
value: phase.to_string(),
|
||||
threshold: None,
|
||||
hint: (!healthy).then(|| "Chromium is restarting or unavailable".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
|
||||
let status = checks
|
||||
.iter()
|
||||
.map(|c| c.status)
|
||||
.max_by_key(|s| s.rank())
|
||||
.unwrap_or(CheckStatus::Ok);
|
||||
|
||||
Ok(Json(HealthReport {
|
||||
status,
|
||||
checks,
|
||||
thresholds,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Percentage `numerator/denominator`, guarding divide-by-zero.
|
||||
fn pct(numerator: i64, denominator: i64) -> f64 {
|
||||
if denominator <= 0 {
|
||||
0.0
|
||||
} else {
|
||||
(numerator as f64) * 100.0 / (denominator as f64)
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_thresholds(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(body): Json<HealthThresholds>,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
body.validate().map_err(AppError::InvalidInput)?;
|
||||
let value = serde_json::to_value(body).map_err(|e| AppError::Other(e.into()))?;
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_health_thresholds",
|
||||
"settings",
|
||||
None,
|
||||
value,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
// Re-evaluate so the client immediately sees checks recomputed against
|
||||
// the new thresholds.
|
||||
health(State(state), admin).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn over_limit_is_ok_below_warn_at_or_above() {
|
||||
assert_eq!(over_limit(10.0, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(19.999, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(20.0, 20.0), CheckStatus::Warn);
|
||||
assert_eq!(over_limit(99.0, 20.0), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exceeds_is_strict_and_safe_at_zero() {
|
||||
assert_eq!(exceeds(0, 0), CheckStatus::Ok); // 0 backlog, 0 limit → no false alarm
|
||||
assert_eq!(exceeds(1, 0), CheckStatus::Warn);
|
||||
assert_eq!(exceeds(100, 100), CheckStatus::Ok);
|
||||
assert_eq!(exceeds(101, 100), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pct_guards_zero_denominator() {
|
||||
assert_eq!(pct(0, 0), 0.0);
|
||||
assert_eq!(pct(5, 0), 0.0);
|
||||
assert_eq!(pct(1, 4), 25.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worst_status_ranking() {
|
||||
assert!(CheckStatus::Critical.rank() > CheckStatus::Warn.rank());
|
||||
assert!(CheckStatus::Warn.rank() > CheckStatus::Ok.rank());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_validate_rejects_out_of_range() {
|
||||
let mut t = HealthThresholds::default();
|
||||
assert!(t.validate().is_ok());
|
||||
t.disk_pct = 0.0;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.dead_jobs_max = -1;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.cron_freshness_hours = 0;
|
||||
assert!(t.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_merge_defaults_for_missing_fields() {
|
||||
// A stored object that predates `missing_covers_max` still decodes.
|
||||
let partial = serde_json::json!({ "disk_pct": 80.0 });
|
||||
let t: HealthThresholds = serde_json::from_value(partial).unwrap();
|
||||
assert_eq!(t.disk_pct, 80.0);
|
||||
assert_eq!(t.missing_covers_max, d_covers());
|
||||
assert_eq!(t.cron_freshness_hours, d_cron());
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@
|
||||
pub mod analysis;
|
||||
pub mod audit;
|
||||
pub mod crawler;
|
||||
pub mod health;
|
||||
pub mod mangas;
|
||||
pub mod overview;
|
||||
pub mod resync;
|
||||
@@ -28,6 +29,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(system::routes())
|
||||
.merge(overview::routes())
|
||||
.merge(audit::routes())
|
||||
.merge(health::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
|
||||
@@ -169,6 +169,21 @@ fn temperatures() -> Vec<TempStat> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lightweight memory-utilization percentage for the health check. Unlike
|
||||
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
|
||||
/// health is polled more often and only needs the memory figure.
|
||||
pub(crate) fn memory_percent_used() -> f64 {
|
||||
let mut sys =
|
||||
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
|
||||
sys.refresh_memory();
|
||||
let total = sys.total_memory();
|
||||
if total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(sys.used_memory() as f64) * 100.0 / (total as f64)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
||||
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
||||
|
||||
Reference in New Issue
Block a user