From 314fc8738b3ea372fa53b407451bf871d355d29a Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Fri, 19 Jun 2026 11:38:18 +0200 Subject: [PATCH] feat(admin): operational health checks page with editable thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New /admin/health rolls up signals already collected — disk/memory sensors, dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata cron freshness, and the live crawler session/browser flags — into a flat checks list (ok/warn/critical) with an overall status and remediation links. The overview gains a compact Health summary banner linking in. GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists the thresholds to app_settings (merged over compiled defaults, forward-compat via serde(default)), validates ranges (400 on bad input), and audit-logs the change in one transaction. New repo::crawler::last_metadata_tick_at getter and a lightweight system::memory_percent_used (no CPU settle delay). Pure status helpers (over_limit for gauges, exceeds for backlog counts so a 0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape, gating, persistence+audit, and validation. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/src/api/admin/health.rs | 444 ++++++++++++++++++ backend/src/api/admin/mod.rs | 2 + backend/src/api/admin/system.rs | 15 + backend/src/repo/crawler.rs | 19 + backend/tests/api_admin_health.rs | 207 ++++++++ frontend/src/lib/api/admin.test.ts | 46 +- frontend/src/lib/api/admin.ts | 43 ++ .../lib/components/admin/HealthChecks.svelte | 306 ++++++++++++ .../admin/HealthChecks.svelte.test.ts | 81 ++++ frontend/src/routes/admin/+layout.svelte | 1 + frontend/src/routes/admin/+page.svelte | 77 ++- frontend/src/routes/admin/health/+page.svelte | 26 + 12 files changed, 1265 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/admin/health.rs create mode 100644 backend/tests/api_admin_health.rs create mode 100644 frontend/src/lib/components/admin/HealthChecks.svelte create mode 100644 frontend/src/lib/components/admin/HealthChecks.svelte.test.ts create mode 100644 frontend/src/routes/admin/health/+page.svelte diff --git a/backend/src/api/admin/health.rs b/backend/src/api/admin/health.rs new file mode 100644 index 0000000..44d1431 --- /dev/null +++ b/backend/src/api/admin/health.rs @@ -0,0 +1,444 @@ +//! GET /admin/health — aggregated operational health checks +//! PUT /admin/health/thresholds — edit the alerting thresholds +//! +//! A single read that rolls up signals already collected elsewhere (system +//! sensors, queue depths, recent failure rates, cron freshness, the live +//! crawler session/browser flags) into a flat list of checks with an +//! overall status. Thresholds live in `app_settings['health_thresholds']` +//! (JSONB, merged over compiled defaults) and are admin-editable + audited. + +use axum::extract::State; +use axum::routing::{get, put}; +use axum::{Json, Router}; +use chrono::{Duration, Utc}; +use futures_util::TryFutureExt; +use serde::{Deserialize, Serialize}; + +use crate::app::AppState; +use crate::auth::extractor::RequireAdmin; +use crate::crawler::browser_manager::RestartPhase; +use crate::error::{AppError, AppResult}; +use crate::repo; + +const KEY: &str = "health_thresholds"; + +pub fn routes() -> Router { + Router::new() + .route("/admin/health", get(health)) + .route("/admin/health/thresholds", put(update_thresholds)) +} + +// --------------------------------------------------------------------------- +// Thresholds (persisted, editable) +// --------------------------------------------------------------------------- + +fn d_disk() -> f64 { + 90.0 +} +fn d_mem() -> f64 { + 90.0 +} +fn d_dead() -> i64 { + 100 +} +fn d_crawl_fail() -> f64 { + 20.0 +} +fn d_analysis_fail() -> f64 { + 20.0 +} +fn d_cron() -> i64 { + 26 +} +fn d_covers() -> i64 { + 500 +} + +/// Alerting thresholds. Each field has a `serde(default)` so a stored row +/// that predates a newly-added field still deserializes (the missing field +/// takes its compiled default — forward-compatible). +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct HealthThresholds { + #[serde(default = "d_disk")] + pub disk_pct: f64, + #[serde(default = "d_mem")] + pub mem_pct: f64, + #[serde(default = "d_dead")] + pub dead_jobs_max: i64, + #[serde(default = "d_crawl_fail")] + pub crawl_fail_pct: f64, + #[serde(default = "d_analysis_fail")] + pub analysis_fail_pct: f64, + #[serde(default = "d_cron")] + pub cron_freshness_hours: i64, + #[serde(default = "d_covers")] + pub missing_covers_max: i64, +} + +impl Default for HealthThresholds { + fn default() -> Self { + Self { + disk_pct: d_disk(), + mem_pct: d_mem(), + dead_jobs_max: d_dead(), + crawl_fail_pct: d_crawl_fail(), + analysis_fail_pct: d_analysis_fail(), + cron_freshness_hours: d_cron(), + missing_covers_max: d_covers(), + } + } +} + +impl HealthThresholds { + /// Reject nonsensical values so a check can't be silently disabled by a + /// fat-fingered edit. Percentages 1..=100; counts/hours must be positive. + fn validate(&self) -> Result<(), String> { + let pct = |v: f64, name: &str| { + if (1.0..=100.0).contains(&v) { + Ok(()) + } else { + Err(format!("{name} must be between 1 and 100")) + } + }; + pct(self.disk_pct, "disk_pct")?; + pct(self.mem_pct, "mem_pct")?; + pct(self.crawl_fail_pct, "crawl_fail_pct")?; + pct(self.analysis_fail_pct, "analysis_fail_pct")?; + if self.dead_jobs_max < 0 { + return Err("dead_jobs_max must be >= 0".into()); + } + if self.missing_covers_max < 0 { + return Err("missing_covers_max must be >= 0".into()); + } + if self.cron_freshness_hours < 1 { + return Err("cron_freshness_hours must be >= 1".into()); + } + Ok(()) + } +} + +async fn load_thresholds(state: &AppState) -> AppResult { + Ok(match repo::app_settings::get(&state.db, KEY).await? { + // serde(default) on each field fills anything the stored object omits. + Some(v) => serde_json::from_value(v).unwrap_or_default(), + None => HealthThresholds::default(), + }) +} + +// --------------------------------------------------------------------------- +// Checks +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CheckStatus { + Ok, + Warn, + Critical, +} + +impl CheckStatus { + fn rank(self) -> u8 { + match self { + CheckStatus::Ok => 0, + CheckStatus::Warn => 1, + CheckStatus::Critical => 2, + } + } +} + +/// `Ok` while `value` is strictly below `limit`, else `Warn`. Used for +/// "near a ceiling" gauges (disk/memory/fail-rate percentages, cron-age +/// hours) where hitting the threshold is itself the alert. Pure so the +/// rule is unit-testable without the DB/sensors. +pub(crate) fn over_limit(value: f64, limit: f64) -> CheckStatus { + if value < limit { + CheckStatus::Ok + } else { + CheckStatus::Warn + } +} + +/// `Ok` until `value` strictly exceeds `limit`. Used for "max allowed" +/// backlog counts so a limit of `0` warns on the first item — and `0` items +/// against a `0` limit stays `Ok` (no false alarm at exactly the boundary). +pub(crate) fn exceeds(value: i64, limit: i64) -> CheckStatus { + if value > limit { + CheckStatus::Warn + } else { + CheckStatus::Ok + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct HealthCheck { + pub id: &'static str, + pub label: &'static str, + pub status: CheckStatus, + pub value: String, + pub threshold: Option, + pub hint: Option, + /// Admin route that remediates this check, when one applies. + pub action_href: Option<&'static str>, +} + +#[derive(Debug, Clone, Serialize)] +pub struct HealthReport { + /// Worst status across all checks (drives the page/summary badge). + pub status: CheckStatus, + pub checks: Vec, + pub thresholds: HealthThresholds, +} + +async fn health( + State(state): State, + _admin: RequireAdmin, +) -> AppResult> { + let thresholds = load_thresholds(&state).await?; + let since_24h = Utc::now() - Duration::hours(24); + + // DB-derived signals run concurrently — all hit existing indexes. + let ((pending, running, dead), missing_covers, crawl, analysis, last_tick) = tokio::try_join!( + repo::crawler::job_state_counts(&state.db).map_err(AppError::from), + repo::crawler::count_missing_covers(&state.db).map_err(AppError::from), + repo::crawl_metrics::summary(&state.db, Some(since_24h)).map_err(AppError::from), + repo::page_analysis::analysis_metrics(&state.db, Some(since_24h)), + repo::crawler::last_metadata_tick_at(&state.db).map_err(AppError::from), + )?; + let _ = (pending, running); // queue depth shown elsewhere; not a check yet + + let mut checks: Vec = Vec::new(); + + // --- system sensors --- + if let Some(d) = state.storage.local_root().and_then(super::system::disk_stats_for) { + checks.push(HealthCheck { + id: "disk", + label: "Disk usage", + status: over_limit(d.percent_used, thresholds.disk_pct), + value: format!("{:.0}%", d.percent_used), + threshold: Some(format!("{:.0}%", thresholds.disk_pct)), + hint: Some("free space on the storage volume".into()), + action_href: None, + }); + } + let mem = super::system::memory_percent_used(); + checks.push(HealthCheck { + id: "memory", + label: "Memory usage", + status: over_limit(mem, thresholds.mem_pct), + value: format!("{mem:.0}%"), + threshold: Some(format!("{:.0}%", thresholds.mem_pct)), + hint: None, + action_href: None, + }); + + // --- queue / backlog --- + checks.push(HealthCheck { + id: "dead_jobs", + label: "Dead jobs", + status: exceeds(dead, thresholds.dead_jobs_max), + value: dead.to_string(), + threshold: Some(thresholds.dead_jobs_max.to_string()), + hint: Some("jobs that exhausted their retries".into()), + action_href: Some("/admin/crawler"), + }); + checks.push(HealthCheck { + id: "missing_covers", + label: "Missing covers", + status: exceeds(missing_covers, thresholds.missing_covers_max), + value: missing_covers.to_string(), + threshold: Some(thresholds.missing_covers_max.to_string()), + hint: Some("mangas still awaiting a cover download".into()), + action_href: Some("/admin/crawler"), + }); + + // --- failure rates (last 24h) --- + let crawl_total: i64 = crawl.iter().map(|s| s.n).sum(); + let crawl_failed: i64 = crawl.iter().map(|s| s.failed).sum(); + let crawl_pct = pct(crawl_failed, crawl_total); + checks.push(HealthCheck { + id: "crawl_fail_rate", + label: "Crawl failure rate (24h)", + status: if crawl_total == 0 { + CheckStatus::Ok + } else { + over_limit(crawl_pct, thresholds.crawl_fail_pct) + }, + value: format!("{crawl_pct:.0}% ({crawl_failed}/{crawl_total})"), + threshold: Some(format!("{:.0}%", thresholds.crawl_fail_pct)), + hint: None, + action_href: Some("/admin/crawler"), + }); + let an_pct = pct(analysis.failed, analysis.n); + checks.push(HealthCheck { + id: "analysis_fail_rate", + label: "Analysis failure rate (24h)", + status: if analysis.n == 0 { + CheckStatus::Ok + } else { + over_limit(an_pct, thresholds.analysis_fail_pct) + }, + value: format!("{an_pct:.0}% ({}/{})", analysis.failed, analysis.n), + threshold: Some(format!("{:.0}%", thresholds.analysis_fail_pct)), + hint: None, + action_href: Some("/admin/analysis"), + }); + + // --- crawler daemon liveness (only when the daemon is configured) --- + if let Some(c) = state.crawler().as_ref() { + // Cron freshness: stale tick = the daily pass hasn't run on time. + match last_tick { + Some(t) => { + let age_h = (Utc::now() - t).num_hours(); + checks.push(HealthCheck { + id: "cron_freshness", + label: "Metadata cron freshness", + status: over_limit(age_h as f64, thresholds.cron_freshness_hours as f64), + value: format!("{age_h}h ago"), + threshold: Some(format!("{}h", thresholds.cron_freshness_hours)), + hint: Some("time since the last metadata pass tick".into()), + action_href: Some("/admin/crawler"), + }); + } + None => checks.push(HealthCheck { + id: "cron_freshness", + label: "Metadata cron freshness", + status: CheckStatus::Ok, + value: "no tick yet".into(), + threshold: None, + hint: Some("the daemon has not completed a pass yet".into()), + action_href: Some("/admin/crawler"), + }), + } + + let expired = c.session.is_expired(); + checks.push(HealthCheck { + id: "crawler_session", + label: "Crawler session", + status: if expired { CheckStatus::Critical } else { CheckStatus::Ok }, + value: if expired { "expired".into() } else { "ok".into() }, + threshold: None, + hint: expired.then(|| "chapter workers are idle until the session is refreshed".into()), + action_href: Some("/admin/crawler"), + }); + + let healthy = matches!(c.browser_manager.phase(), RestartPhase::Healthy); + let phase = match c.browser_manager.phase() { + RestartPhase::Healthy => "healthy", + RestartPhase::Draining => "draining", + RestartPhase::Restarting => "restarting", + }; + checks.push(HealthCheck { + id: "crawler_browser", + label: "Crawler browser", + status: if healthy { CheckStatus::Ok } else { CheckStatus::Warn }, + value: phase.to_string(), + threshold: None, + hint: (!healthy).then(|| "Chromium is restarting or unavailable".into()), + action_href: Some("/admin/crawler"), + }); + } + + let status = checks + .iter() + .map(|c| c.status) + .max_by_key(|s| s.rank()) + .unwrap_or(CheckStatus::Ok); + + Ok(Json(HealthReport { + status, + checks, + thresholds, + })) +} + +/// Percentage `numerator/denominator`, guarding divide-by-zero. +fn pct(numerator: i64, denominator: i64) -> f64 { + if denominator <= 0 { + 0.0 + } else { + (numerator as f64) * 100.0 / (denominator as f64) + } +} + +async fn update_thresholds( + State(state): State, + admin: RequireAdmin, + Json(body): Json, +) -> AppResult> { + body.validate().map_err(AppError::InvalidInput)?; + let value = serde_json::to_value(body).map_err(|e| AppError::Other(e.into()))?; + let mut tx = state.db.begin().await?; + repo::app_settings::upsert(&mut *tx, KEY, &value).await?; + repo::admin_audit::insert( + &mut *tx, + admin.0.id, + "update_health_thresholds", + "settings", + None, + value, + ) + .await?; + tx.commit().await?; + // Re-evaluate so the client immediately sees checks recomputed against + // the new thresholds. + health(State(state), admin).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn over_limit_is_ok_below_warn_at_or_above() { + assert_eq!(over_limit(10.0, 20.0), CheckStatus::Ok); + assert_eq!(over_limit(19.999, 20.0), CheckStatus::Ok); + assert_eq!(over_limit(20.0, 20.0), CheckStatus::Warn); + assert_eq!(over_limit(99.0, 20.0), CheckStatus::Warn); + } + + #[test] + fn exceeds_is_strict_and_safe_at_zero() { + assert_eq!(exceeds(0, 0), CheckStatus::Ok); // 0 backlog, 0 limit → no false alarm + assert_eq!(exceeds(1, 0), CheckStatus::Warn); + assert_eq!(exceeds(100, 100), CheckStatus::Ok); + assert_eq!(exceeds(101, 100), CheckStatus::Warn); + } + + #[test] + fn pct_guards_zero_denominator() { + assert_eq!(pct(0, 0), 0.0); + assert_eq!(pct(5, 0), 0.0); + assert_eq!(pct(1, 4), 25.0); + } + + #[test] + fn worst_status_ranking() { + assert!(CheckStatus::Critical.rank() > CheckStatus::Warn.rank()); + assert!(CheckStatus::Warn.rank() > CheckStatus::Ok.rank()); + } + + #[test] + fn thresholds_validate_rejects_out_of_range() { + let mut t = HealthThresholds::default(); + assert!(t.validate().is_ok()); + t.disk_pct = 0.0; + assert!(t.validate().is_err()); + t = HealthThresholds::default(); + t.dead_jobs_max = -1; + assert!(t.validate().is_err()); + t = HealthThresholds::default(); + t.cron_freshness_hours = 0; + assert!(t.validate().is_err()); + } + + #[test] + fn thresholds_merge_defaults_for_missing_fields() { + // A stored object that predates `missing_covers_max` still decodes. + let partial = serde_json::json!({ "disk_pct": 80.0 }); + let t: HealthThresholds = serde_json::from_value(partial).unwrap(); + assert_eq!(t.disk_pct, 80.0); + assert_eq!(t.missing_covers_max, d_covers()); + assert_eq!(t.cron_freshness_hours, d_cron()); + } +} diff --git a/backend/src/api/admin/mod.rs b/backend/src/api/admin/mod.rs index 34cea80..ffeaa97 100644 --- a/backend/src/api/admin/mod.rs +++ b/backend/src/api/admin/mod.rs @@ -7,6 +7,7 @@ pub mod analysis; pub mod audit; pub mod crawler; +pub mod health; pub mod mangas; pub mod overview; pub mod resync; @@ -28,6 +29,7 @@ pub fn routes() -> Router { .merge(system::routes()) .merge(overview::routes()) .merge(audit::routes()) + .merge(health::routes()) .merge(crawler::routes()) .merge(analysis::routes()) .merge(settings::routes()) diff --git a/backend/src/api/admin/system.rs b/backend/src/api/admin/system.rs index f90c30b..ce88462 100644 --- a/backend/src/api/admin/system.rs +++ b/backend/src/api/admin/system.rs @@ -169,6 +169,21 @@ fn temperatures() -> Vec { .collect() } +/// Lightweight memory-utilization percentage for the health check. Unlike +/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay — +/// health is polled more often and only needs the memory figure. +pub(crate) fn memory_percent_used() -> f64 { + let mut sys = + System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything())); + sys.refresh_memory(); + let total = sys.total_memory(); + if total == 0 { + 0.0 + } else { + (sys.used_memory() as f64) * 100.0 / (total as f64) + } +} + pub(crate) fn disk_stats_for(root: &Path) -> Option { let s = nix::sys::statvfs::statvfs(root).ok()?; // statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail` diff --git a/backend/src/repo/crawler.rs b/backend/src/repo/crawler.rs index 5fb7261..4eaf3b8 100644 --- a/backend/src/repo/crawler.rs +++ b/backend/src/repo/crawler.rs @@ -619,6 +619,25 @@ pub async fn last_run_completed_cleanly( .unwrap_or(true)) } +/// Timestamp of the most recent metadata-pass cron tick, or `None` if the +/// daemon has never ticked (fresh DB / cron disabled). Stored by the daemon +/// under `crawler_state['last_metadata_tick_at']` as `{"at": }`. +/// Used by the health check that flags a stale/stuck cron. +pub async fn last_metadata_tick_at( + pool: &PgPool, +) -> sqlx::Result>> { + let row: Option = + sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = 'last_metadata_tick_at'") + .fetch_optional(pool) + .await?; + Ok(row.and_then(|v| { + v.get("at") + .and_then(|s| s.as_str()) + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|dt| dt.with_timezone(&Utc)) + })) +} + // --------------------------------------------------------------------------- // Reconcile: find source-list mangas missing from the DB. // --------------------------------------------------------------------------- diff --git a/backend/tests/api_admin_health.rs b/backend/tests/api_admin_health.rs new file mode 100644 index 0000000..6557857 --- /dev/null +++ b/backend/tests/api_admin_health.rs @@ -0,0 +1,207 @@ +//! Integration tests for the admin health endpoints: `GET /v1/admin/health` +//! (shape, admin-gating, default checks present without a daemon) and +//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row). + +mod common; + +use axum::http::StatusCode; +use axum::Router; +use serde_json::json; +use sqlx::PgPool; +use tower::ServiceExt; + +use common::{body_json, get_with_cookie, harness, register_user}; + +async fn seed_admin(pool: &PgPool, app: &Router) -> String { + let (username, cookie) = register_user(app).await; + let u = mangalord::repo::user::find_by_username(pool, &username) + .await + .unwrap() + .unwrap(); + mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true) + .await + .unwrap(); + cookie +} + +/// PUT helper (the common module exposes POST helpers; build a PUT here). +fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request { + axum::http::Request::builder() + .method("PUT") + .uri(uri) + .header(axum::http::header::COOKIE, cookie) + .header(axum::http::header::CONTENT_TYPE, "application/json") + .body(axum::body::Body::from(body.to_string())) + .unwrap() +} + +fn valid_thresholds() -> serde_json::Value { + json!({ + "disk_pct": 85.0, + "mem_pct": 80.0, + "dead_jobs_max": 50, + "crawl_fail_pct": 25.0, + "analysis_fail_pct": 30.0, + "cron_freshness_hours": 12, + "missing_covers_max": 200 + }) +} + +#[sqlx::test(migrations = "./migrations")] +async fn requires_admin(pool: PgPool) { + let h = harness(pool.clone()); + let (_u, cookie) = register_user(&h.app).await; + let resp = h + .app + .oneshot(get_with_cookie("/api/v1/admin/health", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrations = "./migrations")] +async fn report_has_status_checks_and_default_thresholds(pool: PgPool) { + let h = harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let resp = h + .app + .oneshot(get_with_cookie("/api/v1/admin/health", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = body_json(resp).await; + // Overall status + a non-empty checks array (memory / dead_jobs / + // missing_covers / fail-rates are always present, daemon or not). + assert!(body["status"].is_string()); + let checks = body["checks"].as_array().unwrap(); + assert!(!checks.is_empty()); + let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect(); + assert!(ids.contains(&"memory")); + assert!(ids.contains(&"dead_jobs")); + assert!(ids.contains(&"crawl_fail_rate")); + // The harness wires no crawler daemon, so session/browser checks are absent. + assert!(!ids.contains(&"crawler_session")); + // Default thresholds echoed back. + assert_eq!(body["thresholds"]["dead_jobs_max"], 100); + assert_eq!(body["thresholds"]["disk_pct"], 90.0); +} + +#[sqlx::test(migrations = "./migrations")] +async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) { + let h = harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + // Lower the threshold to 0, then a single dead job trips the check. + let mut t = valid_thresholds(); + t["dead_jobs_max"] = json!(0); + let resp = h + .app + .clone() + .oneshot(put_json_with_cookie( + "/api/v1/admin/health/thresholds", + t, + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + sqlx::query( + "INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')", + ) + .bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" })) + .execute(&pool) + .await + .unwrap(); + + let resp = h + .app + .clone() + .oneshot(get_with_cookie("/api/v1/admin/health", &cookie)) + .await + .unwrap(); + let body = body_json(resp).await; + let dead = body["checks"] + .as_array() + .unwrap() + .iter() + .find(|c| c["id"] == "dead_jobs") + .unwrap(); + assert_eq!(dead["status"], "warn"); + assert_eq!(body["status"], "warn"); // overall rolls up the worst +} + +#[sqlx::test(migrations = "./migrations")] +async fn update_thresholds_persists_and_audits(pool: PgPool) { + let h = harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let resp = h + .app + .clone() + .oneshot(put_json_with_cookie( + "/api/v1/admin/health/thresholds", + valid_thresholds(), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = body_json(resp).await; + // Response re-evaluates against the new thresholds. + assert_eq!(body["thresholds"]["dead_jobs_max"], 50); + + // Persisted to app_settings. + let stored: serde_json::Value = + sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(stored["mem_pct"], 80.0); + + // Audit row written. + let action: String = + sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(action, "update_health_thresholds"); +} + +#[sqlx::test(migrations = "./migrations")] +async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) { + // Absent → None. + assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool) + .await + .unwrap() + .is_none()); + // Stored in the daemon's `{"at": rfc3339}` shape → parsed back. + sqlx::query( + "INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)", + ) + .bind(json!({ "at": "2026-06-18T09:30:00Z" })) + .execute(&pool) + .await + .unwrap(); + let got = mangalord::repo::crawler::last_metadata_tick_at(&pool) + .await + .unwrap() + .unwrap(); + assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00"); +} + +#[sqlx::test(migrations = "./migrations")] +async fn update_thresholds_rejects_out_of_range(pool: PgPool) { + let h = harness(pool.clone()); + let cookie = seed_admin(&pool, &h.app).await; + let mut bad = valid_thresholds(); + bad["disk_pct"] = json!(150.0); // > 100 + let resp = h + .app + .oneshot(put_json_with_cookie( + "/api/v1/admin/health/thresholds", + bad, + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} diff --git a/frontend/src/lib/api/admin.test.ts b/frontend/src/lib/api/admin.test.ts index 7ac9298..42b5e6e 100644 --- a/frontend/src/lib/api/admin.test.ts +++ b/frontend/src/lib/api/admin.test.ts @@ -43,7 +43,9 @@ import { getCrawlerMetrics, listCrawlerOps, getAnalysisMetrics, - listAuditLog + listAuditLog, + getHealth, + updateHealthThresholds } from './admin'; function ok(body: unknown, status = 200): Response { @@ -506,6 +508,48 @@ describe('admin crawler api client', () => { expect(url).toContain('days=7'); }); + const healthFixture = { + status: 'ok' as const, + checks: [ + { + id: 'memory', + label: 'Memory usage', + status: 'ok' as const, + value: '48%', + threshold: '90%', + hint: null, + action_href: null + } + ], + thresholds: { + disk_pct: 90, + mem_pct: 90, + dead_jobs_max: 100, + crawl_fail_pct: 20, + analysis_fail_pct: 20, + cron_freshness_hours: 26, + missing_covers_max: 500 + } + }; + + it('getHealth GETs /v1/admin/health', async () => { + fetchSpy.mockResolvedValueOnce(ok(healthFixture)); + const r = await getHealth(); + expect(r.status).toBe('ok'); + expect(r.checks[0].id).toBe('memory'); + expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/health$/); + }); + + it('updateHealthThresholds PUTs the thresholds body', async () => { + fetchSpy.mockResolvedValueOnce(ok(healthFixture)); + await updateHealthThresholds(healthFixture.thresholds); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toMatch(/\/v1\/admin\/health\/thresholds$/); + const init = fetchSpy.mock.calls[0][1] as RequestInit; + expect(init.method).toBe('PUT'); + expect(JSON.parse(init.body as string).dead_jobs_max).toBe(100); + }); + it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => { fetchSpy.mockResolvedValueOnce(ok({ started: true })); const r = await runCrawlerPass(); diff --git a/frontend/src/lib/api/admin.ts b/frontend/src/lib/api/admin.ts index 78e62ce..8d1d775 100644 --- a/frontend/src/lib/api/admin.ts +++ b/frontend/src/lib/api/admin.ts @@ -1018,3 +1018,46 @@ export async function listAuditLog( const qs = params.toString(); return request(`/v1/admin/audit${qs ? `?${qs}` : ''}`, init); } + +// ---- operational health checks --------------------------------------------- + +export type CheckStatus = 'ok' | 'warn' | 'critical'; + +export type HealthCheck = { + id: string; + label: string; + status: CheckStatus; + value: string; + threshold: string | null; + hint: string | null; + /** Admin route that remediates this check, when one applies. */ + action_href: string | null; +}; + +export type HealthThresholds = { + disk_pct: number; + mem_pct: number; + dead_jobs_max: number; + crawl_fail_pct: number; + analysis_fail_pct: number; + cron_freshness_hours: number; + missing_covers_max: number; +}; + +export type HealthReport = { + status: CheckStatus; + checks: HealthCheck[]; + thresholds: HealthThresholds; +}; + +export async function getHealth(init?: RequestInit): Promise { + return request('/v1/admin/health', init); +} + +export async function updateHealthThresholds(t: HealthThresholds): Promise { + return request('/v1/admin/health/thresholds', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(t) + }); +} diff --git a/frontend/src/lib/components/admin/HealthChecks.svelte b/frontend/src/lib/components/admin/HealthChecks.svelte new file mode 100644 index 0000000..3158644 --- /dev/null +++ b/frontend/src/lib/components/admin/HealthChecks.svelte @@ -0,0 +1,306 @@ + + +
+ {#if error} + + {/if} + + {#if loading && !report} +

Loading…

+ {:else if report} +
+ {STATUS_GLYPH[report.status]} + {STATUS_LABEL[report.status]} + {#if failing > 0} + · {failing} check{failing === 1 ? '' : 's'} need attention + {/if} +
+ +
    + {#each report.checks as c (c.id)} +
  • + + {c.label} + + {c.value}{#if c.threshold} / {c.threshold}{/if} + + + {#if c.hint}{c.hint}{/if} + {#if c.action_href && c.status !== 'ok'} + Fix → + {/if} + +
    • + {/each} +
+ +
+ + {#if saved && !editing}Saved ✓{/if} + + {#if editing && form} +
{ + e.preventDefault(); + save(); + }} + > + {#each FIELDS as f (f.key)} + + {/each} + {#if saveError}{/if} +
+ + +
+
+ {/if} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/admin/HealthChecks.svelte.test.ts b/frontend/src/lib/components/admin/HealthChecks.svelte.test.ts new file mode 100644 index 0000000..43bd640 --- /dev/null +++ b/frontend/src/lib/components/admin/HealthChecks.svelte.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { render, screen, cleanup, waitFor, fireEvent } from '@testing-library/svelte'; +import HealthChecks from './HealthChecks.svelte'; +import * as adminApi from '$lib/api/admin'; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +function thresholds(): adminApi.HealthThresholds { + return { + disk_pct: 90, + mem_pct: 90, + dead_jobs_max: 100, + crawl_fail_pct: 20, + analysis_fail_pct: 20, + cron_freshness_hours: 26, + missing_covers_max: 500 + }; +} + +function report(over: Partial = {}): adminApi.HealthReport { + return { + status: 'warn', + checks: [ + { + id: 'dead_jobs', + label: 'Dead jobs', + status: 'warn', + value: '137', + threshold: '100', + hint: 'jobs that exhausted their retries', + action_href: '/admin/crawler' + }, + { + id: 'memory', + label: 'Memory usage', + status: 'ok', + value: '48%', + threshold: '90%', + hint: null, + action_href: null + } + ], + thresholds: thresholds(), + ...over + }; +} + +describe('HealthChecks', () => { + it('renders the overall status and each check, with a Fix link on failing ones', async () => { + vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report()); + render(HealthChecks); + + await waitFor(() => expect(screen.getByTestId('health-overall')).toBeTruthy()); + expect(screen.getByTestId('health-check-dead_jobs')).toBeTruthy(); + expect(screen.getByText('137')).toBeTruthy(); + // Failing check exposes a remediation link; the ok one does not. + const fix = screen.getByRole('link', { name: /fix/i }) as HTMLAnchorElement; + expect(fix.getAttribute('href')).toBe('/admin/crawler'); + }); + + it('saves edited thresholds via updateHealthThresholds', async () => { + vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report()); + const update = vi + .spyOn(adminApi, 'updateHealthThresholds') + .mockResolvedValue(report({ status: 'ok' })); + render(HealthChecks); + + await waitFor(() => expect(screen.getByTestId('health-thresholds-toggle')).toBeTruthy()); + await fireEvent.click(screen.getByTestId('health-thresholds-toggle')); + + const input = screen.getByTestId('threshold-dead_jobs_max') as HTMLInputElement; + await fireEvent.input(input, { target: { value: '250' } }); + await fireEvent.click(screen.getByTestId('threshold-save')); + + await waitFor(() => expect(update).toHaveBeenCalledTimes(1)); + expect(update.mock.calls[0][0].dead_jobs_max).toBe(250); + }); +}); diff --git a/frontend/src/routes/admin/+layout.svelte b/frontend/src/routes/admin/+layout.svelte index fc9cb52..15e1e9b 100644 --- a/frontend/src/routes/admin/+layout.svelte +++ b/frontend/src/routes/admin/+layout.svelte @@ -4,6 +4,7 @@ const tabs = [ { href: '/admin', label: 'Overview' }, + { href: '/admin/health', label: 'Health' }, { href: '/admin/users', label: 'Users' }, { href: '/admin/mangas', label: 'Mangas' }, { href: '/admin/crawler', label: 'Crawler' }, diff --git a/frontend/src/routes/admin/+page.svelte b/frontend/src/routes/admin/+page.svelte index bcc3b91..698ff81 100644 --- a/frontend/src/routes/admin/+page.svelte +++ b/frontend/src/routes/admin/+page.svelte @@ -9,11 +9,13 @@ analysisStatusStreamUrl, getCrawlerSettings, getAnalysisSettings, + getHealth, type SystemStats, type OverviewStats, type CrawlerStatus, type AnalysisMetrics, - type AnalysisEvent + type AnalysisEvent, + type HealthReport } from '$lib/api/admin'; import { formatBytes as fmtBytes } from '$lib/upload-validation'; import type { LayoutData } from './$types'; @@ -44,6 +46,8 @@ let liveCrawler = $state(false); let liveAnalysis = $state(false); + let health = $state(null); + let sysTimer: ReturnType | null = null; let overviewTimer: ReturnType | null = null; let crawlerSource: EventSource | null = null; @@ -86,6 +90,17 @@ // ignore } } + async function refreshHealth() { + try { + health = await getHealth(); + } catch { + // keep the last good snapshot; retried next tick + } + } + + const healthFailing = $derived( + health ? health.checks.filter((c) => c.status !== 'ok').length : 0 + ); async function loadSettings() { try { const c = await getCrawlerSettings(); @@ -183,6 +198,7 @@ refreshSys(); refreshOverview(); refreshMetrics(); + refreshHealth(); loadSettings(); getCrawlerStatus() .then((s) => (crawler = s)) @@ -196,6 +212,7 @@ overviewTimer = setInterval(() => { refreshOverview(); refreshMetrics(); + refreshHealth(); }, 60000); if (typeof document !== 'undefined') { document.addEventListener('visibilitychange', onVisibilityChange); @@ -237,6 +254,26 @@

Overview

+{#if health} + + + + {#if health.status === 'ok'} + All health checks passing + {:else} + {healthFailing} health check{healthFailing === 1 ? '' : 's'} need attention + {/if} + + View → + +{/if} + {#if sys.alerts.length > 0 || extraAlerts.length > 0}
{#each sys.alerts as a (a.message)} @@ -434,6 +471,44 @@ h1 { margin: 0 0 var(--space-4) 0; } + .health-summary { + display: flex; + align-items: center; + gap: var(--space-2); + padding: var(--space-2) var(--space-3); + margin-bottom: var(--space-3); + border: 1px solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); + color: var(--text); + font-size: var(--font-sm); + } + .health-summary:hover { + background: var(--surface-elevated); + text-decoration: none; + } + .health-summary .glyph { + font-weight: var(--weight-semibold); + } + .health-summary .hs-view { + margin-left: auto; + color: var(--text-muted); + } + .health-summary.status-ok .glyph { + color: var(--success, #16a34a); + } + .health-summary.status-warn { + border-left: 4px solid var(--warning, #d97706); + } + .health-summary.status-warn .glyph { + color: var(--warning, #d97706); + } + .health-summary.status-critical { + border-left: 4px solid var(--danger, #dc2626); + } + .health-summary.status-critical .glyph { + color: var(--danger, #dc2626); + } .alerts { display: flex; flex-direction: column; diff --git a/frontend/src/routes/admin/health/+page.svelte b/frontend/src/routes/admin/health/+page.svelte new file mode 100644 index 0000000..b25413d --- /dev/null +++ b/frontend/src/routes/admin/health/+page.svelte @@ -0,0 +1,26 @@ + + +Health · Admin + +

Health

+

+ Operational checks rolled up from system sensors, the job queue, recent + failure rates and the crawler daemon. Thresholds are editable below and + apply immediately. +

+ + + +