feat(admin): operational health checks page with editable thresholds
New /admin/health rolls up signals already collected — disk/memory sensors, dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata cron freshness, and the live crawler session/browser flags — into a flat checks list (ok/warn/critical) with an overall status and remediation links. The overview gains a compact Health summary banner linking in. GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists the thresholds to app_settings (merged over compiled defaults, forward-compat via serde(default)), validates ranges (400 on bad input), and audit-logs the change in one transaction. New repo::crawler::last_metadata_tick_at getter and a lightweight system::memory_percent_used (no CPU settle delay). Pure status helpers (over_limit for gauges, exceeds for backlog counts so a 0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape, gating, persistence+audit, and validation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
444
backend/src/api/admin/health.rs
Normal file
444
backend/src/api/admin/health.rs
Normal file
@@ -0,0 +1,444 @@
|
||||
//! GET /admin/health — aggregated operational health checks
|
||||
//! PUT /admin/health/thresholds — edit the alerting thresholds
|
||||
//!
|
||||
//! A single read that rolls up signals already collected elsewhere (system
|
||||
//! sensors, queue depths, recent failure rates, cron freshness, the live
|
||||
//! crawler session/browser flags) into a flat list of checks with an
|
||||
//! overall status. Thresholds live in `app_settings['health_thresholds']`
|
||||
//! (JSONB, merged over compiled defaults) and are admin-editable + audited.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::{get, put};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{Duration, Utc};
|
||||
use futures_util::TryFutureExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::crawler::browser_manager::RestartPhase;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
const KEY: &str = "health_thresholds";
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/health", get(health))
|
||||
.route("/admin/health/thresholds", put(update_thresholds))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Thresholds (persisted, editable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn d_disk() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_mem() -> f64 {
|
||||
90.0
|
||||
}
|
||||
fn d_dead() -> i64 {
|
||||
100
|
||||
}
|
||||
fn d_crawl_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_analysis_fail() -> f64 {
|
||||
20.0
|
||||
}
|
||||
fn d_cron() -> i64 {
|
||||
26
|
||||
}
|
||||
fn d_covers() -> i64 {
|
||||
500
|
||||
}
|
||||
|
||||
/// Alerting thresholds. Each field has a `serde(default)` so a stored row
|
||||
/// that predates a newly-added field still deserializes (the missing field
|
||||
/// takes its compiled default — forward-compatible).
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct HealthThresholds {
|
||||
#[serde(default = "d_disk")]
|
||||
pub disk_pct: f64,
|
||||
#[serde(default = "d_mem")]
|
||||
pub mem_pct: f64,
|
||||
#[serde(default = "d_dead")]
|
||||
pub dead_jobs_max: i64,
|
||||
#[serde(default = "d_crawl_fail")]
|
||||
pub crawl_fail_pct: f64,
|
||||
#[serde(default = "d_analysis_fail")]
|
||||
pub analysis_fail_pct: f64,
|
||||
#[serde(default = "d_cron")]
|
||||
pub cron_freshness_hours: i64,
|
||||
#[serde(default = "d_covers")]
|
||||
pub missing_covers_max: i64,
|
||||
}
|
||||
|
||||
impl Default for HealthThresholds {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
disk_pct: d_disk(),
|
||||
mem_pct: d_mem(),
|
||||
dead_jobs_max: d_dead(),
|
||||
crawl_fail_pct: d_crawl_fail(),
|
||||
analysis_fail_pct: d_analysis_fail(),
|
||||
cron_freshness_hours: d_cron(),
|
||||
missing_covers_max: d_covers(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthThresholds {
|
||||
/// Reject nonsensical values so a check can't be silently disabled by a
|
||||
/// fat-fingered edit. Percentages 1..=100; counts/hours must be positive.
|
||||
fn validate(&self) -> Result<(), String> {
|
||||
let pct = |v: f64, name: &str| {
|
||||
if (1.0..=100.0).contains(&v) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{name} must be between 1 and 100"))
|
||||
}
|
||||
};
|
||||
pct(self.disk_pct, "disk_pct")?;
|
||||
pct(self.mem_pct, "mem_pct")?;
|
||||
pct(self.crawl_fail_pct, "crawl_fail_pct")?;
|
||||
pct(self.analysis_fail_pct, "analysis_fail_pct")?;
|
||||
if self.dead_jobs_max < 0 {
|
||||
return Err("dead_jobs_max must be >= 0".into());
|
||||
}
|
||||
if self.missing_covers_max < 0 {
|
||||
return Err("missing_covers_max must be >= 0".into());
|
||||
}
|
||||
if self.cron_freshness_hours < 1 {
|
||||
return Err("cron_freshness_hours must be >= 1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_thresholds(state: &AppState) -> AppResult<HealthThresholds> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY).await? {
|
||||
// serde(default) on each field fills anything the stored object omits.
|
||||
Some(v) => serde_json::from_value(v).unwrap_or_default(),
|
||||
None => HealthThresholds::default(),
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CheckStatus {
|
||||
Ok,
|
||||
Warn,
|
||||
Critical,
|
||||
}
|
||||
|
||||
impl CheckStatus {
|
||||
fn rank(self) -> u8 {
|
||||
match self {
|
||||
CheckStatus::Ok => 0,
|
||||
CheckStatus::Warn => 1,
|
||||
CheckStatus::Critical => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` while `value` is strictly below `limit`, else `Warn`. Used for
|
||||
/// "near a ceiling" gauges (disk/memory/fail-rate percentages, cron-age
|
||||
/// hours) where hitting the threshold is itself the alert. Pure so the
|
||||
/// rule is unit-testable without the DB/sensors.
|
||||
pub(crate) fn over_limit(value: f64, limit: f64) -> CheckStatus {
|
||||
if value < limit {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
CheckStatus::Warn
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok` until `value` strictly exceeds `limit`. Used for "max allowed"
|
||||
/// backlog counts so a limit of `0` warns on the first item — and `0` items
|
||||
/// against a `0` limit stays `Ok` (no false alarm at exactly the boundary).
|
||||
pub(crate) fn exceeds(value: i64, limit: i64) -> CheckStatus {
|
||||
if value > limit {
|
||||
CheckStatus::Warn
|
||||
} else {
|
||||
CheckStatus::Ok
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthCheck {
|
||||
pub id: &'static str,
|
||||
pub label: &'static str,
|
||||
pub status: CheckStatus,
|
||||
pub value: String,
|
||||
pub threshold: Option<String>,
|
||||
pub hint: Option<String>,
|
||||
/// Admin route that remediates this check, when one applies.
|
||||
pub action_href: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthReport {
|
||||
/// Worst status across all checks (drives the page/summary badge).
|
||||
pub status: CheckStatus,
|
||||
pub checks: Vec<HealthCheck>,
|
||||
pub thresholds: HealthThresholds,
|
||||
}
|
||||
|
||||
async fn health(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
let thresholds = load_thresholds(&state).await?;
|
||||
let since_24h = Utc::now() - Duration::hours(24);
|
||||
|
||||
// DB-derived signals run concurrently — all hit existing indexes.
|
||||
let ((pending, running, dead), missing_covers, crawl, analysis, last_tick) = tokio::try_join!(
|
||||
repo::crawler::job_state_counts(&state.db).map_err(AppError::from),
|
||||
repo::crawler::count_missing_covers(&state.db).map_err(AppError::from),
|
||||
repo::crawl_metrics::summary(&state.db, Some(since_24h)).map_err(AppError::from),
|
||||
repo::page_analysis::analysis_metrics(&state.db, Some(since_24h)),
|
||||
repo::crawler::last_metadata_tick_at(&state.db).map_err(AppError::from),
|
||||
)?;
|
||||
let _ = (pending, running); // queue depth shown elsewhere; not a check yet
|
||||
|
||||
let mut checks: Vec<HealthCheck> = Vec::new();
|
||||
|
||||
// --- system sensors ---
|
||||
if let Some(d) = state.storage.local_root().and_then(super::system::disk_stats_for) {
|
||||
checks.push(HealthCheck {
|
||||
id: "disk",
|
||||
label: "Disk usage",
|
||||
status: over_limit(d.percent_used, thresholds.disk_pct),
|
||||
value: format!("{:.0}%", d.percent_used),
|
||||
threshold: Some(format!("{:.0}%", thresholds.disk_pct)),
|
||||
hint: Some("free space on the storage volume".into()),
|
||||
action_href: None,
|
||||
});
|
||||
}
|
||||
let mem = super::system::memory_percent_used();
|
||||
checks.push(HealthCheck {
|
||||
id: "memory",
|
||||
label: "Memory usage",
|
||||
status: over_limit(mem, thresholds.mem_pct),
|
||||
value: format!("{mem:.0}%"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.mem_pct)),
|
||||
hint: None,
|
||||
action_href: None,
|
||||
});
|
||||
|
||||
// --- queue / backlog ---
|
||||
checks.push(HealthCheck {
|
||||
id: "dead_jobs",
|
||||
label: "Dead jobs",
|
||||
status: exceeds(dead, thresholds.dead_jobs_max),
|
||||
value: dead.to_string(),
|
||||
threshold: Some(thresholds.dead_jobs_max.to_string()),
|
||||
hint: Some("jobs that exhausted their retries".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
checks.push(HealthCheck {
|
||||
id: "missing_covers",
|
||||
label: "Missing covers",
|
||||
status: exceeds(missing_covers, thresholds.missing_covers_max),
|
||||
value: missing_covers.to_string(),
|
||||
threshold: Some(thresholds.missing_covers_max.to_string()),
|
||||
hint: Some("mangas still awaiting a cover download".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
// --- failure rates (last 24h) ---
|
||||
let crawl_total: i64 = crawl.iter().map(|s| s.n).sum();
|
||||
let crawl_failed: i64 = crawl.iter().map(|s| s.failed).sum();
|
||||
let crawl_pct = pct(crawl_failed, crawl_total);
|
||||
checks.push(HealthCheck {
|
||||
id: "crawl_fail_rate",
|
||||
label: "Crawl failure rate (24h)",
|
||||
status: if crawl_total == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(crawl_pct, thresholds.crawl_fail_pct)
|
||||
},
|
||||
value: format!("{crawl_pct:.0}% ({crawl_failed}/{crawl_total})"),
|
||||
threshold: Some(format!("{:.0}%", thresholds.crawl_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
let an_pct = pct(analysis.failed, analysis.n);
|
||||
checks.push(HealthCheck {
|
||||
id: "analysis_fail_rate",
|
||||
label: "Analysis failure rate (24h)",
|
||||
status: if analysis.n == 0 {
|
||||
CheckStatus::Ok
|
||||
} else {
|
||||
over_limit(an_pct, thresholds.analysis_fail_pct)
|
||||
},
|
||||
value: format!("{an_pct:.0}% ({}/{})", analysis.failed, analysis.n),
|
||||
threshold: Some(format!("{:.0}%", thresholds.analysis_fail_pct)),
|
||||
hint: None,
|
||||
action_href: Some("/admin/analysis"),
|
||||
});
|
||||
|
||||
// --- crawler daemon liveness (only when the daemon is configured) ---
|
||||
if let Some(c) = state.crawler().as_ref() {
|
||||
// Cron freshness: stale tick = the daily pass hasn't run on time.
|
||||
match last_tick {
|
||||
Some(t) => {
|
||||
let age_h = (Utc::now() - t).num_hours();
|
||||
checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: over_limit(age_h as f64, thresholds.cron_freshness_hours as f64),
|
||||
value: format!("{age_h}h ago"),
|
||||
threshold: Some(format!("{}h", thresholds.cron_freshness_hours)),
|
||||
hint: Some("time since the last metadata pass tick".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
None => checks.push(HealthCheck {
|
||||
id: "cron_freshness",
|
||||
label: "Metadata cron freshness",
|
||||
status: CheckStatus::Ok,
|
||||
value: "no tick yet".into(),
|
||||
threshold: None,
|
||||
hint: Some("the daemon has not completed a pass yet".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
}),
|
||||
}
|
||||
|
||||
let expired = c.session.is_expired();
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_session",
|
||||
label: "Crawler session",
|
||||
status: if expired { CheckStatus::Critical } else { CheckStatus::Ok },
|
||||
value: if expired { "expired".into() } else { "ok".into() },
|
||||
threshold: None,
|
||||
hint: expired.then(|| "chapter workers are idle until the session is refreshed".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
|
||||
let healthy = matches!(c.browser_manager.phase(), RestartPhase::Healthy);
|
||||
let phase = match c.browser_manager.phase() {
|
||||
RestartPhase::Healthy => "healthy",
|
||||
RestartPhase::Draining => "draining",
|
||||
RestartPhase::Restarting => "restarting",
|
||||
};
|
||||
checks.push(HealthCheck {
|
||||
id: "crawler_browser",
|
||||
label: "Crawler browser",
|
||||
status: if healthy { CheckStatus::Ok } else { CheckStatus::Warn },
|
||||
value: phase.to_string(),
|
||||
threshold: None,
|
||||
hint: (!healthy).then(|| "Chromium is restarting or unavailable".into()),
|
||||
action_href: Some("/admin/crawler"),
|
||||
});
|
||||
}
|
||||
|
||||
let status = checks
|
||||
.iter()
|
||||
.map(|c| c.status)
|
||||
.max_by_key(|s| s.rank())
|
||||
.unwrap_or(CheckStatus::Ok);
|
||||
|
||||
Ok(Json(HealthReport {
|
||||
status,
|
||||
checks,
|
||||
thresholds,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Percentage `numerator/denominator`, guarding divide-by-zero.
|
||||
fn pct(numerator: i64, denominator: i64) -> f64 {
|
||||
if denominator <= 0 {
|
||||
0.0
|
||||
} else {
|
||||
(numerator as f64) * 100.0 / (denominator as f64)
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_thresholds(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(body): Json<HealthThresholds>,
|
||||
) -> AppResult<Json<HealthReport>> {
|
||||
body.validate().map_err(AppError::InvalidInput)?;
|
||||
let value = serde_json::to_value(body).map_err(|e| AppError::Other(e.into()))?;
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_health_thresholds",
|
||||
"settings",
|
||||
None,
|
||||
value,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
// Re-evaluate so the client immediately sees checks recomputed against
|
||||
// the new thresholds.
|
||||
health(State(state), admin).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn over_limit_is_ok_below_warn_at_or_above() {
|
||||
assert_eq!(over_limit(10.0, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(19.999, 20.0), CheckStatus::Ok);
|
||||
assert_eq!(over_limit(20.0, 20.0), CheckStatus::Warn);
|
||||
assert_eq!(over_limit(99.0, 20.0), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exceeds_is_strict_and_safe_at_zero() {
|
||||
assert_eq!(exceeds(0, 0), CheckStatus::Ok); // 0 backlog, 0 limit → no false alarm
|
||||
assert_eq!(exceeds(1, 0), CheckStatus::Warn);
|
||||
assert_eq!(exceeds(100, 100), CheckStatus::Ok);
|
||||
assert_eq!(exceeds(101, 100), CheckStatus::Warn);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pct_guards_zero_denominator() {
|
||||
assert_eq!(pct(0, 0), 0.0);
|
||||
assert_eq!(pct(5, 0), 0.0);
|
||||
assert_eq!(pct(1, 4), 25.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worst_status_ranking() {
|
||||
assert!(CheckStatus::Critical.rank() > CheckStatus::Warn.rank());
|
||||
assert!(CheckStatus::Warn.rank() > CheckStatus::Ok.rank());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_validate_rejects_out_of_range() {
|
||||
let mut t = HealthThresholds::default();
|
||||
assert!(t.validate().is_ok());
|
||||
t.disk_pct = 0.0;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.dead_jobs_max = -1;
|
||||
assert!(t.validate().is_err());
|
||||
t = HealthThresholds::default();
|
||||
t.cron_freshness_hours = 0;
|
||||
assert!(t.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_merge_defaults_for_missing_fields() {
|
||||
// A stored object that predates `missing_covers_max` still decodes.
|
||||
let partial = serde_json::json!({ "disk_pct": 80.0 });
|
||||
let t: HealthThresholds = serde_json::from_value(partial).unwrap();
|
||||
assert_eq!(t.disk_pct, 80.0);
|
||||
assert_eq!(t.missing_covers_max, d_covers());
|
||||
assert_eq!(t.cron_freshness_hours, d_cron());
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@
|
||||
pub mod analysis;
|
||||
pub mod audit;
|
||||
pub mod crawler;
|
||||
pub mod health;
|
||||
pub mod mangas;
|
||||
pub mod overview;
|
||||
pub mod resync;
|
||||
@@ -28,6 +29,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(system::routes())
|
||||
.merge(overview::routes())
|
||||
.merge(audit::routes())
|
||||
.merge(health::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
|
||||
@@ -169,6 +169,21 @@ fn temperatures() -> Vec<TempStat> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lightweight memory-utilization percentage for the health check. Unlike
|
||||
/// [`memory_and_cpu`] this skips CPU sampling and its 250ms settle delay —
|
||||
/// health is polled more often and only needs the memory figure.
|
||||
pub(crate) fn memory_percent_used() -> f64 {
|
||||
let mut sys =
|
||||
System::new_with_specifics(RefreshKind::new().with_memory(MemoryRefreshKind::everything()));
|
||||
sys.refresh_memory();
|
||||
let total = sys.total_memory();
|
||||
if total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(sys.used_memory() as f64) * 100.0 / (total as f64)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn disk_stats_for(root: &Path) -> Option<DiskStats> {
|
||||
let s = nix::sys::statvfs::statvfs(root).ok()?;
|
||||
// statvfs reports `f_frsize * f_blocks` for total bytes. `f_bavail`
|
||||
|
||||
@@ -619,6 +619,25 @@ pub async fn last_run_completed_cleanly(
|
||||
.unwrap_or(true))
|
||||
}
|
||||
|
||||
/// Timestamp of the most recent metadata-pass cron tick, or `None` if the
|
||||
/// daemon has never ticked (fresh DB / cron disabled). Stored by the daemon
|
||||
/// under `crawler_state['last_metadata_tick_at']` as `{"at": <rfc3339>}`.
|
||||
/// Used by the health check that flags a stale/stuck cron.
|
||||
pub async fn last_metadata_tick_at(
|
||||
pool: &PgPool,
|
||||
) -> sqlx::Result<Option<DateTime<Utc>>> {
|
||||
let row: Option<serde_json::Value> =
|
||||
sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = 'last_metadata_tick_at'")
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.and_then(|v| {
|
||||
v.get("at")
|
||||
.and_then(|s| s.as_str())
|
||||
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reconcile: find source-list mangas missing from the DB.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
207
backend/tests/api_admin_health.rs
Normal file
207
backend/tests/api_admin_health.rs
Normal file
@@ -0,0 +1,207 @@
|
||||
//! Integration tests for the admin health endpoints: `GET /v1/admin/health`
|
||||
//! (shape, admin-gating, default checks present without a daemon) and
|
||||
//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row).
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use common::{body_json, get_with_cookie, harness, register_user};
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = register_user(app).await;
|
||||
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// PUT helper (the common module exposes POST helpers; build a PUT here).
|
||||
fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request<axum::body::Body> {
|
||||
axum::http::Request::builder()
|
||||
.method("PUT")
|
||||
.uri(uri)
|
||||
.header(axum::http::header::COOKIE, cookie)
|
||||
.header(axum::http::header::CONTENT_TYPE, "application/json")
|
||||
.body(axum::body::Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn valid_thresholds() -> serde_json::Value {
|
||||
json!({
|
||||
"disk_pct": 85.0,
|
||||
"mem_pct": 80.0,
|
||||
"dead_jobs_max": 50,
|
||||
"crawl_fail_pct": 25.0,
|
||||
"analysis_fail_pct": 30.0,
|
||||
"cron_freshness_hours": 12,
|
||||
"missing_covers_max": 200
|
||||
})
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requires_admin(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn report_has_status_checks_and_default_thresholds(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
// Overall status + a non-empty checks array (memory / dead_jobs /
|
||||
// missing_covers / fail-rates are always present, daemon or not).
|
||||
assert!(body["status"].is_string());
|
||||
let checks = body["checks"].as_array().unwrap();
|
||||
assert!(!checks.is_empty());
|
||||
let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect();
|
||||
assert!(ids.contains(&"memory"));
|
||||
assert!(ids.contains(&"dead_jobs"));
|
||||
assert!(ids.contains(&"crawl_fail_rate"));
|
||||
// The harness wires no crawler daemon, so session/browser checks are absent.
|
||||
assert!(!ids.contains(&"crawler_session"));
|
||||
// Default thresholds echoed back.
|
||||
assert_eq!(body["thresholds"]["dead_jobs_max"], 100);
|
||||
assert_eq!(body["thresholds"]["disk_pct"], 90.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
// Lower the threshold to 0, then a single dead job trips the check.
|
||||
let mut t = valid_thresholds();
|
||||
t["dead_jobs_max"] = json!(0);
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
t,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')",
|
||||
)
|
||||
.bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" }))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
let dead = body["checks"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|c| c["id"] == "dead_jobs")
|
||||
.unwrap();
|
||||
assert_eq!(dead["status"], "warn");
|
||||
assert_eq!(body["status"], "warn"); // overall rolls up the worst
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_thresholds_persists_and_audits(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
valid_thresholds(),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
// Response re-evaluates against the new thresholds.
|
||||
assert_eq!(body["thresholds"]["dead_jobs_max"], 50);
|
||||
|
||||
// Persisted to app_settings.
|
||||
let stored: serde_json::Value =
|
||||
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored["mem_pct"], 80.0);
|
||||
|
||||
// Audit row written.
|
||||
let action: String =
|
||||
sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(action, "update_health_thresholds");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) {
|
||||
// Absent → None.
|
||||
assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
// Stored in the daemon's `{"at": rfc3339}` shape → parsed back.
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)",
|
||||
)
|
||||
.bind(json!({ "at": "2026-06-18T09:30:00Z" }))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let got = mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_thresholds_rejects_out_of_range(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let mut bad = valid_thresholds();
|
||||
bad["disk_pct"] = json!(150.0); // > 100
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(put_json_with_cookie(
|
||||
"/api/v1/admin/health/thresholds",
|
||||
bad,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
@@ -43,7 +43,9 @@ import {
|
||||
getCrawlerMetrics,
|
||||
listCrawlerOps,
|
||||
getAnalysisMetrics,
|
||||
listAuditLog
|
||||
listAuditLog,
|
||||
getHealth,
|
||||
updateHealthThresholds
|
||||
} from './admin';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -506,6 +508,48 @@ describe('admin crawler api client', () => {
|
||||
expect(url).toContain('days=7');
|
||||
});
|
||||
|
||||
const healthFixture = {
|
||||
status: 'ok' as const,
|
||||
checks: [
|
||||
{
|
||||
id: 'memory',
|
||||
label: 'Memory usage',
|
||||
status: 'ok' as const,
|
||||
value: '48%',
|
||||
threshold: '90%',
|
||||
hint: null,
|
||||
action_href: null
|
||||
}
|
||||
],
|
||||
thresholds: {
|
||||
disk_pct: 90,
|
||||
mem_pct: 90,
|
||||
dead_jobs_max: 100,
|
||||
crawl_fail_pct: 20,
|
||||
analysis_fail_pct: 20,
|
||||
cron_freshness_hours: 26,
|
||||
missing_covers_max: 500
|
||||
}
|
||||
};
|
||||
|
||||
it('getHealth GETs /v1/admin/health', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||
const r = await getHealth();
|
||||
expect(r.status).toBe('ok');
|
||||
expect(r.checks[0].id).toBe('memory');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/health$/);
|
||||
});
|
||||
|
||||
it('updateHealthThresholds PUTs the thresholds body', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
|
||||
await updateHealthThresholds(healthFixture.thresholds);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/health\/thresholds$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('PUT');
|
||||
expect(JSON.parse(init.body as string).dead_jobs_max).toBe(100);
|
||||
});
|
||||
|
||||
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||
const r = await runCrawlerPass();
|
||||
|
||||
@@ -1018,3 +1018,46 @@ export async function listAuditLog(
|
||||
const qs = params.toString();
|
||||
return request<AuditPage>(`/v1/admin/audit${qs ? `?${qs}` : ''}`, init);
|
||||
}
|
||||
|
||||
// ---- operational health checks ---------------------------------------------
|
||||
|
||||
export type CheckStatus = 'ok' | 'warn' | 'critical';
|
||||
|
||||
export type HealthCheck = {
|
||||
id: string;
|
||||
label: string;
|
||||
status: CheckStatus;
|
||||
value: string;
|
||||
threshold: string | null;
|
||||
hint: string | null;
|
||||
/** Admin route that remediates this check, when one applies. */
|
||||
action_href: string | null;
|
||||
};
|
||||
|
||||
export type HealthThresholds = {
|
||||
disk_pct: number;
|
||||
mem_pct: number;
|
||||
dead_jobs_max: number;
|
||||
crawl_fail_pct: number;
|
||||
analysis_fail_pct: number;
|
||||
cron_freshness_hours: number;
|
||||
missing_covers_max: number;
|
||||
};
|
||||
|
||||
export type HealthReport = {
|
||||
status: CheckStatus;
|
||||
checks: HealthCheck[];
|
||||
thresholds: HealthThresholds;
|
||||
};
|
||||
|
||||
export async function getHealth(init?: RequestInit): Promise<HealthReport> {
|
||||
return request<HealthReport>('/v1/admin/health', init);
|
||||
}
|
||||
|
||||
export async function updateHealthThresholds(t: HealthThresholds): Promise<HealthReport> {
|
||||
return request<HealthReport>('/v1/admin/health/thresholds', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(t)
|
||||
});
|
||||
}
|
||||
|
||||
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
306
frontend/src/lib/components/admin/HealthChecks.svelte
Normal file
@@ -0,0 +1,306 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import {
|
||||
getHealth,
|
||||
updateHealthThresholds,
|
||||
type HealthReport,
|
||||
type HealthThresholds,
|
||||
type CheckStatus
|
||||
} from '$lib/api/admin';
|
||||
|
||||
const POLL_MS = 30_000;
|
||||
|
||||
let report = $state<HealthReport | null>(null);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let inflight: AbortController | null = null;
|
||||
|
||||
// Threshold editor state.
|
||||
let editing = $state(false);
|
||||
let form = $state<HealthThresholds | null>(null);
|
||||
let saving = $state(false);
|
||||
let saveError = $state<string | null>(null);
|
||||
let saved = $state(false);
|
||||
|
||||
async function load() {
|
||||
inflight?.abort();
|
||||
const ctrl = new AbortController();
|
||||
inflight = ctrl;
|
||||
try {
|
||||
const r = await getHealth({ signal: ctrl.signal });
|
||||
report = r;
|
||||
error = null;
|
||||
} catch (e) {
|
||||
if ((e as Error)?.name === 'AbortError') return;
|
||||
error = e instanceof Error ? e.message : 'Failed to load health.';
|
||||
} finally {
|
||||
if (inflight === ctrl) loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
load();
|
||||
timer = setInterval(load, POLL_MS);
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (timer) clearInterval(timer);
|
||||
inflight?.abort();
|
||||
});
|
||||
|
||||
function startEdit() {
|
||||
// Snapshot current thresholds into an editable copy.
|
||||
form = report ? { ...report.thresholds } : null;
|
||||
saveError = null;
|
||||
saved = false;
|
||||
editing = true;
|
||||
}
|
||||
|
||||
async function save() {
|
||||
if (!form) return;
|
||||
saving = true;
|
||||
saveError = null;
|
||||
saved = false;
|
||||
try {
|
||||
report = await updateHealthThresholds(form);
|
||||
saved = true;
|
||||
editing = false;
|
||||
} catch (e) {
|
||||
saveError = e instanceof Error ? e.message : 'Failed to save thresholds.';
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
const STATUS_GLYPH: Record<CheckStatus, string> = {
|
||||
ok: '✓',
|
||||
warn: '⚠',
|
||||
critical: '✗'
|
||||
};
|
||||
const STATUS_LABEL: Record<CheckStatus, string> = {
|
||||
ok: 'Healthy',
|
||||
warn: 'Needs attention',
|
||||
critical: 'Critical'
|
||||
};
|
||||
|
||||
const failing = $derived(
|
||||
report ? report.checks.filter((c) => c.status !== 'ok').length : 0
|
||||
);
|
||||
|
||||
// Editable threshold fields, with units for the form labels.
|
||||
const FIELDS: { key: keyof HealthThresholds; label: string; unit: string }[] = [
|
||||
{ key: 'disk_pct', label: 'Disk usage', unit: '%' },
|
||||
{ key: 'mem_pct', label: 'Memory usage', unit: '%' },
|
||||
{ key: 'dead_jobs_max', label: 'Dead jobs (max)', unit: '' },
|
||||
{ key: 'crawl_fail_pct', label: 'Crawl failure rate', unit: '%' },
|
||||
{ key: 'analysis_fail_pct', label: 'Analysis failure rate', unit: '%' },
|
||||
{ key: 'cron_freshness_hours', label: 'Cron freshness', unit: 'h' },
|
||||
{ key: 'missing_covers_max', label: 'Missing covers (max)', unit: '' }
|
||||
];
|
||||
</script>
|
||||
|
||||
<section class="health" data-testid="health-checks">
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if loading && !report}
|
||||
<p class="muted" data-testid="health-loading">Loading…</p>
|
||||
{:else if report}
|
||||
<header class="overall status-{report.status}" data-testid="health-overall">
|
||||
<span class="glyph">{STATUS_GLYPH[report.status]}</span>
|
||||
<span class="overall-label">{STATUS_LABEL[report.status]}</span>
|
||||
{#if failing > 0}
|
||||
<span class="muted">· {failing} check{failing === 1 ? '' : 's'} need attention</span>
|
||||
{/if}
|
||||
</header>
|
||||
|
||||
<ul class="checks">
|
||||
{#each report.checks as c (c.id)}
|
||||
<li class="check status-{c.status}" data-testid={`health-check-${c.id}`}>
|
||||
<span class="glyph" aria-hidden="true">{STATUS_GLYPH[c.status]}</span>
|
||||
<span class="label">{c.label}</span>
|
||||
<span class="value">
|
||||
{c.value}{#if c.threshold}<span class="muted"> / {c.threshold}</span>{/if}
|
||||
</span>
|
||||
<span class="meta">
|
||||
{#if c.hint}<span class="hint muted">{c.hint}</span>{/if}
|
||||
{#if c.action_href && c.status !== 'ok'}
|
||||
<a class="fix" href={c.action_href}>Fix →</a>
|
||||
{/if}
|
||||
</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
|
||||
<div class="thresholds">
|
||||
<button
|
||||
type="button"
|
||||
class="disclosure"
|
||||
onclick={() => (editing ? (editing = false) : startEdit())}
|
||||
aria-expanded={editing}
|
||||
data-testid="health-thresholds-toggle"
|
||||
>
|
||||
{editing ? '▾' : '▸'} Configure thresholds
|
||||
</button>
|
||||
{#if saved && !editing}<span class="ok-note">Saved ✓</span>{/if}
|
||||
|
||||
{#if editing && form}
|
||||
<form
|
||||
class="threshold-form"
|
||||
onsubmit={(e) => {
|
||||
e.preventDefault();
|
||||
save();
|
||||
}}
|
||||
>
|
||||
{#each FIELDS as f (f.key)}
|
||||
<label>
|
||||
<span>{f.label}{f.unit ? ` (${f.unit})` : ''}</span>
|
||||
<input
|
||||
type="number"
|
||||
step={f.unit === '%' ? '1' : '1'}
|
||||
min="0"
|
||||
bind:value={form[f.key]}
|
||||
data-testid={`threshold-${f.key}`}
|
||||
/>
|
||||
</label>
|
||||
{/each}
|
||||
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
||||
<div class="actions">
|
||||
<button type="submit" disabled={saving} data-testid="threshold-save">
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
<button type="button" class="secondary" onclick={() => (editing = false)}>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.overall {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-semibold);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.overall .glyph {
|
||||
font-size: var(--font-lg);
|
||||
}
|
||||
.checks {
|
||||
list-style: none;
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
padding: 0;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.check {
|
||||
display: grid;
|
||||
grid-template-columns: 1.4rem 1fr auto;
|
||||
grid-template-areas: 'glyph label value' '. meta meta';
|
||||
align-items: center;
|
||||
gap: var(--space-1) var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.check:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.check .glyph {
|
||||
grid-area: glyph;
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.label {
|
||||
grid-area: label;
|
||||
}
|
||||
.value {
|
||||
grid-area: value;
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.meta {
|
||||
grid-area: meta;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.hint {
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.fix {
|
||||
white-space: nowrap;
|
||||
}
|
||||
/* Status colors on the glyph + left accent. */
|
||||
.status-ok .glyph {
|
||||
color: var(--success, #16a34a);
|
||||
}
|
||||
.status-warn {
|
||||
box-shadow: inset 3px 0 0 var(--warning, #d97706);
|
||||
}
|
||||
.status-warn .glyph {
|
||||
color: var(--warning, #d97706);
|
||||
}
|
||||
.status-critical {
|
||||
box-shadow: inset 3px 0 0 var(--danger, #dc2626);
|
||||
}
|
||||
.status-critical .glyph {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.disclosure {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
padding: 0;
|
||||
}
|
||||
.ok-note {
|
||||
margin-left: var(--space-2);
|
||||
color: var(--success, #16a34a);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.threshold-form {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
margin-top: var(--space-3);
|
||||
max-width: 40rem;
|
||||
}
|
||||
.threshold-form label {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.threshold-form input {
|
||||
height: 34px;
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.threshold-form .actions {
|
||||
grid-column: 1 / -1;
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
button.secondary {
|
||||
background: var(--surface-elevated);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { render, screen, cleanup, waitFor, fireEvent } from '@testing-library/svelte';
|
||||
import HealthChecks from './HealthChecks.svelte';
|
||||
import * as adminApi from '$lib/api/admin';
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function thresholds(): adminApi.HealthThresholds {
|
||||
return {
|
||||
disk_pct: 90,
|
||||
mem_pct: 90,
|
||||
dead_jobs_max: 100,
|
||||
crawl_fail_pct: 20,
|
||||
analysis_fail_pct: 20,
|
||||
cron_freshness_hours: 26,
|
||||
missing_covers_max: 500
|
||||
};
|
||||
}
|
||||
|
||||
function report(over: Partial<adminApi.HealthReport> = {}): adminApi.HealthReport {
|
||||
return {
|
||||
status: 'warn',
|
||||
checks: [
|
||||
{
|
||||
id: 'dead_jobs',
|
||||
label: 'Dead jobs',
|
||||
status: 'warn',
|
||||
value: '137',
|
||||
threshold: '100',
|
||||
hint: 'jobs that exhausted their retries',
|
||||
action_href: '/admin/crawler'
|
||||
},
|
||||
{
|
||||
id: 'memory',
|
||||
label: 'Memory usage',
|
||||
status: 'ok',
|
||||
value: '48%',
|
||||
threshold: '90%',
|
||||
hint: null,
|
||||
action_href: null
|
||||
}
|
||||
],
|
||||
thresholds: thresholds(),
|
||||
...over
|
||||
};
|
||||
}
|
||||
|
||||
describe('HealthChecks', () => {
|
||||
it('renders the overall status and each check, with a Fix link on failing ones', async () => {
|
||||
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||
render(HealthChecks);
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId('health-overall')).toBeTruthy());
|
||||
expect(screen.getByTestId('health-check-dead_jobs')).toBeTruthy();
|
||||
expect(screen.getByText('137')).toBeTruthy();
|
||||
// Failing check exposes a remediation link; the ok one does not.
|
||||
const fix = screen.getByRole('link', { name: /fix/i }) as HTMLAnchorElement;
|
||||
expect(fix.getAttribute('href')).toBe('/admin/crawler');
|
||||
});
|
||||
|
||||
it('saves edited thresholds via updateHealthThresholds', async () => {
|
||||
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
|
||||
const update = vi
|
||||
.spyOn(adminApi, 'updateHealthThresholds')
|
||||
.mockResolvedValue(report({ status: 'ok' }));
|
||||
render(HealthChecks);
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId('health-thresholds-toggle')).toBeTruthy());
|
||||
await fireEvent.click(screen.getByTestId('health-thresholds-toggle'));
|
||||
|
||||
const input = screen.getByTestId('threshold-dead_jobs_max') as HTMLInputElement;
|
||||
await fireEvent.input(input, { target: { value: '250' } });
|
||||
await fireEvent.click(screen.getByTestId('threshold-save'));
|
||||
|
||||
await waitFor(() => expect(update).toHaveBeenCalledTimes(1));
|
||||
expect(update.mock.calls[0][0].dead_jobs_max).toBe(250);
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
const tabs = [
|
||||
{ href: '/admin', label: 'Overview' },
|
||||
{ href: '/admin/health', label: 'Health' },
|
||||
{ href: '/admin/users', label: 'Users' },
|
||||
{ href: '/admin/mangas', label: 'Mangas' },
|
||||
{ href: '/admin/crawler', label: 'Crawler' },
|
||||
|
||||
@@ -9,11 +9,13 @@
|
||||
analysisStatusStreamUrl,
|
||||
getCrawlerSettings,
|
||||
getAnalysisSettings,
|
||||
getHealth,
|
||||
type SystemStats,
|
||||
type OverviewStats,
|
||||
type CrawlerStatus,
|
||||
type AnalysisMetrics,
|
||||
type AnalysisEvent
|
||||
type AnalysisEvent,
|
||||
type HealthReport
|
||||
} from '$lib/api/admin';
|
||||
import { formatBytes as fmtBytes } from '$lib/upload-validation';
|
||||
import type { LayoutData } from './$types';
|
||||
@@ -44,6 +46,8 @@
|
||||
let liveCrawler = $state(false);
|
||||
let liveAnalysis = $state(false);
|
||||
|
||||
let health = $state<HealthReport | null>(null);
|
||||
|
||||
let sysTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let overviewTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let crawlerSource: EventSource | null = null;
|
||||
@@ -86,6 +90,17 @@
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
async function refreshHealth() {
|
||||
try {
|
||||
health = await getHealth();
|
||||
} catch {
|
||||
// keep the last good snapshot; retried next tick
|
||||
}
|
||||
}
|
||||
|
||||
const healthFailing = $derived(
|
||||
health ? health.checks.filter((c) => c.status !== 'ok').length : 0
|
||||
);
|
||||
async function loadSettings() {
|
||||
try {
|
||||
const c = await getCrawlerSettings();
|
||||
@@ -183,6 +198,7 @@
|
||||
refreshSys();
|
||||
refreshOverview();
|
||||
refreshMetrics();
|
||||
refreshHealth();
|
||||
loadSettings();
|
||||
getCrawlerStatus()
|
||||
.then((s) => (crawler = s))
|
||||
@@ -196,6 +212,7 @@
|
||||
overviewTimer = setInterval(() => {
|
||||
refreshOverview();
|
||||
refreshMetrics();
|
||||
refreshHealth();
|
||||
}, 60000);
|
||||
if (typeof document !== 'undefined') {
|
||||
document.addEventListener('visibilitychange', onVisibilityChange);
|
||||
@@ -237,6 +254,26 @@
|
||||
|
||||
<h1>Overview</h1>
|
||||
|
||||
{#if health}
|
||||
<a
|
||||
class="health-summary status-{health.status}"
|
||||
href="/admin/health"
|
||||
data-testid="overview-health"
|
||||
>
|
||||
<span class="glyph" aria-hidden="true"
|
||||
>{health.status === 'ok' ? '✓' : health.status === 'warn' ? '⚠' : '✗'}</span
|
||||
>
|
||||
<span class="hs-label">
|
||||
{#if health.status === 'ok'}
|
||||
All health checks passing
|
||||
{:else}
|
||||
{healthFailing} health check{healthFailing === 1 ? '' : 's'} need attention
|
||||
{/if}
|
||||
</span>
|
||||
<span class="hs-view">View →</span>
|
||||
</a>
|
||||
{/if}
|
||||
|
||||
{#if sys.alerts.length > 0 || extraAlerts.length > 0}
|
||||
<section class="alerts" data-testid="admin-alerts">
|
||||
{#each sys.alerts as a (a.message)}
|
||||
@@ -434,6 +471,44 @@
|
||||
h1 {
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
}
|
||||
.health-summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
margin-bottom: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.health-summary:hover {
|
||||
background: var(--surface-elevated);
|
||||
text-decoration: none;
|
||||
}
|
||||
.health-summary .glyph {
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.health-summary .hs-view {
|
||||
margin-left: auto;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.health-summary.status-ok .glyph {
|
||||
color: var(--success, #16a34a);
|
||||
}
|
||||
.health-summary.status-warn {
|
||||
border-left: 4px solid var(--warning, #d97706);
|
||||
}
|
||||
.health-summary.status-warn .glyph {
|
||||
color: var(--warning, #d97706);
|
||||
}
|
||||
.health-summary.status-critical {
|
||||
border-left: 4px solid var(--danger, #dc2626);
|
||||
}
|
||||
.health-summary.status-critical .glyph {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
.alerts {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
26
frontend/src/routes/admin/health/+page.svelte
Normal file
26
frontend/src/routes/admin/health/+page.svelte
Normal file
@@ -0,0 +1,26 @@
|
||||
<script lang="ts">
|
||||
import HealthChecks from '$lib/components/admin/HealthChecks.svelte';
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Health · Admin</title></svelte:head>
|
||||
|
||||
<h1>Health</h1>
|
||||
<p class="lead">
|
||||
Operational checks rolled up from system sensors, the job queue, recent
|
||||
failure rates and the crawler daemon. Thresholds are editable below and
|
||||
apply immediately.
|
||||
</p>
|
||||
|
||||
<HealthChecks />
|
||||
|
||||
<style>
|
||||
h1 {
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
.lead {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
margin-bottom: var(--space-4);
|
||||
max-width: 44rem;
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user