feat(admin): operational health checks page with editable thresholds

New /admin/health rolls up signals already collected — disk/memory sensors,
dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata
cron freshness, and the live crawler session/browser flags — into a flat
checks list (ok/warn/critical) with an overall status and remediation links.
The overview gains a compact Health summary banner linking in.

GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via
try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists
the thresholds to app_settings (merged over compiled defaults, forward-compat
via serde(default)), validates ranges (400 on bad input), and audit-logs the
change in one transaction. New repo::crawler::last_metadata_tick_at getter and
a lightweight system::memory_percent_used (no CPU settle delay).

Pure status helpers (over_limit for gauges, exceeds for backlog counts so a
0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape,
gating, persistence+audit, and validation.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/tests/api_admin_health.rs

@@ -0,0 +1,207 @@
+//! Integration tests for the admin health endpoints: `GET /v1/admin/health`
+//! (shape, admin-gating, default checks present without a daemon) and
+//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row).
+
+mod common;
+
+use axum::http::StatusCode;
+use axum::Router;
+use serde_json::json;
+use sqlx::PgPool;
+use tower::ServiceExt;
+
+use common::{body_json, get_with_cookie, harness, register_user};
+
+async fn seed_admin(pool: &PgPool, app: &Router) -> String {
+    let (username, cookie) = register_user(app).await;
+    let u = mangalord::repo::user::find_by_username(pool, &username)
+        .await
+        .unwrap()
+        .unwrap();
+    mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
+        .await
+        .unwrap();
+    cookie
+}
+
+/// PUT helper (the common module exposes POST helpers; build a PUT here).
+fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request<axum::body::Body> {
+    axum::http::Request::builder()
+        .method("PUT")
+        .uri(uri)
+        .header(axum::http::header::COOKIE, cookie)
+        .header(axum::http::header::CONTENT_TYPE, "application/json")
+        .body(axum::body::Body::from(body.to_string()))
+        .unwrap()
+}
+
+fn valid_thresholds() -> serde_json::Value {
+    json!({
+        "disk_pct": 85.0,
+        "mem_pct": 80.0,
+        "dead_jobs_max": 50,
+        "crawl_fail_pct": 25.0,
+        "analysis_fail_pct": 30.0,
+        "cron_freshness_hours": 12,
+        "missing_covers_max": 200
+    })
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn requires_admin(pool: PgPool) {
+    let h = harness(pool.clone());
+    let (_u, cookie) = register_user(&h.app).await;
+    let resp = h
+        .app
+        .oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::FORBIDDEN);
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn report_has_status_checks_and_default_thresholds(pool: PgPool) {
+    let h = harness(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    let resp = h
+        .app
+        .oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::OK);
+    let body = body_json(resp).await;
+    // Overall status + a non-empty checks array (memory / dead_jobs /
+    // missing_covers / fail-rates are always present, daemon or not).
+    assert!(body["status"].is_string());
+    let checks = body["checks"].as_array().unwrap();
+    assert!(!checks.is_empty());
+    let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect();
+    assert!(ids.contains(&"memory"));
+    assert!(ids.contains(&"dead_jobs"));
+    assert!(ids.contains(&"crawl_fail_rate"));
+    // The harness wires no crawler daemon, so session/browser checks are absent.
+    assert!(!ids.contains(&"crawler_session"));
+    // Default thresholds echoed back.
+    assert_eq!(body["thresholds"]["dead_jobs_max"], 100);
+    assert_eq!(body["thresholds"]["disk_pct"], 90.0);
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) {
+    let h = harness(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    // Lower the threshold to 0, then a single dead job trips the check.
+    let mut t = valid_thresholds();
+    t["dead_jobs_max"] = json!(0);
+    let resp = h
+        .app
+        .clone()
+        .oneshot(put_json_with_cookie(
+            "/api/v1/admin/health/thresholds",
+            t,
+            &cookie,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::OK);
+
+    sqlx::query(
+        "INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')",
+    )
+    .bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" }))
+    .execute(&pool)
+    .await
+    .unwrap();
+
+    let resp = h
+        .app
+        .clone()
+        .oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
+        .await
+        .unwrap();
+    let body = body_json(resp).await;
+    let dead = body["checks"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|c| c["id"] == "dead_jobs")
+        .unwrap();
+    assert_eq!(dead["status"], "warn");
+    assert_eq!(body["status"], "warn"); // overall rolls up the worst
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn update_thresholds_persists_and_audits(pool: PgPool) {
+    let h = harness(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    let resp = h
+        .app
+        .clone()
+        .oneshot(put_json_with_cookie(
+            "/api/v1/admin/health/thresholds",
+            valid_thresholds(),
+            &cookie,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::OK);
+    let body = body_json(resp).await;
+    // Response re-evaluates against the new thresholds.
+    assert_eq!(body["thresholds"]["dead_jobs_max"], 50);
+
+    // Persisted to app_settings.
+    let stored: serde_json::Value =
+        sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'")
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+    assert_eq!(stored["mem_pct"], 80.0);
+
+    // Audit row written.
+    let action: String =
+        sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1")
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+    assert_eq!(action, "update_health_thresholds");
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) {
+    // Absent → None.
+    assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool)
+        .await
+        .unwrap()
+        .is_none());
+    // Stored in the daemon's `{"at": rfc3339}` shape → parsed back.
+    sqlx::query(
+        "INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)",
+    )
+    .bind(json!({ "at": "2026-06-18T09:30:00Z" }))
+    .execute(&pool)
+    .await
+    .unwrap();
+    let got = mangalord::repo::crawler::last_metadata_tick_at(&pool)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00");
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn update_thresholds_rejects_out_of_range(pool: PgPool) {
+    let h = harness(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    let mut bad = valid_thresholds();
+    bad["disk_pct"] = json!(150.0); // > 100
+    let resp = h
+        .app
+        .oneshot(put_json_with_cookie(
+            "/api/v1/admin/health/thresholds",
+            bad,
+            &cookie,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+}