New /admin/health rolls up signals already collected — disk/memory sensors, dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata cron freshness, and the live crawler session/browser flags — into a flat checks list (ok/warn/critical) with an overall status and remediation links. The overview gains a compact Health summary banner linking in. GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists the thresholds to app_settings (merged over compiled defaults, forward-compat via serde(default)), validates ranges (400 on bad input), and audit-logs the change in one transaction. New repo::crawler::last_metadata_tick_at getter and a lightweight system::memory_percent_used (no CPU settle delay). Pure status helpers (over_limit for gauges, exceeds for backlog counts so a 0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape, gating, persistence+audit, and validation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
208 lines
6.6 KiB
Rust
208 lines
6.6 KiB
Rust
//! Integration tests for the admin health endpoints: `GET /v1/admin/health`
|
|
//! (shape, admin-gating, default checks present without a daemon) and
|
|
//! `PUT /v1/admin/health/thresholds` (validation, persistence, audit row).
|
|
|
|
mod common;
|
|
|
|
use axum::http::StatusCode;
|
|
use axum::Router;
|
|
use serde_json::json;
|
|
use sqlx::PgPool;
|
|
use tower::ServiceExt;
|
|
|
|
use common::{body_json, get_with_cookie, harness, register_user};
|
|
|
|
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
|
let (username, cookie) = register_user(app).await;
|
|
let u = mangalord::repo::user::find_by_username(pool, &username)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
|
.await
|
|
.unwrap();
|
|
cookie
|
|
}
|
|
|
|
/// PUT helper (the common module exposes POST helpers; build a PUT here).
|
|
fn put_json_with_cookie(uri: &str, body: serde_json::Value, cookie: &str) -> axum::http::Request<axum::body::Body> {
|
|
axum::http::Request::builder()
|
|
.method("PUT")
|
|
.uri(uri)
|
|
.header(axum::http::header::COOKIE, cookie)
|
|
.header(axum::http::header::CONTENT_TYPE, "application/json")
|
|
.body(axum::body::Body::from(body.to_string()))
|
|
.unwrap()
|
|
}
|
|
|
|
fn valid_thresholds() -> serde_json::Value {
|
|
json!({
|
|
"disk_pct": 85.0,
|
|
"mem_pct": 80.0,
|
|
"dead_jobs_max": 50,
|
|
"crawl_fail_pct": 25.0,
|
|
"analysis_fail_pct": 30.0,
|
|
"cron_freshness_hours": 12,
|
|
"missing_covers_max": 200
|
|
})
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn requires_admin(pool: PgPool) {
|
|
let h = harness(pool.clone());
|
|
let (_u, cookie) = register_user(&h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn report_has_status_checks_and_default_thresholds(pool: PgPool) {
|
|
let h = harness(pool.clone());
|
|
let cookie = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let body = body_json(resp).await;
|
|
// Overall status + a non-empty checks array (memory / dead_jobs /
|
|
// missing_covers / fail-rates are always present, daemon or not).
|
|
assert!(body["status"].is_string());
|
|
let checks = body["checks"].as_array().unwrap();
|
|
assert!(!checks.is_empty());
|
|
let ids: Vec<&str> = checks.iter().map(|c| c["id"].as_str().unwrap()).collect();
|
|
assert!(ids.contains(&"memory"));
|
|
assert!(ids.contains(&"dead_jobs"));
|
|
assert!(ids.contains(&"crawl_fail_rate"));
|
|
// The harness wires no crawler daemon, so session/browser checks are absent.
|
|
assert!(!ids.contains(&"crawler_session"));
|
|
// Default thresholds echoed back.
|
|
assert_eq!(body["thresholds"]["dead_jobs_max"], 100);
|
|
assert_eq!(body["thresholds"]["disk_pct"], 90.0);
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn dead_jobs_check_warns_when_over_threshold(pool: PgPool) {
|
|
let h = harness(pool.clone());
|
|
let cookie = seed_admin(&pool, &h.app).await;
|
|
// Lower the threshold to 0, then a single dead job trips the check.
|
|
let mut t = valid_thresholds();
|
|
t["dead_jobs_max"] = json!(0);
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(put_json_with_cookie(
|
|
"/api/v1/admin/health/thresholds",
|
|
t,
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
|
|
sqlx::query(
|
|
"INSERT INTO crawler_jobs (payload, state) VALUES ($1, 'dead')",
|
|
)
|
|
.bind(json!({ "kind": "sync_manga", "source_id": "target", "source_manga_key": "k", "url": "http://x", "title": "T" }))
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(get_with_cookie("/api/v1/admin/health", &cookie))
|
|
.await
|
|
.unwrap();
|
|
let body = body_json(resp).await;
|
|
let dead = body["checks"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.find(|c| c["id"] == "dead_jobs")
|
|
.unwrap();
|
|
assert_eq!(dead["status"], "warn");
|
|
assert_eq!(body["status"], "warn"); // overall rolls up the worst
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn update_thresholds_persists_and_audits(pool: PgPool) {
|
|
let h = harness(pool.clone());
|
|
let cookie = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(put_json_with_cookie(
|
|
"/api/v1/admin/health/thresholds",
|
|
valid_thresholds(),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let body = body_json(resp).await;
|
|
// Response re-evaluates against the new thresholds.
|
|
assert_eq!(body["thresholds"]["dead_jobs_max"], 50);
|
|
|
|
// Persisted to app_settings.
|
|
let stored: serde_json::Value =
|
|
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = 'health_thresholds'")
|
|
.fetch_one(&pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(stored["mem_pct"], 80.0);
|
|
|
|
// Audit row written.
|
|
let action: String =
|
|
sqlx::query_scalar("SELECT action FROM admin_audit ORDER BY at DESC LIMIT 1")
|
|
.fetch_one(&pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(action, "update_health_thresholds");
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn last_metadata_tick_at_reads_crawler_state(pool: PgPool) {
|
|
// Absent → None.
|
|
assert!(mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
|
.await
|
|
.unwrap()
|
|
.is_none());
|
|
// Stored in the daemon's `{"at": rfc3339}` shape → parsed back.
|
|
sqlx::query(
|
|
"INSERT INTO crawler_state (key, value) VALUES ('last_metadata_tick_at', $1)",
|
|
)
|
|
.bind(json!({ "at": "2026-06-18T09:30:00Z" }))
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
let got = mangalord::repo::crawler::last_metadata_tick_at(&pool)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(got.to_rfc3339(), "2026-06-18T09:30:00+00:00");
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn update_thresholds_rejects_out_of_range(pool: PgPool) {
|
|
let h = harness(pool.clone());
|
|
let cookie = seed_admin(&pool, &h.app).await;
|
|
let mut bad = valid_thresholds();
|
|
bad["disk_pct"] = json!(150.0); // > 100
|
|
let resp = h
|
|
.app
|
|
.oneshot(put_json_with_cookie(
|
|
"/api/v1/admin/health/thresholds",
|
|
bad,
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
|
}
|