feat(admin): observability — job history, live "now analyzing", durations & metrics
Crawler + analysis admin dashboards gain a Live / History / Metrics segmented view. History: searchable, filterable, paginated job log per subsystem (crawler_jobs across all states/kinds; page_analysis terminal outcomes), with inline dead-job requeue and a Duration column. Live: enrich analysis SSE events with manga title + chapter number and add a sticky "Now analyzing" banner that jumps to and highlights the page. Metrics: new durable crawl_metrics table (migration 0028) timing every crawl op (manga list walk, manga detail, cover, whole chapter; per-page derived from chapter) plus page_analysis.duration_ms for analysis. New endpoints serve per-type average durations + success rates and a recent-ops log; a cron reaper (CRAWL_METRICS_RETENTION_DAYS) bounds growth. Tested: repo + API integration tests, vitest for the API client and fmtDuration, and Playwright for the History/Metrics tabs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
|
||||
assert_eq!(payload["requeued"], 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable list over all states/kinds.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a chapter-content job in a given state and return its (manga, chapter).
|
||||
async fn seed_history_chapter_job(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
number: i32,
|
||||
state: &str,
|
||||
) -> (Uuid, Uuid) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
/// Seed a `done` analyze_page job pointing at a real page so the history
|
||||
/// row resolves manga/chapter/page-number through the page breadcrumb.
|
||||
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
let page_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'k', 'image/png')",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(chapter_id)
|
||||
.bind(page_number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
|
||||
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
|
||||
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
|
||||
seed_history_analyze_job(&pool, "One Piece", 7).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: all four jobs.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Filter by state.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?state=done",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
// Naruto (sync_chapter done) + One Piece (analyze_page done).
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
|
||||
// Filter by kind.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=analyze_page",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "One Piece");
|
||||
assert_eq!(body["items"][0]["page_number"], 7);
|
||||
assert_eq!(body["items"][0]["chapter_number"], 3);
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?search=Bleach",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Bleach");
|
||||
assert_eq!(body["items"][0]["state"], "running");
|
||||
|
||||
// Pagination: limit clamps the page slice but total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operation metrics: durations, averages, recent-ops log.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn record_metric(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
) {
|
||||
mangalord::repo::crawl_metrics::record(
|
||||
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
|
||||
// Two chapter ops (one failed) + a cover op.
|
||||
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
|
||||
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
|
||||
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Summary: per-type averages + success split.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let summary = body["summary"].as_array().unwrap();
|
||||
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
|
||||
assert_eq!(chapter["n"], 2);
|
||||
assert_eq!(chapter["ok"], 1);
|
||||
assert_eq!(chapter["failed"], 1);
|
||||
assert_eq!(chapter["avg_ms"], 7000.0);
|
||||
|
||||
// Recent-ops log + outcome filter.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["op"], "chapter");
|
||||
assert_eq!(body["items"][0]["outcome"], "failed");
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_includes_chapter_duration(pool: PgPool) {
|
||||
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
|
||||
// A chapter metric for that chapter → history row should surface its duration.
|
||||
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 6100);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
|
||||
Reference in New Issue
Block a user