feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)
Some checks failed
deploy / test-backend (push) Failing after 19m59s
deploy / test-frontend (push) Successful in 9m54s
deploy / build-and-push (push) Has been skipped
deploy / deploy (push) Has been skipped

This commit was merged in pull request #6.
This commit is contained in:
2026-06-16 12:21:13 +00:00
parent 790549636f
commit d51ab2a049
41 changed files with 3655 additions and 21 deletions

View File

@@ -158,6 +158,16 @@ async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
handle.shutdown().await;
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
// The worker stamps how long the dispatch took onto the existing row
// (recorded after ack; shutdown has awaited the in-flight job).
let dur: Option<i64> =
sqlx::query_scalar("SELECT duration_ms FROM page_analysis WHERE page_id = $1")
.bind(page_id)
.fetch_one(&pool)
.await
.unwrap();
assert!(dur.is_some(), "worker should record analysis duration_ms");
}
#[sqlx::test(migrations = "./migrations")]

View File

@@ -601,3 +601,255 @@ async fn coverage_requires_admin(pool: PgPool) {
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Analysis history: terminal-outcome log, searchable + filterable.
// ---------------------------------------------------------------------------
/// Seed a manga → chapter → one page, then write a terminal `page_analysis`
/// row for that page. Returns the page id.
async fn seed_analyzed_page(
pool: &PgPool,
title: &str,
status: &str,
is_nsfw: bool,
error: Option<&str>,
) -> uuid::Uuid {
let manga_id: uuid::Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
let page_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO page_analysis (page_id, status, is_nsfw, model, error, analyzed_at) \
VALUES ($1, $2, $3, 'vis', $4, now())",
)
.bind(page_id)
.bind(status)
.bind(is_nsfw)
.bind(error)
.execute(pool)
.await
.unwrap();
page_id
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_lists_filters_and_paginates(pool: PgPool) {
seed_analyzed_page(&pool, "Alpha", "done", false, None).await;
seed_analyzed_page(&pool, "Beta", "failed", false, Some("boom")).await;
seed_analyzed_page(&pool, "Gamma", "done", true, None).await;
// A still-pending row must NOT appear in history (terminal outcomes only).
let pending_page = seed_analyzed_page(&pool, "Delta", "pending", false, None).await;
let _ = pending_page;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Unfiltered: the three terminal rows, pending excluded.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 3);
// Filter by status=failed.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?status=failed",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Beta");
assert_eq!(body["items"][0]["error"], "boom");
// NSFW-only.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?nsfw=true",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Gamma");
// Search by manga title.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?search=Alpha",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Alpha");
assert_eq!(body["items"][0]["status"], "done");
// Pagination: limit clamps the slice; total reflects the full set.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?limit=1",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["items"].as_array().unwrap().len(), 1);
assert_eq!(body["page"]["total"], 3);
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_requires_admin(pool: PgPool) {
let h = common::harness(pool.clone());
let (_u, cookie) = common::register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Analysis metrics: durations + averages + by-model + success.
// ---------------------------------------------------------------------------
/// Seed a manga→chapter→page + a terminal page_analysis row carrying a
/// duration and model. Returns the page id.
async fn seed_timed_page(
pool: &PgPool,
title: &str,
status: &str,
model: &str,
duration_ms: i64,
) -> uuid::Uuid {
let manga_id: uuid::Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
let page_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO page_analysis (page_id, status, model, duration_ms, analyzed_at) \
VALUES ($1, $2, $3, $4, now())",
)
.bind(page_id)
.bind(status)
.bind(model)
.bind(duration_ms)
.execute(pool)
.await
.unwrap();
page_id
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_metrics_over_http(pool: PgPool) {
seed_timed_page(&pool, "Alpha", "done", "qwen", 2000).await;
seed_timed_page(&pool, "Beta", "done", "qwen", 4000).await;
seed_timed_page(&pool, "Gamma", "done", "llava", 3000).await;
seed_timed_page(&pool, "Delta", "failed", "qwen", 1000).await;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/metrics",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = common::body_json(resp).await;
assert_eq!(body["n"], 4);
assert_eq!(body["ok"], 3);
assert_eq!(body["failed"], 1);
// Headline mean is done-only (2000 + 4000 + 3000)/3 — the failed row's
// 1000ms is excluded so it reconciles with the by-model breakdown.
assert_eq!(body["avg_ms"], 3000.0);
// by_model: qwen has the done rows 2000 + 4000 → avg 3000, n 2
// (the failed qwen row is excluded — done only).
let by_model = body["by_model"].as_array().unwrap();
let qwen = by_model.iter().find(|m| m["model"] == "qwen").unwrap();
assert_eq!(qwen["n"], 2);
assert_eq!(qwen["avg_ms"], 3000.0);
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_carries_duration(pool: PgPool) {
seed_timed_page(&pool, "Alpha", "done", "qwen", 2500).await;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["items"][0]["duration_ms"], 2500);
}

View File

@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
assert_eq!(payload["requeued"], 1);
}
// ---------------------------------------------------------------------------
// Job history: unified, searchable, filterable list over all states/kinds.
// ---------------------------------------------------------------------------
/// Seed a chapter-content job in a given state and return its (manga, chapter).
async fn seed_history_chapter_job(
pool: &PgPool,
title: &str,
number: i32,
state: &str,
) -> (Uuid, Uuid) {
let manga_id = Uuid::new_v4();
let chapter_id = Uuid::new_v4();
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
.bind(manga_id)
.bind(title)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
.bind(chapter_id)
.bind(manga_id)
.bind(number)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
.bind(Uuid::new_v4())
.bind(json!({
"kind": "sync_chapter_content",
"source_id": "target",
"chapter_id": chapter_id,
"source_chapter_key": "k",
}))
.bind(state)
.execute(pool)
.await
.unwrap();
(manga_id, chapter_id)
}
/// Seed a `done` analyze_page job pointing at a real page so the history
/// row resolves manga/chapter/page-number through the page breadcrumb.
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
let manga_id = Uuid::new_v4();
let chapter_id = Uuid::new_v4();
let page_id = Uuid::new_v4();
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
.bind(manga_id)
.bind(title)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
.bind(chapter_id)
.bind(manga_id)
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
VALUES ($1, $2, $3, 'k', 'image/png')",
)
.bind(page_id)
.bind(chapter_id)
.bind(page_number)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
.bind(Uuid::new_v4())
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
.execute(pool)
.await
.unwrap();
}
#[sqlx::test(migrations = "./migrations")]
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
seed_history_analyze_job(&pool, "One Piece", 7).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Unfiltered: all four jobs.
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 4);
// Filter by state.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?state=done",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
// Naruto (sync_chapter done) + One Piece (analyze_page done).
assert_eq!(body["page"]["total"], 2);
// Filter by kind.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?kind=analyze_page",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "One Piece");
assert_eq!(body["items"][0]["page_number"], 7);
assert_eq!(body["items"][0]["chapter_number"], 3);
// Search by manga title.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?search=Bleach",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Bleach");
assert_eq!(body["items"][0]["state"], "running");
// Pagination: limit clamps the page slice but total reflects the full set.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?limit=1",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["items"].as_array().unwrap().len(), 1);
assert_eq!(body["page"]["total"], 4);
// Admin-gated.
let (_u, plain) = register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Operation metrics: durations, averages, recent-ops log.
// ---------------------------------------------------------------------------
async fn record_metric(
pool: &PgPool,
op: &str,
manga_id: Option<Uuid>,
chapter_id: Option<Uuid>,
outcome: &str,
duration_ms: i64,
items: Option<i32>,
) {
mangalord::repo::crawl_metrics::record(
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
)
.await
.unwrap();
}
#[sqlx::test(migrations = "./migrations")]
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
// Two chapter ops (one failed) + a cover op.
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Summary: per-type averages + success split.
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = body_json(resp).await;
let summary = body["summary"].as_array().unwrap();
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
assert_eq!(chapter["n"], 2);
assert_eq!(chapter["ok"], 1);
assert_eq!(chapter["failed"], 1);
assert_eq!(chapter["avg_ms"], 7000.0);
// Recent-ops log + outcome filter.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["op"], "chapter");
assert_eq!(body["items"][0]["outcome"], "failed");
// Admin-gated.
let (_u, plain) = register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
#[sqlx::test(migrations = "./migrations")]
async fn job_history_includes_chapter_duration(pool: PgPool) {
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
// A chapter metric for that chapter → history row should surface its duration.
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["items"][0]["duration_ms"], 6100);
}
#[sqlx::test(migrations = "./migrations")]
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
let job_id = seed_dead_job(&pool, "Bleach").await;

View File

@@ -0,0 +1,140 @@
//! Repo-level integration tests for the crawl_metrics timing log.
use chrono::Utc;
use mangalord::repo::crawl_metrics::{
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
};
use sqlx::PgPool;
use uuid::Uuid;
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
let manga_id: Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
(manga_id, chapter_id)
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
// Two chapter ops (one ok, one failed) + one cover op.
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
.await
.unwrap();
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
assert_eq!(chapter.n, 2);
assert_eq!(chapter.ok, 1);
assert_eq!(chapter.failed, 1);
assert_eq!(chapter.avg_ms, Some(7000.0));
assert_eq!(chapter.avg_items, Some(15.0));
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
assert_eq!(cover.n, 1);
assert_eq!(cover.avg_ms, Some(500.0));
assert_eq!(cover.avg_items, None);
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_window_excludes_old_rows(pool: PgPool) {
let (m, _c) = seed_manga_chapter(&pool, "X").await;
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
.await
.unwrap();
// Backdate it two days.
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
.execute(&pool)
.await
.unwrap();
let since = Utc::now() - chrono::Duration::days(1);
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
assert!(windowed.is_empty(), "row older than the window must be excluded");
let all = crawl_metrics::summary(&pool, None).await.unwrap();
assert_eq!(all.len(), 1);
}
#[sqlx::test(migrations = "./migrations")]
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
.await
.unwrap();
// Unfiltered: both, newest-first, with labels resolved.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 2);
assert_eq!(rows.len(), 2);
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
assert_eq!(chapter_row.chapter_number, Some(12));
assert_eq!(chapter_row.items, Some(20));
// Filter by op.
let (rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
assert_eq!(rows[0].outcome, "failed");
assert_eq!(rows[0].error.as_deref(), Some("oops"));
// Filter by outcome.
let (_rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
// Pagination: limit clamps the slice; total reflects the full set.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(total, 2);
}
#[sqlx::test(migrations = "./migrations")]
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
.await
.unwrap();
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
.execute(&pool)
.await
.unwrap();
// 0 disables — nothing removed.
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
// 90-day retention removes the 100-day-old row.
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
let (_rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 0);
}

View File

@@ -39,6 +39,7 @@ fn make_cfg(
daily_at: far_future_daily_at(),
tz: Tz::UTC,
retention_days: 7,
metrics_retention_days: 90,
session_expired,
status: mangalord::crawler::status::StatusHandle::new(workers),
job_timeout: Duration::from_secs(60),