feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)
This commit was merged in pull request #6.
This commit is contained in:
@@ -158,6 +158,16 @@ async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
|
||||
|
||||
// The worker stamps how long the dispatch took onto the existing row
|
||||
// (recorded after ack; shutdown has awaited the in-flight job).
|
||||
let dur: Option<i64> =
|
||||
sqlx::query_scalar("SELECT duration_ms FROM page_analysis WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(dur.is_some(), "worker should record analysis duration_ms");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -601,3 +601,255 @@ async fn coverage_requires_admin(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis history: terminal-outcome log, searchable + filterable.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga → chapter → one page, then write a terminal `page_analysis`
|
||||
/// row for that page. Returns the page id.
|
||||
async fn seed_analyzed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
is_nsfw: bool,
|
||||
error: Option<&str>,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, is_nsfw, model, error, analyzed_at) \
|
||||
VALUES ($1, $2, $3, 'vis', $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(is_nsfw)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_analyzed_page(&pool, "Alpha", "done", false, None).await;
|
||||
seed_analyzed_page(&pool, "Beta", "failed", false, Some("boom")).await;
|
||||
seed_analyzed_page(&pool, "Gamma", "done", true, None).await;
|
||||
// A still-pending row must NOT appear in history (terminal outcomes only).
|
||||
let pending_page = seed_analyzed_page(&pool, "Delta", "pending", false, None).await;
|
||||
let _ = pending_page;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: the three terminal rows, pending excluded.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
|
||||
// Filter by status=failed.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?status=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Beta");
|
||||
assert_eq!(body["items"][0]["error"], "boom");
|
||||
|
||||
// NSFW-only.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?nsfw=true",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Gamma");
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?search=Alpha",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Alpha");
|
||||
assert_eq!(body["items"][0]["status"], "done");
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis metrics: durations + averages + by-model + success.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga→chapter→page + a terminal page_analysis row carrying a
|
||||
/// duration and model. Returns the page id.
|
||||
async fn seed_timed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
model: &str,
|
||||
duration_ms: i64,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, model, duration_ms, analyzed_at) \
|
||||
VALUES ($1, $2, $3, $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(model)
|
||||
.bind(duration_ms)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_metrics_over_http(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2000).await;
|
||||
seed_timed_page(&pool, "Beta", "done", "qwen", 4000).await;
|
||||
seed_timed_page(&pool, "Gamma", "done", "llava", 3000).await;
|
||||
seed_timed_page(&pool, "Delta", "failed", "qwen", 1000).await;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/metrics",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["n"], 4);
|
||||
assert_eq!(body["ok"], 3);
|
||||
assert_eq!(body["failed"], 1);
|
||||
// Headline mean is done-only (2000 + 4000 + 3000)/3 — the failed row's
|
||||
// 1000ms is excluded so it reconciles with the by-model breakdown.
|
||||
assert_eq!(body["avg_ms"], 3000.0);
|
||||
|
||||
// by_model: qwen has the done rows 2000 + 4000 → avg 3000, n 2
|
||||
// (the failed qwen row is excluded — done only).
|
||||
let by_model = body["by_model"].as_array().unwrap();
|
||||
let qwen = by_model.iter().find(|m| m["model"] == "qwen").unwrap();
|
||||
assert_eq!(qwen["n"], 2);
|
||||
assert_eq!(qwen["avg_ms"], 3000.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_carries_duration(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2500).await;
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 2500);
|
||||
}
|
||||
|
||||
@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
|
||||
assert_eq!(payload["requeued"], 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable list over all states/kinds.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a chapter-content job in a given state and return its (manga, chapter).
|
||||
async fn seed_history_chapter_job(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
number: i32,
|
||||
state: &str,
|
||||
) -> (Uuid, Uuid) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
/// Seed a `done` analyze_page job pointing at a real page so the history
|
||||
/// row resolves manga/chapter/page-number through the page breadcrumb.
|
||||
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
let page_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'k', 'image/png')",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(chapter_id)
|
||||
.bind(page_number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
|
||||
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
|
||||
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
|
||||
seed_history_analyze_job(&pool, "One Piece", 7).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: all four jobs.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Filter by state.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?state=done",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
// Naruto (sync_chapter done) + One Piece (analyze_page done).
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
|
||||
// Filter by kind.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=analyze_page",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "One Piece");
|
||||
assert_eq!(body["items"][0]["page_number"], 7);
|
||||
assert_eq!(body["items"][0]["chapter_number"], 3);
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?search=Bleach",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Bleach");
|
||||
assert_eq!(body["items"][0]["state"], "running");
|
||||
|
||||
// Pagination: limit clamps the page slice but total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operation metrics: durations, averages, recent-ops log.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn record_metric(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
) {
|
||||
mangalord::repo::crawl_metrics::record(
|
||||
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
|
||||
// Two chapter ops (one failed) + a cover op.
|
||||
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
|
||||
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
|
||||
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Summary: per-type averages + success split.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let summary = body["summary"].as_array().unwrap();
|
||||
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
|
||||
assert_eq!(chapter["n"], 2);
|
||||
assert_eq!(chapter["ok"], 1);
|
||||
assert_eq!(chapter["failed"], 1);
|
||||
assert_eq!(chapter["avg_ms"], 7000.0);
|
||||
|
||||
// Recent-ops log + outcome filter.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["op"], "chapter");
|
||||
assert_eq!(body["items"][0]["outcome"], "failed");
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_includes_chapter_duration(pool: PgPool) {
|
||||
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
|
||||
// A chapter metric for that chapter → history row should surface its duration.
|
||||
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 6100);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
|
||||
140
backend/tests/crawl_metrics.rs
Normal file
140
backend/tests/crawl_metrics.rs
Normal file
@@ -0,0 +1,140 @@
|
||||
//! Repo-level integration tests for the crawl_metrics timing log.
|
||||
|
||||
use chrono::Utc;
|
||||
use mangalord::repo::crawl_metrics::{
|
||||
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
|
||||
|
||||
// Two chapter ops (one ok, one failed) + one cover op.
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(
|
||||
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter.n, 2);
|
||||
assert_eq!(chapter.ok, 1);
|
||||
assert_eq!(chapter.failed, 1);
|
||||
assert_eq!(chapter.avg_ms, Some(7000.0));
|
||||
assert_eq!(chapter.avg_items, Some(15.0));
|
||||
|
||||
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
|
||||
assert_eq!(cover.n, 1);
|
||||
assert_eq!(cover.avg_ms, Some(500.0));
|
||||
assert_eq!(cover.avg_items, None);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_window_excludes_old_rows(pool: PgPool) {
|
||||
let (m, _c) = seed_manga_chapter(&pool, "X").await;
|
||||
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// Backdate it two days.
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let since = Utc::now() - chrono::Duration::days(1);
|
||||
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
|
||||
assert!(windowed.is_empty(), "row older than the window must be excluded");
|
||||
|
||||
let all = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Unfiltered: both, newest-first, with labels resolved.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(rows.len(), 2);
|
||||
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
|
||||
assert_eq!(chapter_row.chapter_number, Some(12));
|
||||
assert_eq!(chapter_row.items, Some(20));
|
||||
|
||||
// Filter by op.
|
||||
let (rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(rows[0].outcome, "failed");
|
||||
assert_eq!(rows[0].error.as_deref(), Some("oops"));
|
||||
|
||||
// Filter by outcome.
|
||||
let (_rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 0 disables — nothing removed.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
|
||||
// 90-day retention removes the 100-day-old row.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
|
||||
let (_rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 0);
|
||||
}
|
||||
@@ -39,6 +39,7 @@ fn make_cfg(
|
||||
daily_at: far_future_daily_at(),
|
||||
tz: Tz::UTC,
|
||||
retention_days: 7,
|
||||
metrics_retention_days: 90,
|
||||
session_expired,
|
||||
status: mangalord::crawler::status::StatusHandle::new(workers),
|
||||
job_timeout: Duration::from_secs(60),
|
||||
|
||||
Reference in New Issue
Block a user