Files
Mangalord/backend/tests/crawl_metrics.rs
MechaCat02 5fa4442904
Some checks failed
deploy / test-frontend (pull_request) Waiting to run
deploy / build-and-push (pull_request) Blocked by required conditions
deploy / deploy (pull_request) Blocked by required conditions
deploy / test-backend (pull_request) Has been cancelled
feat(admin): observability — job history, live "now analyzing", durations & metrics
Crawler + analysis admin dashboards gain a Live / History / Metrics
segmented view.

History: searchable, filterable, paginated job log per subsystem
(crawler_jobs across all states/kinds; page_analysis terminal outcomes),
with inline dead-job requeue and a Duration column.

Live: enrich analysis SSE events with manga title + chapter number and add
a sticky "Now analyzing" banner that jumps to and highlights the page.

Metrics: new durable crawl_metrics table (migration 0028) timing every
crawl op (manga list walk, manga detail, cover, whole chapter; per-page
derived from chapter) plus page_analysis.duration_ms for analysis. New
endpoints serve per-type average durations + success rates and a recent-ops
log; a cron reaper (CRAWL_METRICS_RETENTION_DAYS) bounds growth.

Tested: repo + API integration tests, vitest for the API client and
fmtDuration, and Playwright for the History/Metrics tabs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-16 14:07:10 +02:00

141 lines
4.9 KiB
Rust

//! Repo-level integration tests for the crawl_metrics timing log.
use chrono::Utc;
use mangalord::repo::crawl_metrics::{
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
};
use sqlx::PgPool;
use uuid::Uuid;
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
let manga_id: Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
(manga_id, chapter_id)
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
// Two chapter ops (one ok, one failed) + one cover op.
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
.await
.unwrap();
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
assert_eq!(chapter.n, 2);
assert_eq!(chapter.ok, 1);
assert_eq!(chapter.failed, 1);
assert_eq!(chapter.avg_ms, Some(7000.0));
assert_eq!(chapter.avg_items, Some(15.0));
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
assert_eq!(cover.n, 1);
assert_eq!(cover.avg_ms, Some(500.0));
assert_eq!(cover.avg_items, None);
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_window_excludes_old_rows(pool: PgPool) {
let (m, _c) = seed_manga_chapter(&pool, "X").await;
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
.await
.unwrap();
// Backdate it two days.
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
.execute(&pool)
.await
.unwrap();
let since = Utc::now() - chrono::Duration::days(1);
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
assert!(windowed.is_empty(), "row older than the window must be excluded");
let all = crawl_metrics::summary(&pool, None).await.unwrap();
assert_eq!(all.len(), 1);
}
#[sqlx::test(migrations = "./migrations")]
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
.await
.unwrap();
// Unfiltered: both, newest-first, with labels resolved.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 2);
assert_eq!(rows.len(), 2);
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
assert_eq!(chapter_row.chapter_number, Some(12));
assert_eq!(chapter_row.items, Some(20));
// Filter by op.
let (rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
assert_eq!(rows[0].outcome, "failed");
assert_eq!(rows[0].error.as_deref(), Some("oops"));
// Filter by outcome.
let (_rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
// Pagination: limit clamps the slice; total reflects the full set.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(total, 2);
}
#[sqlx::test(migrations = "./migrations")]
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
.await
.unwrap();
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
.execute(&pool)
.await
.unwrap();
// 0 disables — nothing removed.
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
// 90-day retention removes the 100-day-old row.
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
let (_rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 0);
}