feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)
This commit was merged in pull request #6.
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.82.0"
|
||||
version = "0.84.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.82.0"
|
||||
version = "0.84.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
39
backend/migrations/0028_operation_metrics.sql
Normal file
39
backend/migrations/0028_operation_metrics.sql
Normal file
@@ -0,0 +1,39 @@
|
||||
-- Durable timing log for crawler operations + a duration column for page
|
||||
-- analysis. The dashboard's job history shows *what* ran; this records *how
|
||||
-- long it took* so operators can see per-operation durations and averages
|
||||
-- broken down by type/granularity (manga list walk, manga detail, cover,
|
||||
-- whole chapter — per-page crawl timing is derived from the chapter row's
|
||||
-- `items` count rather than stored per image).
|
||||
--
|
||||
-- This table is durable on purpose: it outlives the `crawler_jobs` done-job
|
||||
-- reaper so averages stay meaningful, and it captures the INLINE operations
|
||||
-- (list walk, manga detail, cover) that are not queue jobs at all. Volume is
|
||||
-- low (one row per manga / chapter / pass, not per image), so a generous
|
||||
-- retention reaper is hygiene rather than a necessity.
|
||||
CREATE TABLE crawl_metrics (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
-- Operation granularity. analyze_page is intentionally absent — analysis
|
||||
-- duration lives on page_analysis.duration_ms (one row per page already).
|
||||
op text NOT NULL
|
||||
CHECK (op IN ('manga_list', 'manga_detail', 'manga_cover', 'chapter')),
|
||||
-- Best-effort target context for labeling/drill. SET NULL so deleting a
|
||||
-- manga/chapter doesn't erase its historical timings.
|
||||
manga_id uuid REFERENCES mangas(id) ON DELETE SET NULL,
|
||||
chapter_id uuid REFERENCES chapters(id) ON DELETE SET NULL,
|
||||
outcome text NOT NULL CHECK (outcome IN ('ok', 'failed')),
|
||||
duration_ms bigint NOT NULL,
|
||||
-- Unit count for the op: chapter = pages stored; manga_list = mangas
|
||||
-- discovered. Drives the derived "per page" average. NULL when N/A.
|
||||
items integer,
|
||||
error text,
|
||||
finished_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Per-type averages + the recent-ops log both filter by op and order by time.
|
||||
CREATE INDEX crawl_metrics_op_time_idx ON crawl_metrics (op, finished_at DESC);
|
||||
-- The window filter (last 24h / 7d / 30d) and the retention reaper scan by time.
|
||||
CREATE INDEX crawl_metrics_time_idx ON crawl_metrics (finished_at);
|
||||
|
||||
-- Wall-clock the analysis worker spent on a page (vision dispatch). NULL for
|
||||
-- pre-existing rows and any page analyzed before this migration.
|
||||
ALTER TABLE page_analysis ADD COLUMN duration_ms bigint;
|
||||
@@ -216,16 +216,23 @@ impl WorkerContext {
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the page breadcrumb once so live events carry the
|
||||
// manga/chapter/number the dashboard keys on. A missing breadcrumb
|
||||
// (page deleted) just suppresses events — the dispatch still runs
|
||||
// and acks normally.
|
||||
let breadcrumb = repo::page::locate(&self.pool, page_id).await.ok().flatten();
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
// Resolve the labeled page breadcrumb once so live events carry the
|
||||
// manga/chapter/number AND human labels the dashboard's "now
|
||||
// analyzing" banner names. A missing breadcrumb (page deleted) just
|
||||
// suppresses events — the dispatch still runs and acks normally.
|
||||
let breadcrumb = repo::page::locate_labeled(&self.pool, page_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
|
||||
breadcrumb.clone()
|
||||
{
|
||||
self.events.publish(AnalysisEvent::Started {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
});
|
||||
}
|
||||
@@ -246,8 +253,10 @@ impl WorkerContext {
|
||||
})
|
||||
};
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(page_id)).catch_unwind();
|
||||
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
|
||||
let elapsed_ms = started.elapsed().as_millis() as i64;
|
||||
heartbeat.abort();
|
||||
|
||||
// Flatten timeout / panic / dispatch error into one Result + message.
|
||||
@@ -258,11 +267,27 @@ impl WorkerContext {
|
||||
Ok(Ok(Ok(()))) => Ok(()),
|
||||
};
|
||||
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
|
||||
breadcrumb
|
||||
{
|
||||
let event = if result.is_ok() {
|
||||
AnalysisEvent::Completed { page_id, manga_id, chapter_id, page_number }
|
||||
AnalysisEvent::Completed {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
}
|
||||
} else {
|
||||
AnalysisEvent::Failed { page_id, manga_id, chapter_id, page_number }
|
||||
AnalysisEvent::Failed {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
}
|
||||
};
|
||||
self.events.publish(event);
|
||||
}
|
||||
@@ -294,6 +319,11 @@ impl WorkerContext {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stamp how long the vision dispatch took. Best-effort and ordered
|
||||
// after the row is written (persist on success / mark_failed on
|
||||
// terminal failure); a transient failure with no row updates nothing.
|
||||
let _ = repo::page_analysis::record_duration(&self.pool, page_id, elapsed_ms).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,25 +25,33 @@ pub enum AnalysisEvent {
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
},
|
||||
/// The worker began analyzing a page.
|
||||
/// The worker began analyzing a page. Carries the manga title and
|
||||
/// chapter number (alongside the ids) so the dashboard's "now
|
||||
/// analyzing" banner can name the target without a lookup.
|
||||
Started {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker finished a page successfully.
|
||||
Completed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker failed a page (this attempt).
|
||||
Failed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
}
|
||||
@@ -76,3 +84,25 @@ impl Default for AnalysisEvents {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn started_serializes_kind_and_labels() {
|
||||
let ev = AnalysisEvent::Started {
|
||||
page_id: Uuid::nil(),
|
||||
manga_id: Uuid::nil(),
|
||||
manga_title: "Berserk".into(),
|
||||
chapter_id: Uuid::nil(),
|
||||
chapter_number: 12,
|
||||
page_number: 7,
|
||||
};
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
assert_eq!(v["kind"], "started");
|
||||
assert_eq!(v["manga_title"], "Berserk");
|
||||
assert_eq!(v["chapter_number"], 12);
|
||||
assert_eq!(v["page_number"], 7);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,8 @@ use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, MangaCoverage, PageAnalysisDetail, PageStatusItem,
|
||||
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, MangaCoverage, PageAnalysisDetail,
|
||||
PageStatusItem,
|
||||
};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
@@ -41,6 +42,8 @@ pub fn routes() -> Router<AppState> {
|
||||
.route("/admin/analysis/mangas/:id/chapters", get(coverage_chapters))
|
||||
.route("/admin/analysis/chapters/:id/pages", get(chapter_pages))
|
||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||
.route("/admin/analysis/history", get(list_history))
|
||||
.route("/admin/analysis/metrics", get(metrics))
|
||||
.route("/admin/analysis/status/stream", get(stream_status))
|
||||
}
|
||||
|
||||
@@ -149,6 +152,79 @@ struct ItemsResponse<T> {
|
||||
items: Vec<T>,
|
||||
}
|
||||
|
||||
/// Status/NSFW filters + pagination/search for the analysis history list.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct HistoryParams {
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
#[serde(default)]
|
||||
pub nsfw: bool,
|
||||
#[serde(default)]
|
||||
pub search: Option<String>,
|
||||
#[serde(default = "default_coverage_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
}
|
||||
|
||||
async fn list_history(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<HistoryParams>,
|
||||
) -> AppResult<Json<PagedResponse<AnalysisHistoryRow>>> {
|
||||
let limit = params.limit.clamp(1, 100);
|
||||
let offset = params.offset.max(0);
|
||||
let status = params
|
||||
.status
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let search = params
|
||||
.search
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let (items, total) = repo::page_analysis::list_history(
|
||||
&state.db,
|
||||
repo::page_analysis::AnalysisHistoryFilter {
|
||||
status,
|
||||
nsfw_only: params.nsfw,
|
||||
search,
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
/// `?days=` window for the metrics endpoints. `0` (or absent) = all time.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct MetricsParams {
|
||||
#[serde(default)]
|
||||
pub days: i64,
|
||||
}
|
||||
|
||||
/// Convert a `days` window param into a lower-bound timestamp. `<= 0` → all
|
||||
/// time (`None`); otherwise `now - days` (capped at a year).
|
||||
pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
if days <= 0 {
|
||||
None
|
||||
} else {
|
||||
Some(chrono::Utc::now() - chrono::Duration::days(days.min(365)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn metrics(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<MetricsParams>,
|
||||
) -> AppResult<Json<AnalysisMetrics>> {
|
||||
let since = window_since(params.days);
|
||||
let m = repo::page_analysis::analysis_metrics(&state.db, since).await?;
|
||||
Ok(Json(m))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
|
||||
63
backend/src/api/admin/crawler/history.rs
Normal file
63
backend/src/api/admin/crawler/history.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
//! GET /admin/crawler/history — unified, searchable, filterable job log.
|
||||
//!
|
||||
//! A pure DB-derived read over the `crawler_jobs` table across every state
|
||||
//! and kind. Drives the "History" tab on the crawler dashboard. Depth is
|
||||
//! bounded by the done-job reaper (recent window); `dead` jobs persist.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::{JobHistoryFilter, JobHistoryRow};
|
||||
|
||||
use super::default_limit;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new().route("/admin/crawler/history", get(list_history))
|
||||
}
|
||||
|
||||
/// State + kind filters and pagination/search for the history list.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct HistoryParams {
|
||||
#[serde(default)]
|
||||
state: Option<String>,
|
||||
#[serde(default)]
|
||||
kind: Option<String>,
|
||||
#[serde(default)]
|
||||
search: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_history(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<HistoryParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<JobHistoryRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let state_f = params.state.filter(|s| !s.trim().is_empty());
|
||||
let kind_f = params.kind.filter(|s| !s.trim().is_empty());
|
||||
let search_f = params.search.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) = repo::crawler::list_job_history(
|
||||
&state.db,
|
||||
JobHistoryFilter {
|
||||
state: state_f.as_deref(),
|
||||
kind: kind_f.as_deref(),
|
||||
search: search_f.as_deref(),
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
88
backend/src/api/admin/crawler/metrics.rs
Normal file
88
backend/src/api/admin/crawler/metrics.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
//! GET /admin/crawler/metrics — per-type average durations + success
|
||||
//! GET /admin/crawler/metrics/ops — paginated recent timed-operations log
|
||||
//!
|
||||
//! Pure DB reads over the durable `crawl_metrics` table. Drive the "Metrics"
|
||||
//! tab on the crawler dashboard. `days` windows the rows (`0`/absent = all).
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawl_metrics::OpFilter;
|
||||
|
||||
use super::default_limit;
|
||||
// Reuse the analysis handler's window helper so both endpoints clamp `days`
|
||||
// identically.
|
||||
use crate::api::admin::analysis::window_since;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/metrics", get(summary))
|
||||
.route("/admin/crawler/metrics/ops", get(list_ops))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct SummaryParams {
|
||||
#[serde(default)]
|
||||
days: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SummaryResponse {
|
||||
summary: Vec<OpSummary>,
|
||||
}
|
||||
|
||||
async fn summary(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SummaryParams>,
|
||||
) -> AppResult<Json<SummaryResponse>> {
|
||||
let since = window_since(params.days);
|
||||
let summary = repo::crawl_metrics::summary(&state.db, since).await?;
|
||||
Ok(Json(SummaryResponse { summary }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct OpsParams {
|
||||
#[serde(default)]
|
||||
op: Option<String>,
|
||||
#[serde(default)]
|
||||
outcome: Option<String>,
|
||||
#[serde(default)]
|
||||
days: i64,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_ops(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<OpsParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<OpRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let op = params.op.filter(|s| !s.trim().is_empty());
|
||||
let outcome = params.outcome.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) = repo::crawl_metrics::list_ops(
|
||||
&state.db,
|
||||
OpFilter {
|
||||
op: op.as_deref(),
|
||||
outcome: outcome.as_deref(),
|
||||
since: window_since(params.days),
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
@@ -16,6 +16,8 @@
|
||||
mod backlog;
|
||||
mod control;
|
||||
mod dead_jobs;
|
||||
mod history;
|
||||
mod metrics;
|
||||
mod status;
|
||||
|
||||
use axum::Router;
|
||||
@@ -29,6 +31,8 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(control::routes())
|
||||
.merge(dead_jobs::routes())
|
||||
.merge(backlog::routes())
|
||||
.merge(history::routes())
|
||||
.merge(metrics::routes())
|
||||
}
|
||||
|
||||
/// Default page size for the backlog list endpoints.
|
||||
|
||||
@@ -666,6 +666,7 @@ async fn spawn_crawler_daemon(
|
||||
daily_at: cfg.daily_at,
|
||||
tz: cfg.tz,
|
||||
retention_days: cfg.retention_days,
|
||||
metrics_retention_days: cfg.metrics_retention_days,
|
||||
session_expired,
|
||||
status: status.clone(),
|
||||
job_timeout: cfg.job_timeout,
|
||||
|
||||
@@ -321,6 +321,10 @@ pub struct CrawlerConfig {
|
||||
pub idle_timeout: Duration,
|
||||
pub chapter_workers: usize,
|
||||
pub retention_days: u32,
|
||||
/// Days to keep `crawl_metrics` timing rows before reaping. `0`
|
||||
/// disables the reaper. Defaults to 90 (volume is low — one row per
|
||||
/// manga/chapter/pass, not per image). `CRAWL_METRICS_RETENTION_DAYS`.
|
||||
pub metrics_retention_days: u32,
|
||||
pub start_url: Option<String>,
|
||||
pub rate_ms: u64,
|
||||
pub cdn_host: Option<String>,
|
||||
@@ -379,6 +383,7 @@ impl Default for CrawlerConfig {
|
||||
idle_timeout: Duration::from_secs(600),
|
||||
chapter_workers: 1,
|
||||
retention_days: 7,
|
||||
metrics_retention_days: 90,
|
||||
start_url: None,
|
||||
rate_ms: 1000,
|
||||
cdn_host: None,
|
||||
@@ -500,6 +505,7 @@ impl CrawlerConfig {
|
||||
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
|
||||
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
|
||||
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
|
||||
metrics_retention_days: env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32,
|
||||
start_url,
|
||||
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
|
||||
cdn_host,
|
||||
@@ -583,7 +589,7 @@ fn build_download_allowlist(
|
||||
allow
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, default: u64) -> u64 {
|
||||
pub(crate) fn env_u64(name: &str, default: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
|
||||
@@ -197,8 +197,69 @@ where
|
||||
/// On any failure the chapter stays at `page_count = 0` (no partial
|
||||
/// rows) and the blobs already written are deleted best-effort by
|
||||
/// [`cleanup_orphans`], so a retry starts clean.
|
||||
/// Timing wrapper over [`sync_chapter_content_inner`]: records one `chapter`
|
||||
/// row in `crawl_metrics` with the wall-clock and outcome. Best-effort — a
|
||||
/// metrics-write failure never fails the chapter sync. `Skipped` /
|
||||
/// `SessionExpired` perform no fetch, so they aren't timed.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn sync_chapter_content(
|
||||
browser: &chromiumoxide::Browser,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
http: &reqwest::Client,
|
||||
rate: &HostRateLimiters,
|
||||
chapter_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
source_url: &str,
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
progress: Option<&crate::crawler::status::StatusHandle>,
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
let started = std::time::Instant::now();
|
||||
let result = sync_chapter_content_inner(
|
||||
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
|
||||
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
|
||||
)
|
||||
.await;
|
||||
let duration_ms = started.elapsed().as_millis() as i64;
|
||||
match &result {
|
||||
Ok(SyncOutcome::Fetched { pages }) => {
|
||||
let _ = crate::repo::crawl_metrics::record(
|
||||
db,
|
||||
crate::repo::crawl_metrics::OP_CHAPTER,
|
||||
Some(manga_id),
|
||||
Some(chapter_id),
|
||||
"ok",
|
||||
duration_ms,
|
||||
Some(*pages as i32),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = crate::repo::crawl_metrics::record(
|
||||
db,
|
||||
crate::repo::crawl_metrics::OP_CHAPTER,
|
||||
Some(manga_id),
|
||||
Some(chapter_id),
|
||||
"failed",
|
||||
duration_ms,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// Skipped / SessionExpired: no fetch performed → not a timed op.
|
||||
Ok(_) => {}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn sync_chapter_content_inner(
|
||||
browser: &chromiumoxide::Browser,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
|
||||
@@ -86,6 +86,7 @@ pub struct DaemonConfig {
|
||||
pub daily_at: NaiveTime,
|
||||
pub tz: Tz,
|
||||
pub retention_days: u32,
|
||||
pub metrics_retention_days: u32,
|
||||
pub session_expired: Arc<AtomicBool>,
|
||||
/// Live status surface updated by the cron + workers.
|
||||
pub status: StatusHandle,
|
||||
@@ -139,6 +140,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
daily_at,
|
||||
tz,
|
||||
retention_days,
|
||||
metrics_retention_days,
|
||||
session_expired,
|
||||
status,
|
||||
job_timeout,
|
||||
@@ -152,6 +154,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
daily_at,
|
||||
tz,
|
||||
retention_days,
|
||||
metrics_retention_days,
|
||||
metadata,
|
||||
status: status.clone(),
|
||||
};
|
||||
@@ -190,6 +193,7 @@ struct CronContext {
|
||||
daily_at: NaiveTime,
|
||||
tz: Tz,
|
||||
retention_days: u32,
|
||||
metrics_retention_days: u32,
|
||||
metadata: Arc<dyn MetadataPass>,
|
||||
status: StatusHandle,
|
||||
}
|
||||
@@ -271,6 +275,7 @@ impl CronContext {
|
||||
let metadata = &self.metadata;
|
||||
let pool = &self.pool;
|
||||
let retention_days = self.retention_days;
|
||||
let metrics_retention_days = self.metrics_retention_days;
|
||||
let status = &self.status;
|
||||
let body = async move {
|
||||
match metadata.run().await {
|
||||
@@ -290,6 +295,10 @@ impl CronContext {
|
||||
Ok(n) => tracing::info!(reaped = n, "cron: done-job reaper finished"),
|
||||
Err(e) => tracing::error!(?e, "cron: done-job reaper failed"),
|
||||
}
|
||||
match crate::repo::crawl_metrics::reap(pool, metrics_retention_days).await {
|
||||
Ok(n) => tracing::info!(reaped = n, "cron: crawl-metrics reaper finished"),
|
||||
Err(e) => tracing::error!(?e, "cron: crawl-metrics reaper failed"),
|
||||
}
|
||||
if let Err(e) = write_last_tick(pool, Utc::now()).await {
|
||||
tracing::warn!(?e, "cron: persist last_metadata_tick_at failed");
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ pub async fn run_metadata_pass(
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<MetadataStats> {
|
||||
let pass_started = std::time::Instant::now();
|
||||
let lease = browser_manager
|
||||
.acquire()
|
||||
.await
|
||||
@@ -243,6 +244,7 @@ pub async fn run_metadata_pass(
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => {
|
||||
consecutive_failures = 0;
|
||||
@@ -255,6 +257,19 @@ pub async fn run_metadata_pass(
|
||||
error = ?e,
|
||||
"fetch_manga failed"
|
||||
);
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the
|
||||
// manga was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
stats.mangas_failed += 1;
|
||||
consecutive_failures += 1;
|
||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||
@@ -270,6 +285,9 @@ pub async fn run_metadata_pass(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a
|
||||
// prior count > 0 is overwhelmingly a chromium snapshot
|
||||
@@ -330,6 +348,18 @@ pub async fn run_metadata_pass(
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures below are non-fatal and don't roll this back —
|
||||
// metadata is the durable source of truth for the dedup.
|
||||
@@ -358,6 +388,7 @@ pub async fn run_metadata_pass(
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
@@ -370,6 +401,18 @@ pub async fn run_metadata_pass(
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
@@ -458,6 +501,21 @@ pub async fn run_metadata_pass(
|
||||
"metadata pass complete"
|
||||
);
|
||||
|
||||
// Record the whole-list-walk timing (best-effort). `items` = mangas
|
||||
// discovered this pass; outcome is `failed` only when the failure breaker
|
||||
// tripped (a degraded walk), else `ok`.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_LIST,
|
||||
None,
|
||||
None,
|
||||
if hit_failure_breaker { "failed" } else { "ok" },
|
||||
pass_started.elapsed().as_millis() as i64,
|
||||
Some(stats.discovered as i32),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(lease);
|
||||
Ok(stats)
|
||||
}
|
||||
@@ -694,6 +752,7 @@ pub async fn backfill_missing_covers(
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
@@ -706,6 +765,17 @@ pub async fn backfill_missing_covers(
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(entry.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.fetched += 1,
|
||||
Err(e) => {
|
||||
|
||||
43
backend/src/domain/crawl_metrics.rs
Normal file
43
backend/src/domain/crawl_metrics.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
//! Timing metrics for crawler operations (see `crawl_metrics` table, 0028).
|
||||
//!
|
||||
//! One [`OpRow`] per completed operation (manga list walk, manga detail,
|
||||
//! cover, whole chapter); [`OpSummary`] is the per-type average roll-up that
|
||||
//! drives the admin Metrics tab. Per-page crawl timing is derived on the
|
||||
//! client from the `chapter` summary's `avg_items` (pages/chapter), not
|
||||
//! stored per image. Analysis duration lives on `page_analysis`, not here.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Per-operation-type average roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpSummary {
|
||||
/// `manga_list` | `manga_detail` | `manga_cover` | `chapter`.
|
||||
pub op: String,
|
||||
/// Mean duration in milliseconds (`NULL`→absent only when `n = 0`).
|
||||
pub avg_ms: Option<f64>,
|
||||
/// Total operations in the window.
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
/// Mean `items` (pages/chapter, mangas/list-walk); `None` when unused.
|
||||
pub avg_items: Option<f64>,
|
||||
}
|
||||
|
||||
/// One timed operation, resolved to manga/chapter labels for the recent-ops log.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpRow {
|
||||
pub id: Uuid,
|
||||
pub op: String,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub chapter_number: Option<i32>,
|
||||
pub outcome: String,
|
||||
pub duration_ms: i64,
|
||||
pub items: Option<i32>,
|
||||
pub error: Option<String>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -4,6 +4,7 @@ pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
pub mod collection;
|
||||
pub mod crawl_metrics;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
pub mod page;
|
||||
@@ -25,6 +26,7 @@ pub use author::{Author, AuthorRef, AuthorWithCount};
|
||||
pub use bookmark::{Bookmark, BookmarkSummary};
|
||||
pub use chapter::Chapter;
|
||||
pub use collection::{Collection, CollectionPageItem, CollectionSummary};
|
||||
pub use crawl_metrics::{OpRow, OpSummary};
|
||||
pub use genre::{Genre, GenreRef};
|
||||
pub use manga::{Manga, MangaCard, MangaDetail};
|
||||
pub use page::Page;
|
||||
|
||||
@@ -157,6 +157,46 @@ pub struct PageStatusItem {
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
/// One row in the admin analysis-history table: a completed or failed
|
||||
/// analysis pass resolved to its page/chapter/manga context. Sourced from
|
||||
/// the persistent `page_analysis` table (not the reaped job queue), so it
|
||||
/// survives indefinitely. `status` is `done` | `failed`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct AnalysisHistoryRow {
|
||||
pub page_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_id: Uuid,
|
||||
pub chapter_number: i32,
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
pub status: String,
|
||||
pub is_nsfw: bool,
|
||||
pub model: Option<String>,
|
||||
pub error: Option<String>,
|
||||
pub analyzed_at: Option<DateTime<Utc>>,
|
||||
/// Wall-clock the worker spent on this page; `None` for rows analyzed
|
||||
/// before duration tracking landed.
|
||||
pub duration_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Per-model average analysis duration (admin Metrics tab).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct ModelDuration {
|
||||
pub model: Option<String>,
|
||||
pub avg_ms: Option<f64>,
|
||||
pub n: i64,
|
||||
}
|
||||
|
||||
/// Aggregate analysis timing/outcome roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct AnalysisMetrics {
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
pub avg_ms: Option<f64>,
|
||||
pub by_model: Vec<ModelDuration>,
|
||||
}
|
||||
|
||||
/// One OCR line in the page-detail view.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OcrLine {
|
||||
|
||||
158
backend/src/repo/crawl_metrics.rs
Normal file
158
backend/src/repo/crawl_metrics.rs
Normal file
@@ -0,0 +1,158 @@
|
||||
//! DB access for the `crawl_metrics` timing log (migration 0028).
|
||||
//!
|
||||
//! `record` is the best-effort write called from each crawl operation;
|
||||
//! `summary` and `list_ops` are the admin Metrics-tab reads. Mirrors the
|
||||
//! plain-fn + `query_as` style used across `repo`.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
|
||||
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
||||
/// don't drift from a free-typed literal.
|
||||
pub const OP_MANGA_LIST: &str = "manga_list";
|
||||
pub const OP_MANGA_DETAIL: &str = "manga_detail";
|
||||
pub const OP_MANGA_COVER: &str = "manga_cover";
|
||||
pub const OP_CHAPTER: &str = "chapter";
|
||||
|
||||
/// Insert one completed-operation timing row. Best-effort: callers log and
|
||||
/// continue on error so a metrics-write failure never fails the actual crawl
|
||||
/// work. `outcome` is `"ok"` or `"failed"`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn record(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
error: Option<&str>,
|
||||
) -> sqlx::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO crawl_metrics \
|
||||
(op, manga_id, chapter_id, outcome, duration_ms, items, error) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)",
|
||||
)
|
||||
.bind(op)
|
||||
.bind(manga_id)
|
||||
.bind(chapter_id)
|
||||
.bind(outcome)
|
||||
.bind(duration_ms)
|
||||
.bind(items)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Per-op average roll-up over an optional time window (`since = None` → all
|
||||
/// time). One row per `op` that has any metric in the window, with mean
|
||||
/// duration, counts, success split, and mean `items`.
|
||||
pub async fn summary(
|
||||
pool: &PgPool,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> sqlx::Result<Vec<OpSummary>> {
|
||||
sqlx::query_as::<_, OpSummary>(
|
||||
r#"
|
||||
SELECT
|
||||
op,
|
||||
AVG(duration_ms)::float8 AS avg_ms,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
|
||||
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
|
||||
AVG(items)::float8 AS avg_items
|
||||
FROM crawl_metrics
|
||||
WHERE ($1::timestamptz IS NULL OR finished_at >= $1)
|
||||
GROUP BY op
|
||||
ORDER BY op
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct OpFilter<'a> {
|
||||
pub op: Option<&'a str>,
|
||||
pub outcome: Option<&'a str>,
|
||||
pub since: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first recent-operations log resolved to manga/chapter
|
||||
/// labels. Returns the page slice plus the filtered total.
|
||||
pub async fn list_ops(
|
||||
pool: &PgPool,
|
||||
filter: OpFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<OpRow>, i64)> {
|
||||
let items = sqlx::query_as::<_, OpRow>(
|
||||
r#"
|
||||
SELECT
|
||||
cm.id,
|
||||
cm.op,
|
||||
cm.manga_id,
|
||||
m.title AS manga_title,
|
||||
cm.chapter_id,
|
||||
c.number AS chapter_number,
|
||||
cm.outcome,
|
||||
cm.duration_ms,
|
||||
cm.items,
|
||||
cm.error,
|
||||
cm.finished_at
|
||||
FROM crawl_metrics cm
|
||||
LEFT JOIN mangas m ON m.id = cm.manga_id
|
||||
LEFT JOIN chapters c ON c.id = cm.chapter_id
|
||||
WHERE ($1::text IS NULL OR cm.op = $1)
|
||||
AND ($2::text IS NULL OR cm.outcome = $2)
|
||||
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
|
||||
ORDER BY cm.finished_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.op)
|
||||
.bind(filter.outcome)
|
||||
.bind(filter.since)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawl_metrics cm
|
||||
WHERE ($1::text IS NULL OR cm.op = $1)
|
||||
AND ($2::text IS NULL OR cm.outcome = $2)
|
||||
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.op)
|
||||
.bind(filter.outcome)
|
||||
.bind(filter.since)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// Delete metric rows older than `retention_days`. `0` disables the reaper
|
||||
/// (returns 0 without touching the table). Mirrors `jobs::reap_done`.
|
||||
pub async fn reap(pool: &PgPool, retention_days: u32) -> sqlx::Result<u64> {
|
||||
if retention_days == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawl_metrics \
|
||||
WHERE finished_at < now() - ($1::bigint || ' days')::interval",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
@@ -774,6 +774,139 @@ pub async fn list_active_jobs(
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable view over the queue table.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A `crawler_jobs` row resolved to human context for the admin history
|
||||
/// table. Works across all job kinds: the target columns are best-effort
|
||||
/// `Option`s resolved through whichever payload reference the kind carries
|
||||
/// (`chapter_id`, `manga_id`, or `page_id` via the page breadcrumb), so a
|
||||
/// `sync_manga` job (which has neither yet) simply leaves them `None` and
|
||||
/// falls back to `source_key`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct JobHistoryRow {
|
||||
pub id: Uuid,
|
||||
/// `pending` | `running` | `done` | `dead` (the live queue states).
|
||||
pub state: String,
|
||||
/// `sync_manga` | `sync_chapter_list` | `sync_chapter_content` | `analyze_page`.
|
||||
pub kind: Option<String>,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub chapter_number: Option<i32>,
|
||||
/// Set only for `analyze_page` (resolved through the page breadcrumb).
|
||||
pub page_number: Option<i32>,
|
||||
/// Source-side key, the only target a `sync_manga` job carries.
|
||||
pub source_key: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
/// How long the job's work took, when a timing was recorded: the latest
|
||||
/// `chapter` crawl-metric for a chapter job, else the page's analysis
|
||||
/// duration for an `analyze_page` job. `None` for kinds we don't time or
|
||||
/// jobs not yet completed.
|
||||
pub duration_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Filters for [`list_job_history`]. All optional; `None` widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct JobHistoryFilter<'a> {
|
||||
/// Exact queue state (`done`, `dead`, `running`, `pending`).
|
||||
pub state: Option<&'a str>,
|
||||
/// Exact payload `kind`.
|
||||
pub kind: Option<&'a str>,
|
||||
/// Case-insensitive manga-title substring.
|
||||
pub search: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first view of the job queue across every state and
|
||||
/// kind — the searchable/filterable history surface. Joins each job to its
|
||||
/// manga/chapter/page context (best-effort) so the table can label rows.
|
||||
/// Returns the page slice plus the filtered total for pagination.
|
||||
///
|
||||
/// History depth is bounded by the done-job reaper (`reap_done`): completed
|
||||
/// jobs older than the retention window are gone. Terminal `dead` jobs
|
||||
/// persist until requeued.
|
||||
pub async fn list_job_history(
|
||||
pool: &PgPool,
|
||||
filter: JobHistoryFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<JobHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
// The same FROM/JOIN/WHERE drives both the page slice and the count, so
|
||||
// they stay in lockstep. `pg` resolves analyze_page → page → chapter;
|
||||
// `COALESCE` lets one set of joins serve every kind.
|
||||
let items: Vec<JobHistoryRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
cj.id,
|
||||
cj.state,
|
||||
cj.payload->>'kind' AS kind,
|
||||
COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid) AS manga_id,
|
||||
m.title AS manga_title,
|
||||
COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id) AS chapter_id,
|
||||
c.number AS chapter_number,
|
||||
pg.page_number AS page_number,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
cj.updated_at,
|
||||
COALESCE(cm.duration_ms, pa.duration_ms) AS duration_ms
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT duration_ms FROM crawl_metrics
|
||||
WHERE op = 'chapter'
|
||||
AND chapter_id = (cj.payload->>'chapter_id')::uuid
|
||||
ORDER BY finished_at DESC LIMIT 1
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
.bind(filter.kind)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
.bind(filter.kind)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// A manga whose cover is still missing (queued for cover fetch).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MissingCoverRow {
|
||||
|
||||
@@ -6,6 +6,7 @@ pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
pub mod collection;
|
||||
pub mod crawl_metrics;
|
||||
pub mod crawler;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
|
||||
@@ -60,6 +60,27 @@ pub async fn locate(
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Like [`locate`] but also resolves the manga title and chapter number, so
|
||||
/// live analysis events can carry human labels for the "now analyzing"
|
||||
/// banner without the dashboard having to look them up. Returns
|
||||
/// `(manga_id, manga_title, chapter_id, chapter_number, page_number)`.
|
||||
pub async fn locate_labeled(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<Option<(Uuid, String, Uuid, i32, i32)>> {
|
||||
let row: Option<(Uuid, String, Uuid, i32, i32)> = sqlx::query_as(
|
||||
"SELECT c.manga_id, m.title, p.chapter_id, c.number, p.page_number \
|
||||
FROM pages p \
|
||||
JOIN chapters c ON c.id = p.chapter_id \
|
||||
JOIN mangas m ON m.id = c.manga_id \
|
||||
WHERE p.id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
pub async fn list_for_chapter(pool: &PgPool, chapter_id: Uuid) -> AppResult<Vec<Page>> {
|
||||
let rows = sqlx::query_as::<_, Page>(
|
||||
r#"
|
||||
|
||||
@@ -16,9 +16,11 @@ use uuid::Uuid;
|
||||
|
||||
use crate::crawler::jobs::{self, JobPayload};
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, ContentWarning, MangaCoverage, OcrKind, OcrLine, PageAnalysis,
|
||||
PageAnalysisDetail, PageSearchItem, PageStatusItem, VisionAnalysis,
|
||||
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, ContentWarning, MangaCoverage,
|
||||
ModelDuration, OcrKind, OcrLine, PageAnalysis, PageAnalysisDetail, PageSearchItem,
|
||||
PageStatusItem, VisionAnalysis,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use crate::error::AppResult;
|
||||
|
||||
/// Filter set for [`page_search`]. `tags` are AND-ed (a page must carry
|
||||
@@ -212,6 +214,90 @@ pub async fn chapter_page_status(
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Filters for [`list_history`]. `None` widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct AnalysisHistoryFilter<'a> {
|
||||
/// Exact analysis status (`done` or `failed`).
|
||||
pub status: Option<&'a str>,
|
||||
/// Only NSFW-flagged pages.
|
||||
pub nsfw_only: bool,
|
||||
/// Case-insensitive manga-title substring.
|
||||
pub search: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first analysis history — every completed/failed page
|
||||
/// analysis resolved to its manga/chapter/page context. Reads the
|
||||
/// persistent `page_analysis` table, so unlike the crawler job history it
|
||||
/// is not bounded by job reaping. Returns the page slice plus the filtered
|
||||
/// total. `pending` rows are excluded — history is terminal outcomes only.
|
||||
pub async fn list_history(
|
||||
pool: &PgPool,
|
||||
filter: AnalysisHistoryFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<AnalysisHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, AnalysisHistoryRow>(
|
||||
r#"
|
||||
SELECT
|
||||
pa.page_id,
|
||||
p.page_number,
|
||||
p.chapter_id,
|
||||
c.number AS chapter_number,
|
||||
c.manga_id,
|
||||
m.title AS manga_title,
|
||||
pa.status,
|
||||
pa.is_nsfw,
|
||||
pa.model,
|
||||
pa.error,
|
||||
pa.analyzed_at,
|
||||
pa.duration_ms
|
||||
FROM page_analysis pa
|
||||
JOIN pages p ON p.id = pa.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
ORDER BY pa.analyzed_at DESC NULLS LAST, pa.page_id
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
.bind(filter.nsfw_only)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*)
|
||||
FROM page_analysis pa
|
||||
JOIN pages p ON p.id = pa.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
.bind(filter.nsfw_only)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// Full analysis detail for one page. `None` when the page itself doesn't
|
||||
/// exist; an existing-but-unanalyzed page returns `status = "none"` with
|
||||
/// empty OCR/tags/warnings so the UI can show "not analyzed yet".
|
||||
@@ -331,6 +417,66 @@ pub async fn mark_failed(pool: &PgPool, page_id: Uuid, error: &str) -> AppResult
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record the wall-clock the worker spent on a page. Best-effort: updates the
|
||||
/// existing `page_analysis` row (written by `persist_analysis`/`mark_failed`);
|
||||
/// a transient failure with no row yet simply updates nothing.
|
||||
pub async fn record_duration(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
duration_ms: i64,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query("UPDATE page_analysis SET duration_ms = $2 WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.bind(duration_ms)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Aggregate analysis timing + outcome roll-up over an optional time window
|
||||
/// (`since = None` → all time). Mean duration ignores NULL `duration_ms`
|
||||
/// (pre-tracking rows); `by_model` breaks the mean out per model.
|
||||
pub async fn analysis_metrics(
|
||||
pool: &PgPool,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> AppResult<AnalysisMetrics> {
|
||||
let (n, ok, failed, avg_ms): (i64, i64, i64, Option<f64>) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE status = 'done') AS ok,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||
-- Mean over successful pages only, so it reconciles with the
|
||||
-- by-model breakdown below (which is done-only). Failed /
|
||||
-- timed-out durations would otherwise skew the headline up.
|
||||
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
|
||||
FROM page_analysis
|
||||
WHERE status <> 'pending'
|
||||
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
let by_model = sqlx::query_as::<_, ModelDuration>(
|
||||
r#"
|
||||
SELECT model, AVG(duration_ms)::float8 AS avg_ms, COUNT(*) AS n
|
||||
FROM page_analysis
|
||||
WHERE status = 'done'
|
||||
AND duration_ms IS NOT NULL
|
||||
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
|
||||
GROUP BY model
|
||||
ORDER BY n DESC
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
||||
}
|
||||
|
||||
/// Persist a completed analysis for a page, replacing any previous result.
|
||||
///
|
||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||
|
||||
@@ -192,6 +192,11 @@ impl CrawlerSettings {
|
||||
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||
retention_days: self.retention_days,
|
||||
// Not a runtime-editable setting; re-read the env default so a
|
||||
// settings reload keeps whatever CRAWL_METRICS_RETENTION_DAYS was
|
||||
// configured at boot.
|
||||
metrics_retention_days: crate::config::env_u64("CRAWL_METRICS_RETENTION_DAYS", 90)
|
||||
as u32,
|
||||
start_url,
|
||||
rate_ms: self.rate_ms,
|
||||
cdn_host,
|
||||
|
||||
@@ -158,6 +158,16 @@ async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
|
||||
|
||||
// The worker stamps how long the dispatch took onto the existing row
|
||||
// (recorded after ack; shutdown has awaited the in-flight job).
|
||||
let dur: Option<i64> =
|
||||
sqlx::query_scalar("SELECT duration_ms FROM page_analysis WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(dur.is_some(), "worker should record analysis duration_ms");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -601,3 +601,255 @@ async fn coverage_requires_admin(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis history: terminal-outcome log, searchable + filterable.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga → chapter → one page, then write a terminal `page_analysis`
|
||||
/// row for that page. Returns the page id.
|
||||
async fn seed_analyzed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
is_nsfw: bool,
|
||||
error: Option<&str>,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, is_nsfw, model, error, analyzed_at) \
|
||||
VALUES ($1, $2, $3, 'vis', $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(is_nsfw)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_analyzed_page(&pool, "Alpha", "done", false, None).await;
|
||||
seed_analyzed_page(&pool, "Beta", "failed", false, Some("boom")).await;
|
||||
seed_analyzed_page(&pool, "Gamma", "done", true, None).await;
|
||||
// A still-pending row must NOT appear in history (terminal outcomes only).
|
||||
let pending_page = seed_analyzed_page(&pool, "Delta", "pending", false, None).await;
|
||||
let _ = pending_page;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: the three terminal rows, pending excluded.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
|
||||
// Filter by status=failed.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?status=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Beta");
|
||||
assert_eq!(body["items"][0]["error"], "boom");
|
||||
|
||||
// NSFW-only.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?nsfw=true",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Gamma");
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?search=Alpha",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Alpha");
|
||||
assert_eq!(body["items"][0]["status"], "done");
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis metrics: durations + averages + by-model + success.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga→chapter→page + a terminal page_analysis row carrying a
|
||||
/// duration and model. Returns the page id.
|
||||
async fn seed_timed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
model: &str,
|
||||
duration_ms: i64,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, model, duration_ms, analyzed_at) \
|
||||
VALUES ($1, $2, $3, $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(model)
|
||||
.bind(duration_ms)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_metrics_over_http(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2000).await;
|
||||
seed_timed_page(&pool, "Beta", "done", "qwen", 4000).await;
|
||||
seed_timed_page(&pool, "Gamma", "done", "llava", 3000).await;
|
||||
seed_timed_page(&pool, "Delta", "failed", "qwen", 1000).await;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/metrics",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["n"], 4);
|
||||
assert_eq!(body["ok"], 3);
|
||||
assert_eq!(body["failed"], 1);
|
||||
// Headline mean is done-only (2000 + 4000 + 3000)/3 — the failed row's
|
||||
// 1000ms is excluded so it reconciles with the by-model breakdown.
|
||||
assert_eq!(body["avg_ms"], 3000.0);
|
||||
|
||||
// by_model: qwen has the done rows 2000 + 4000 → avg 3000, n 2
|
||||
// (the failed qwen row is excluded — done only).
|
||||
let by_model = body["by_model"].as_array().unwrap();
|
||||
let qwen = by_model.iter().find(|m| m["model"] == "qwen").unwrap();
|
||||
assert_eq!(qwen["n"], 2);
|
||||
assert_eq!(qwen["avg_ms"], 3000.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_carries_duration(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2500).await;
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 2500);
|
||||
}
|
||||
|
||||
@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
|
||||
assert_eq!(payload["requeued"], 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable list over all states/kinds.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a chapter-content job in a given state and return its (manga, chapter).
|
||||
async fn seed_history_chapter_job(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
number: i32,
|
||||
state: &str,
|
||||
) -> (Uuid, Uuid) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
/// Seed a `done` analyze_page job pointing at a real page so the history
|
||||
/// row resolves manga/chapter/page-number through the page breadcrumb.
|
||||
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
let page_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'k', 'image/png')",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(chapter_id)
|
||||
.bind(page_number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
|
||||
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
|
||||
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
|
||||
seed_history_analyze_job(&pool, "One Piece", 7).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: all four jobs.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Filter by state.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?state=done",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
// Naruto (sync_chapter done) + One Piece (analyze_page done).
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
|
||||
// Filter by kind.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=analyze_page",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "One Piece");
|
||||
assert_eq!(body["items"][0]["page_number"], 7);
|
||||
assert_eq!(body["items"][0]["chapter_number"], 3);
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?search=Bleach",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Bleach");
|
||||
assert_eq!(body["items"][0]["state"], "running");
|
||||
|
||||
// Pagination: limit clamps the page slice but total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operation metrics: durations, averages, recent-ops log.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn record_metric(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
) {
|
||||
mangalord::repo::crawl_metrics::record(
|
||||
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
|
||||
// Two chapter ops (one failed) + a cover op.
|
||||
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
|
||||
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
|
||||
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Summary: per-type averages + success split.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let summary = body["summary"].as_array().unwrap();
|
||||
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
|
||||
assert_eq!(chapter["n"], 2);
|
||||
assert_eq!(chapter["ok"], 1);
|
||||
assert_eq!(chapter["failed"], 1);
|
||||
assert_eq!(chapter["avg_ms"], 7000.0);
|
||||
|
||||
// Recent-ops log + outcome filter.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["op"], "chapter");
|
||||
assert_eq!(body["items"][0]["outcome"], "failed");
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_includes_chapter_duration(pool: PgPool) {
|
||||
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
|
||||
// A chapter metric for that chapter → history row should surface its duration.
|
||||
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 6100);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
|
||||
140
backend/tests/crawl_metrics.rs
Normal file
140
backend/tests/crawl_metrics.rs
Normal file
@@ -0,0 +1,140 @@
|
||||
//! Repo-level integration tests for the crawl_metrics timing log.
|
||||
|
||||
use chrono::Utc;
|
||||
use mangalord::repo::crawl_metrics::{
|
||||
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
|
||||
|
||||
// Two chapter ops (one ok, one failed) + one cover op.
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(
|
||||
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter.n, 2);
|
||||
assert_eq!(chapter.ok, 1);
|
||||
assert_eq!(chapter.failed, 1);
|
||||
assert_eq!(chapter.avg_ms, Some(7000.0));
|
||||
assert_eq!(chapter.avg_items, Some(15.0));
|
||||
|
||||
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
|
||||
assert_eq!(cover.n, 1);
|
||||
assert_eq!(cover.avg_ms, Some(500.0));
|
||||
assert_eq!(cover.avg_items, None);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_window_excludes_old_rows(pool: PgPool) {
|
||||
let (m, _c) = seed_manga_chapter(&pool, "X").await;
|
||||
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// Backdate it two days.
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let since = Utc::now() - chrono::Duration::days(1);
|
||||
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
|
||||
assert!(windowed.is_empty(), "row older than the window must be excluded");
|
||||
|
||||
let all = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Unfiltered: both, newest-first, with labels resolved.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(rows.len(), 2);
|
||||
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
|
||||
assert_eq!(chapter_row.chapter_number, Some(12));
|
||||
assert_eq!(chapter_row.items, Some(20));
|
||||
|
||||
// Filter by op.
|
||||
let (rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(rows[0].outcome, "failed");
|
||||
assert_eq!(rows[0].error.as_deref(), Some("oops"));
|
||||
|
||||
// Filter by outcome.
|
||||
let (_rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 0 disables — nothing removed.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
|
||||
// 90-day retention removes the 100-day-old row.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
|
||||
let (_rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 0);
|
||||
}
|
||||
@@ -39,6 +39,7 @@ fn make_cfg(
|
||||
daily_at: far_future_daily_at(),
|
||||
tz: Tz::UTC,
|
||||
retention_days: 7,
|
||||
metrics_retention_days: 90,
|
||||
session_expired,
|
||||
status: mangalord::crawler::status::StatusHandle::new(workers),
|
||||
job_timeout: Duration::from_secs(60),
|
||||
|
||||
Reference in New Issue
Block a user