feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) (#6)
This commit was merged in pull request #6.
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.82.0"
|
||||
version = "0.84.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.82.0"
|
||||
version = "0.84.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
39
backend/migrations/0028_operation_metrics.sql
Normal file
39
backend/migrations/0028_operation_metrics.sql
Normal file
@@ -0,0 +1,39 @@
|
||||
-- Durable timing log for crawler operations + a duration column for page
|
||||
-- analysis. The dashboard's job history shows *what* ran; this records *how
|
||||
-- long it took* so operators can see per-operation durations and averages
|
||||
-- broken down by type/granularity (manga list walk, manga detail, cover,
|
||||
-- whole chapter — per-page crawl timing is derived from the chapter row's
|
||||
-- `items` count rather than stored per image).
|
||||
--
|
||||
-- This table is durable on purpose: it outlives the `crawler_jobs` done-job
|
||||
-- reaper so averages stay meaningful, and it captures the INLINE operations
|
||||
-- (list walk, manga detail, cover) that are not queue jobs at all. Volume is
|
||||
-- low (one row per manga / chapter / pass, not per image), so a generous
|
||||
-- retention reaper is hygiene rather than a necessity.
|
||||
CREATE TABLE crawl_metrics (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
-- Operation granularity. analyze_page is intentionally absent — analysis
|
||||
-- duration lives on page_analysis.duration_ms (one row per page already).
|
||||
op text NOT NULL
|
||||
CHECK (op IN ('manga_list', 'manga_detail', 'manga_cover', 'chapter')),
|
||||
-- Best-effort target context for labeling/drill. SET NULL so deleting a
|
||||
-- manga/chapter doesn't erase its historical timings.
|
||||
manga_id uuid REFERENCES mangas(id) ON DELETE SET NULL,
|
||||
chapter_id uuid REFERENCES chapters(id) ON DELETE SET NULL,
|
||||
outcome text NOT NULL CHECK (outcome IN ('ok', 'failed')),
|
||||
duration_ms bigint NOT NULL,
|
||||
-- Unit count for the op: chapter = pages stored; manga_list = mangas
|
||||
-- discovered. Drives the derived "per page" average. NULL when N/A.
|
||||
items integer,
|
||||
error text,
|
||||
finished_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Per-type averages + the recent-ops log both filter by op and order by time.
|
||||
CREATE INDEX crawl_metrics_op_time_idx ON crawl_metrics (op, finished_at DESC);
|
||||
-- The window filter (last 24h / 7d / 30d) and the retention reaper scan by time.
|
||||
CREATE INDEX crawl_metrics_time_idx ON crawl_metrics (finished_at);
|
||||
|
||||
-- Wall-clock the analysis worker spent on a page (vision dispatch). NULL for
|
||||
-- pre-existing rows and any page analyzed before this migration.
|
||||
ALTER TABLE page_analysis ADD COLUMN duration_ms bigint;
|
||||
@@ -216,16 +216,23 @@ impl WorkerContext {
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the page breadcrumb once so live events carry the
|
||||
// manga/chapter/number the dashboard keys on. A missing breadcrumb
|
||||
// (page deleted) just suppresses events — the dispatch still runs
|
||||
// and acks normally.
|
||||
let breadcrumb = repo::page::locate(&self.pool, page_id).await.ok().flatten();
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
// Resolve the labeled page breadcrumb once so live events carry the
|
||||
// manga/chapter/number AND human labels the dashboard's "now
|
||||
// analyzing" banner names. A missing breadcrumb (page deleted) just
|
||||
// suppresses events — the dispatch still runs and acks normally.
|
||||
let breadcrumb = repo::page::locate_labeled(&self.pool, page_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
|
||||
breadcrumb.clone()
|
||||
{
|
||||
self.events.publish(AnalysisEvent::Started {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
});
|
||||
}
|
||||
@@ -246,8 +253,10 @@ impl WorkerContext {
|
||||
})
|
||||
};
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(page_id)).catch_unwind();
|
||||
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
|
||||
let elapsed_ms = started.elapsed().as_millis() as i64;
|
||||
heartbeat.abort();
|
||||
|
||||
// Flatten timeout / panic / dispatch error into one Result + message.
|
||||
@@ -258,11 +267,27 @@ impl WorkerContext {
|
||||
Ok(Ok(Ok(()))) => Ok(()),
|
||||
};
|
||||
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
|
||||
breadcrumb
|
||||
{
|
||||
let event = if result.is_ok() {
|
||||
AnalysisEvent::Completed { page_id, manga_id, chapter_id, page_number }
|
||||
AnalysisEvent::Completed {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
}
|
||||
} else {
|
||||
AnalysisEvent::Failed { page_id, manga_id, chapter_id, page_number }
|
||||
AnalysisEvent::Failed {
|
||||
page_id,
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
page_number,
|
||||
}
|
||||
};
|
||||
self.events.publish(event);
|
||||
}
|
||||
@@ -294,6 +319,11 @@ impl WorkerContext {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stamp how long the vision dispatch took. Best-effort and ordered
|
||||
// after the row is written (persist on success / mark_failed on
|
||||
// terminal failure); a transient failure with no row updates nothing.
|
||||
let _ = repo::page_analysis::record_duration(&self.pool, page_id, elapsed_ms).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,25 +25,33 @@ pub enum AnalysisEvent {
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
},
|
||||
/// The worker began analyzing a page.
|
||||
/// The worker began analyzing a page. Carries the manga title and
|
||||
/// chapter number (alongside the ids) so the dashboard's "now
|
||||
/// analyzing" banner can name the target without a lookup.
|
||||
Started {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker finished a page successfully.
|
||||
Completed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker failed a page (this attempt).
|
||||
Failed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
manga_title: String,
|
||||
chapter_id: Uuid,
|
||||
chapter_number: i32,
|
||||
page_number: i32,
|
||||
},
|
||||
}
|
||||
@@ -76,3 +84,25 @@ impl Default for AnalysisEvents {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn started_serializes_kind_and_labels() {
|
||||
let ev = AnalysisEvent::Started {
|
||||
page_id: Uuid::nil(),
|
||||
manga_id: Uuid::nil(),
|
||||
manga_title: "Berserk".into(),
|
||||
chapter_id: Uuid::nil(),
|
||||
chapter_number: 12,
|
||||
page_number: 7,
|
||||
};
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
assert_eq!(v["kind"], "started");
|
||||
assert_eq!(v["manga_title"], "Berserk");
|
||||
assert_eq!(v["chapter_number"], 12);
|
||||
assert_eq!(v["page_number"], 7);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,8 @@ use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, MangaCoverage, PageAnalysisDetail, PageStatusItem,
|
||||
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, MangaCoverage, PageAnalysisDetail,
|
||||
PageStatusItem,
|
||||
};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
@@ -41,6 +42,8 @@ pub fn routes() -> Router<AppState> {
|
||||
.route("/admin/analysis/mangas/:id/chapters", get(coverage_chapters))
|
||||
.route("/admin/analysis/chapters/:id/pages", get(chapter_pages))
|
||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||
.route("/admin/analysis/history", get(list_history))
|
||||
.route("/admin/analysis/metrics", get(metrics))
|
||||
.route("/admin/analysis/status/stream", get(stream_status))
|
||||
}
|
||||
|
||||
@@ -149,6 +152,79 @@ struct ItemsResponse<T> {
|
||||
items: Vec<T>,
|
||||
}
|
||||
|
||||
/// Status/NSFW filters + pagination/search for the analysis history list.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct HistoryParams {
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
#[serde(default)]
|
||||
pub nsfw: bool,
|
||||
#[serde(default)]
|
||||
pub search: Option<String>,
|
||||
#[serde(default = "default_coverage_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
}
|
||||
|
||||
async fn list_history(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<HistoryParams>,
|
||||
) -> AppResult<Json<PagedResponse<AnalysisHistoryRow>>> {
|
||||
let limit = params.limit.clamp(1, 100);
|
||||
let offset = params.offset.max(0);
|
||||
let status = params
|
||||
.status
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let search = params
|
||||
.search
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let (items, total) = repo::page_analysis::list_history(
|
||||
&state.db,
|
||||
repo::page_analysis::AnalysisHistoryFilter {
|
||||
status,
|
||||
nsfw_only: params.nsfw,
|
||||
search,
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
/// `?days=` window for the metrics endpoints. `0` (or absent) = all time.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct MetricsParams {
|
||||
#[serde(default)]
|
||||
pub days: i64,
|
||||
}
|
||||
|
||||
/// Convert a `days` window param into a lower-bound timestamp. `<= 0` → all
|
||||
/// time (`None`); otherwise `now - days` (capped at a year).
|
||||
pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
if days <= 0 {
|
||||
None
|
||||
} else {
|
||||
Some(chrono::Utc::now() - chrono::Duration::days(days.min(365)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn metrics(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<MetricsParams>,
|
||||
) -> AppResult<Json<AnalysisMetrics>> {
|
||||
let since = window_since(params.days);
|
||||
let m = repo::page_analysis::analysis_metrics(&state.db, since).await?;
|
||||
Ok(Json(m))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
|
||||
63
backend/src/api/admin/crawler/history.rs
Normal file
63
backend/src/api/admin/crawler/history.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
//! GET /admin/crawler/history — unified, searchable, filterable job log.
|
||||
//!
|
||||
//! A pure DB-derived read over the `crawler_jobs` table across every state
|
||||
//! and kind. Drives the "History" tab on the crawler dashboard. Depth is
|
||||
//! bounded by the done-job reaper (recent window); `dead` jobs persist.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::{JobHistoryFilter, JobHistoryRow};
|
||||
|
||||
use super::default_limit;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new().route("/admin/crawler/history", get(list_history))
|
||||
}
|
||||
|
||||
/// State + kind filters and pagination/search for the history list.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct HistoryParams {
|
||||
#[serde(default)]
|
||||
state: Option<String>,
|
||||
#[serde(default)]
|
||||
kind: Option<String>,
|
||||
#[serde(default)]
|
||||
search: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_history(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<HistoryParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<JobHistoryRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let state_f = params.state.filter(|s| !s.trim().is_empty());
|
||||
let kind_f = params.kind.filter(|s| !s.trim().is_empty());
|
||||
let search_f = params.search.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) = repo::crawler::list_job_history(
|
||||
&state.db,
|
||||
JobHistoryFilter {
|
||||
state: state_f.as_deref(),
|
||||
kind: kind_f.as_deref(),
|
||||
search: search_f.as_deref(),
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
88
backend/src/api/admin/crawler/metrics.rs
Normal file
88
backend/src/api/admin/crawler/metrics.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
//! GET /admin/crawler/metrics — per-type average durations + success
|
||||
//! GET /admin/crawler/metrics/ops — paginated recent timed-operations log
|
||||
//!
|
||||
//! Pure DB reads over the durable `crawl_metrics` table. Drive the "Metrics"
|
||||
//! tab on the crawler dashboard. `days` windows the rows (`0`/absent = all).
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawl_metrics::OpFilter;
|
||||
|
||||
use super::default_limit;
|
||||
// Reuse the analysis handler's window helper so both endpoints clamp `days`
|
||||
// identically.
|
||||
use crate::api::admin::analysis::window_since;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/metrics", get(summary))
|
||||
.route("/admin/crawler/metrics/ops", get(list_ops))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct SummaryParams {
|
||||
#[serde(default)]
|
||||
days: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SummaryResponse {
|
||||
summary: Vec<OpSummary>,
|
||||
}
|
||||
|
||||
async fn summary(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SummaryParams>,
|
||||
) -> AppResult<Json<SummaryResponse>> {
|
||||
let since = window_since(params.days);
|
||||
let summary = repo::crawl_metrics::summary(&state.db, since).await?;
|
||||
Ok(Json(SummaryResponse { summary }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct OpsParams {
|
||||
#[serde(default)]
|
||||
op: Option<String>,
|
||||
#[serde(default)]
|
||||
outcome: Option<String>,
|
||||
#[serde(default)]
|
||||
days: i64,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_ops(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<OpsParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<OpRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let op = params.op.filter(|s| !s.trim().is_empty());
|
||||
let outcome = params.outcome.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) = repo::crawl_metrics::list_ops(
|
||||
&state.db,
|
||||
OpFilter {
|
||||
op: op.as_deref(),
|
||||
outcome: outcome.as_deref(),
|
||||
since: window_since(params.days),
|
||||
},
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
@@ -16,6 +16,8 @@
|
||||
mod backlog;
|
||||
mod control;
|
||||
mod dead_jobs;
|
||||
mod history;
|
||||
mod metrics;
|
||||
mod status;
|
||||
|
||||
use axum::Router;
|
||||
@@ -29,6 +31,8 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(control::routes())
|
||||
.merge(dead_jobs::routes())
|
||||
.merge(backlog::routes())
|
||||
.merge(history::routes())
|
||||
.merge(metrics::routes())
|
||||
}
|
||||
|
||||
/// Default page size for the backlog list endpoints.
|
||||
|
||||
@@ -666,6 +666,7 @@ async fn spawn_crawler_daemon(
|
||||
daily_at: cfg.daily_at,
|
||||
tz: cfg.tz,
|
||||
retention_days: cfg.retention_days,
|
||||
metrics_retention_days: cfg.metrics_retention_days,
|
||||
session_expired,
|
||||
status: status.clone(),
|
||||
job_timeout: cfg.job_timeout,
|
||||
|
||||
@@ -321,6 +321,10 @@ pub struct CrawlerConfig {
|
||||
pub idle_timeout: Duration,
|
||||
pub chapter_workers: usize,
|
||||
pub retention_days: u32,
|
||||
/// Days to keep `crawl_metrics` timing rows before reaping. `0`
|
||||
/// disables the reaper. Defaults to 90 (volume is low — one row per
|
||||
/// manga/chapter/pass, not per image). `CRAWL_METRICS_RETENTION_DAYS`.
|
||||
pub metrics_retention_days: u32,
|
||||
pub start_url: Option<String>,
|
||||
pub rate_ms: u64,
|
||||
pub cdn_host: Option<String>,
|
||||
@@ -379,6 +383,7 @@ impl Default for CrawlerConfig {
|
||||
idle_timeout: Duration::from_secs(600),
|
||||
chapter_workers: 1,
|
||||
retention_days: 7,
|
||||
metrics_retention_days: 90,
|
||||
start_url: None,
|
||||
rate_ms: 1000,
|
||||
cdn_host: None,
|
||||
@@ -500,6 +505,7 @@ impl CrawlerConfig {
|
||||
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
|
||||
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
|
||||
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
|
||||
metrics_retention_days: env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32,
|
||||
start_url,
|
||||
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
|
||||
cdn_host,
|
||||
@@ -583,7 +589,7 @@ fn build_download_allowlist(
|
||||
allow
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, default: u64) -> u64 {
|
||||
pub(crate) fn env_u64(name: &str, default: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
|
||||
@@ -197,8 +197,69 @@ where
|
||||
/// On any failure the chapter stays at `page_count = 0` (no partial
|
||||
/// rows) and the blobs already written are deleted best-effort by
|
||||
/// [`cleanup_orphans`], so a retry starts clean.
|
||||
/// Timing wrapper over [`sync_chapter_content_inner`]: records one `chapter`
|
||||
/// row in `crawl_metrics` with the wall-clock and outcome. Best-effort — a
|
||||
/// metrics-write failure never fails the chapter sync. `Skipped` /
|
||||
/// `SessionExpired` perform no fetch, so they aren't timed.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn sync_chapter_content(
|
||||
browser: &chromiumoxide::Browser,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
http: &reqwest::Client,
|
||||
rate: &HostRateLimiters,
|
||||
chapter_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
source_url: &str,
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
progress: Option<&crate::crawler::status::StatusHandle>,
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
let started = std::time::Instant::now();
|
||||
let result = sync_chapter_content_inner(
|
||||
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
|
||||
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
|
||||
)
|
||||
.await;
|
||||
let duration_ms = started.elapsed().as_millis() as i64;
|
||||
match &result {
|
||||
Ok(SyncOutcome::Fetched { pages }) => {
|
||||
let _ = crate::repo::crawl_metrics::record(
|
||||
db,
|
||||
crate::repo::crawl_metrics::OP_CHAPTER,
|
||||
Some(manga_id),
|
||||
Some(chapter_id),
|
||||
"ok",
|
||||
duration_ms,
|
||||
Some(*pages as i32),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = crate::repo::crawl_metrics::record(
|
||||
db,
|
||||
crate::repo::crawl_metrics::OP_CHAPTER,
|
||||
Some(manga_id),
|
||||
Some(chapter_id),
|
||||
"failed",
|
||||
duration_ms,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// Skipped / SessionExpired: no fetch performed → not a timed op.
|
||||
Ok(_) => {}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn sync_chapter_content_inner(
|
||||
browser: &chromiumoxide::Browser,
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
|
||||
@@ -86,6 +86,7 @@ pub struct DaemonConfig {
|
||||
pub daily_at: NaiveTime,
|
||||
pub tz: Tz,
|
||||
pub retention_days: u32,
|
||||
pub metrics_retention_days: u32,
|
||||
pub session_expired: Arc<AtomicBool>,
|
||||
/// Live status surface updated by the cron + workers.
|
||||
pub status: StatusHandle,
|
||||
@@ -139,6 +140,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
daily_at,
|
||||
tz,
|
||||
retention_days,
|
||||
metrics_retention_days,
|
||||
session_expired,
|
||||
status,
|
||||
job_timeout,
|
||||
@@ -152,6 +154,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
daily_at,
|
||||
tz,
|
||||
retention_days,
|
||||
metrics_retention_days,
|
||||
metadata,
|
||||
status: status.clone(),
|
||||
};
|
||||
@@ -190,6 +193,7 @@ struct CronContext {
|
||||
daily_at: NaiveTime,
|
||||
tz: Tz,
|
||||
retention_days: u32,
|
||||
metrics_retention_days: u32,
|
||||
metadata: Arc<dyn MetadataPass>,
|
||||
status: StatusHandle,
|
||||
}
|
||||
@@ -271,6 +275,7 @@ impl CronContext {
|
||||
let metadata = &self.metadata;
|
||||
let pool = &self.pool;
|
||||
let retention_days = self.retention_days;
|
||||
let metrics_retention_days = self.metrics_retention_days;
|
||||
let status = &self.status;
|
||||
let body = async move {
|
||||
match metadata.run().await {
|
||||
@@ -290,6 +295,10 @@ impl CronContext {
|
||||
Ok(n) => tracing::info!(reaped = n, "cron: done-job reaper finished"),
|
||||
Err(e) => tracing::error!(?e, "cron: done-job reaper failed"),
|
||||
}
|
||||
match crate::repo::crawl_metrics::reap(pool, metrics_retention_days).await {
|
||||
Ok(n) => tracing::info!(reaped = n, "cron: crawl-metrics reaper finished"),
|
||||
Err(e) => tracing::error!(?e, "cron: crawl-metrics reaper failed"),
|
||||
}
|
||||
if let Err(e) = write_last_tick(pool, Utc::now()).await {
|
||||
tracing::warn!(?e, "cron: persist last_metadata_tick_at failed");
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ pub async fn run_metadata_pass(
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<MetadataStats> {
|
||||
let pass_started = std::time::Instant::now();
|
||||
let lease = browser_manager
|
||||
.acquire()
|
||||
.await
|
||||
@@ -243,6 +244,7 @@ pub async fn run_metadata_pass(
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let detail_started = std::time::Instant::now();
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => {
|
||||
consecutive_failures = 0;
|
||||
@@ -255,6 +257,19 @@ pub async fn run_metadata_pass(
|
||||
error = ?e,
|
||||
"fetch_manga failed"
|
||||
);
|
||||
// Detail-fetch timing (failed). manga_id is unknown — the
|
||||
// manga was never upserted.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
None,
|
||||
None,
|
||||
"failed",
|
||||
detail_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
Some(&format!("{e:#}")),
|
||||
)
|
||||
.await;
|
||||
stats.mangas_failed += 1;
|
||||
consecutive_failures += 1;
|
||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||
@@ -270,6 +285,9 @@ pub async fn run_metadata_pass(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Detail fetch succeeded; stash its duration to record once the
|
||||
// manga_id is known (after upsert).
|
||||
let detail_ms = detail_started.elapsed().as_millis() as i64;
|
||||
|
||||
// Partial-render guard: an empty chapter list paired with a
|
||||
// prior count > 0 is overwhelmingly a chromium snapshot
|
||||
@@ -330,6 +348,18 @@ pub async fn run_metadata_pass(
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
// Detail-fetch timing (ok), now that we have the manga_id.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_DETAIL,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
"ok",
|
||||
detail_ms,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
// Record success in the dedup set. Cover and chapter-sync
|
||||
// failures below are non-fatal and don't roll this back —
|
||||
// metadata is the durable source of truth for the dedup.
|
||||
@@ -358,6 +388,7 @@ pub async fn run_metadata_pass(
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
@@ -370,6 +401,18 @@ pub async fn run_metadata_pass(
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let cover_ms = cover_started.elapsed().as_millis() as i64;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(upsert.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_ms,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
@@ -458,6 +501,21 @@ pub async fn run_metadata_pass(
|
||||
"metadata pass complete"
|
||||
);
|
||||
|
||||
// Record the whole-list-walk timing (best-effort). `items` = mangas
|
||||
// discovered this pass; outcome is `failed` only when the failure breaker
|
||||
// tripped (a degraded walk), else `ok`.
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_LIST,
|
||||
None,
|
||||
None,
|
||||
if hit_failure_breaker { "failed" } else { "ok" },
|
||||
pass_started.elapsed().as_millis() as i64,
|
||||
Some(stats.discovered as i32),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(lease);
|
||||
Ok(stats)
|
||||
}
|
||||
@@ -694,6 +752,7 @@ pub async fn backfill_missing_covers(
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_started = std::time::Instant::now();
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
@@ -706,6 +765,17 @@ pub async fn backfill_missing_covers(
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
let _ = repo::crawl_metrics::record(
|
||||
db,
|
||||
repo::crawl_metrics::OP_MANGA_COVER,
|
||||
Some(entry.manga_id),
|
||||
None,
|
||||
if cover_result.is_ok() { "ok" } else { "failed" },
|
||||
cover_started.elapsed().as_millis() as i64,
|
||||
None,
|
||||
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
|
||||
)
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.fetched += 1,
|
||||
Err(e) => {
|
||||
|
||||
43
backend/src/domain/crawl_metrics.rs
Normal file
43
backend/src/domain/crawl_metrics.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
//! Timing metrics for crawler operations (see `crawl_metrics` table, 0028).
|
||||
//!
|
||||
//! One [`OpRow`] per completed operation (manga list walk, manga detail,
|
||||
//! cover, whole chapter); [`OpSummary`] is the per-type average roll-up that
|
||||
//! drives the admin Metrics tab. Per-page crawl timing is derived on the
|
||||
//! client from the `chapter` summary's `avg_items` (pages/chapter), not
|
||||
//! stored per image. Analysis duration lives on `page_analysis`, not here.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Per-operation-type average roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpSummary {
|
||||
/// `manga_list` | `manga_detail` | `manga_cover` | `chapter`.
|
||||
pub op: String,
|
||||
/// Mean duration in milliseconds (`NULL`→absent only when `n = 0`).
|
||||
pub avg_ms: Option<f64>,
|
||||
/// Total operations in the window.
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
/// Mean `items` (pages/chapter, mangas/list-walk); `None` when unused.
|
||||
pub avg_items: Option<f64>,
|
||||
}
|
||||
|
||||
/// One timed operation, resolved to manga/chapter labels for the recent-ops log.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpRow {
|
||||
pub id: Uuid,
|
||||
pub op: String,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub chapter_number: Option<i32>,
|
||||
pub outcome: String,
|
||||
pub duration_ms: i64,
|
||||
pub items: Option<i32>,
|
||||
pub error: Option<String>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -4,6 +4,7 @@ pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
pub mod collection;
|
||||
pub mod crawl_metrics;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
pub mod page;
|
||||
@@ -25,6 +26,7 @@ pub use author::{Author, AuthorRef, AuthorWithCount};
|
||||
pub use bookmark::{Bookmark, BookmarkSummary};
|
||||
pub use chapter::Chapter;
|
||||
pub use collection::{Collection, CollectionPageItem, CollectionSummary};
|
||||
pub use crawl_metrics::{OpRow, OpSummary};
|
||||
pub use genre::{Genre, GenreRef};
|
||||
pub use manga::{Manga, MangaCard, MangaDetail};
|
||||
pub use page::Page;
|
||||
|
||||
@@ -157,6 +157,46 @@ pub struct PageStatusItem {
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
/// One row in the admin analysis-history table: a completed or failed
|
||||
/// analysis pass resolved to its page/chapter/manga context. Sourced from
|
||||
/// the persistent `page_analysis` table (not the reaped job queue), so it
|
||||
/// survives indefinitely. `status` is `done` | `failed`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct AnalysisHistoryRow {
|
||||
pub page_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_id: Uuid,
|
||||
pub chapter_number: i32,
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
pub status: String,
|
||||
pub is_nsfw: bool,
|
||||
pub model: Option<String>,
|
||||
pub error: Option<String>,
|
||||
pub analyzed_at: Option<DateTime<Utc>>,
|
||||
/// Wall-clock the worker spent on this page; `None` for rows analyzed
|
||||
/// before duration tracking landed.
|
||||
pub duration_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Per-model average analysis duration (admin Metrics tab).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct ModelDuration {
|
||||
pub model: Option<String>,
|
||||
pub avg_ms: Option<f64>,
|
||||
pub n: i64,
|
||||
}
|
||||
|
||||
/// Aggregate analysis timing/outcome roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct AnalysisMetrics {
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
pub avg_ms: Option<f64>,
|
||||
pub by_model: Vec<ModelDuration>,
|
||||
}
|
||||
|
||||
/// One OCR line in the page-detail view.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OcrLine {
|
||||
|
||||
158
backend/src/repo/crawl_metrics.rs
Normal file
158
backend/src/repo/crawl_metrics.rs
Normal file
@@ -0,0 +1,158 @@
|
||||
//! DB access for the `crawl_metrics` timing log (migration 0028).
|
||||
//!
|
||||
//! `record` is the best-effort write called from each crawl operation;
|
||||
//! `summary` and `list_ops` are the admin Metrics-tab reads. Mirrors the
|
||||
//! plain-fn + `query_as` style used across `repo`.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
|
||||
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
||||
/// don't drift from a free-typed literal.
|
||||
pub const OP_MANGA_LIST: &str = "manga_list";
|
||||
pub const OP_MANGA_DETAIL: &str = "manga_detail";
|
||||
pub const OP_MANGA_COVER: &str = "manga_cover";
|
||||
pub const OP_CHAPTER: &str = "chapter";
|
||||
|
||||
/// Insert one completed-operation timing row. Best-effort: callers log and
|
||||
/// continue on error so a metrics-write failure never fails the actual crawl
|
||||
/// work. `outcome` is `"ok"` or `"failed"`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn record(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
error: Option<&str>,
|
||||
) -> sqlx::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO crawl_metrics \
|
||||
(op, manga_id, chapter_id, outcome, duration_ms, items, error) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)",
|
||||
)
|
||||
.bind(op)
|
||||
.bind(manga_id)
|
||||
.bind(chapter_id)
|
||||
.bind(outcome)
|
||||
.bind(duration_ms)
|
||||
.bind(items)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Per-op average roll-up over an optional time window (`since = None` → all
|
||||
/// time). One row per `op` that has any metric in the window, with mean
|
||||
/// duration, counts, success split, and mean `items`.
|
||||
pub async fn summary(
|
||||
pool: &PgPool,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> sqlx::Result<Vec<OpSummary>> {
|
||||
sqlx::query_as::<_, OpSummary>(
|
||||
r#"
|
||||
SELECT
|
||||
op,
|
||||
AVG(duration_ms)::float8 AS avg_ms,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
|
||||
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
|
||||
AVG(items)::float8 AS avg_items
|
||||
FROM crawl_metrics
|
||||
WHERE ($1::timestamptz IS NULL OR finished_at >= $1)
|
||||
GROUP BY op
|
||||
ORDER BY op
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct OpFilter<'a> {
|
||||
pub op: Option<&'a str>,
|
||||
pub outcome: Option<&'a str>,
|
||||
pub since: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first recent-operations log resolved to manga/chapter
|
||||
/// labels. Returns the page slice plus the filtered total.
|
||||
pub async fn list_ops(
|
||||
pool: &PgPool,
|
||||
filter: OpFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<OpRow>, i64)> {
|
||||
let items = sqlx::query_as::<_, OpRow>(
|
||||
r#"
|
||||
SELECT
|
||||
cm.id,
|
||||
cm.op,
|
||||
cm.manga_id,
|
||||
m.title AS manga_title,
|
||||
cm.chapter_id,
|
||||
c.number AS chapter_number,
|
||||
cm.outcome,
|
||||
cm.duration_ms,
|
||||
cm.items,
|
||||
cm.error,
|
||||
cm.finished_at
|
||||
FROM crawl_metrics cm
|
||||
LEFT JOIN mangas m ON m.id = cm.manga_id
|
||||
LEFT JOIN chapters c ON c.id = cm.chapter_id
|
||||
WHERE ($1::text IS NULL OR cm.op = $1)
|
||||
AND ($2::text IS NULL OR cm.outcome = $2)
|
||||
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
|
||||
ORDER BY cm.finished_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.op)
|
||||
.bind(filter.outcome)
|
||||
.bind(filter.since)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawl_metrics cm
|
||||
WHERE ($1::text IS NULL OR cm.op = $1)
|
||||
AND ($2::text IS NULL OR cm.outcome = $2)
|
||||
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.op)
|
||||
.bind(filter.outcome)
|
||||
.bind(filter.since)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// Delete metric rows older than `retention_days`. `0` disables the reaper
|
||||
/// (returns 0 without touching the table). Mirrors `jobs::reap_done`.
|
||||
pub async fn reap(pool: &PgPool, retention_days: u32) -> sqlx::Result<u64> {
|
||||
if retention_days == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawl_metrics \
|
||||
WHERE finished_at < now() - ($1::bigint || ' days')::interval",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
@@ -774,6 +774,139 @@ pub async fn list_active_jobs(
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable view over the queue table.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A `crawler_jobs` row resolved to human context for the admin history
|
||||
/// table. Works across all job kinds: the target columns are best-effort
|
||||
/// `Option`s resolved through whichever payload reference the kind carries
|
||||
/// (`chapter_id`, `manga_id`, or `page_id` via the page breadcrumb), so a
|
||||
/// `sync_manga` job (which has neither yet) simply leaves them `None` and
|
||||
/// falls back to `source_key`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct JobHistoryRow {
|
||||
pub id: Uuid,
|
||||
/// `pending` | `running` | `done` | `dead` (the live queue states).
|
||||
pub state: String,
|
||||
/// `sync_manga` | `sync_chapter_list` | `sync_chapter_content` | `analyze_page`.
|
||||
pub kind: Option<String>,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub chapter_number: Option<i32>,
|
||||
/// Set only for `analyze_page` (resolved through the page breadcrumb).
|
||||
pub page_number: Option<i32>,
|
||||
/// Source-side key, the only target a `sync_manga` job carries.
|
||||
pub source_key: Option<String>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
/// How long the job's work took, when a timing was recorded: the latest
|
||||
/// `chapter` crawl-metric for a chapter job, else the page's analysis
|
||||
/// duration for an `analyze_page` job. `None` for kinds we don't time or
|
||||
/// jobs not yet completed.
|
||||
pub duration_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Filters for [`list_job_history`]. All optional; `None` widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct JobHistoryFilter<'a> {
|
||||
/// Exact queue state (`done`, `dead`, `running`, `pending`).
|
||||
pub state: Option<&'a str>,
|
||||
/// Exact payload `kind`.
|
||||
pub kind: Option<&'a str>,
|
||||
/// Case-insensitive manga-title substring.
|
||||
pub search: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first view of the job queue across every state and
|
||||
/// kind — the searchable/filterable history surface. Joins each job to its
|
||||
/// manga/chapter/page context (best-effort) so the table can label rows.
|
||||
/// Returns the page slice plus the filtered total for pagination.
|
||||
///
|
||||
/// History depth is bounded by the done-job reaper (`reap_done`): completed
|
||||
/// jobs older than the retention window are gone. Terminal `dead` jobs
|
||||
/// persist until requeued.
|
||||
pub async fn list_job_history(
|
||||
pool: &PgPool,
|
||||
filter: JobHistoryFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<JobHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
// The same FROM/JOIN/WHERE drives both the page slice and the count, so
|
||||
// they stay in lockstep. `pg` resolves analyze_page → page → chapter;
|
||||
// `COALESCE` lets one set of joins serve every kind.
|
||||
let items: Vec<JobHistoryRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
cj.id,
|
||||
cj.state,
|
||||
cj.payload->>'kind' AS kind,
|
||||
COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid) AS manga_id,
|
||||
m.title AS manga_title,
|
||||
COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id) AS chapter_id,
|
||||
c.number AS chapter_number,
|
||||
pg.page_number AS page_number,
|
||||
cj.payload->>'source_manga_key' AS source_key,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
cj.updated_at,
|
||||
COALESCE(cm.duration_ms, pa.duration_ms) AS duration_ms
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT duration_ms FROM crawl_metrics
|
||||
WHERE op = 'chapter'
|
||||
AND chapter_id = (cj.payload->>'chapter_id')::uuid
|
||||
ORDER BY finished_at DESC LIMIT 1
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
.bind(filter.kind)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
|
||||
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
.bind(filter.kind)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// A manga whose cover is still missing (queued for cover fetch).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MissingCoverRow {
|
||||
|
||||
@@ -6,6 +6,7 @@ pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
pub mod collection;
|
||||
pub mod crawl_metrics;
|
||||
pub mod crawler;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
|
||||
@@ -60,6 +60,27 @@ pub async fn locate(
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Like [`locate`] but also resolves the manga title and chapter number, so
|
||||
/// live analysis events can carry human labels for the "now analyzing"
|
||||
/// banner without the dashboard having to look them up. Returns
|
||||
/// `(manga_id, manga_title, chapter_id, chapter_number, page_number)`.
|
||||
pub async fn locate_labeled(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<Option<(Uuid, String, Uuid, i32, i32)>> {
|
||||
let row: Option<(Uuid, String, Uuid, i32, i32)> = sqlx::query_as(
|
||||
"SELECT c.manga_id, m.title, p.chapter_id, c.number, p.page_number \
|
||||
FROM pages p \
|
||||
JOIN chapters c ON c.id = p.chapter_id \
|
||||
JOIN mangas m ON m.id = c.manga_id \
|
||||
WHERE p.id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
pub async fn list_for_chapter(pool: &PgPool, chapter_id: Uuid) -> AppResult<Vec<Page>> {
|
||||
let rows = sqlx::query_as::<_, Page>(
|
||||
r#"
|
||||
|
||||
@@ -16,9 +16,11 @@ use uuid::Uuid;
|
||||
|
||||
use crate::crawler::jobs::{self, JobPayload};
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, ContentWarning, MangaCoverage, OcrKind, OcrLine, PageAnalysis,
|
||||
PageAnalysisDetail, PageSearchItem, PageStatusItem, VisionAnalysis,
|
||||
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, ContentWarning, MangaCoverage,
|
||||
ModelDuration, OcrKind, OcrLine, PageAnalysis, PageAnalysisDetail, PageSearchItem,
|
||||
PageStatusItem, VisionAnalysis,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use crate::error::AppResult;
|
||||
|
||||
/// Filter set for [`page_search`]. `tags` are AND-ed (a page must carry
|
||||
@@ -212,6 +214,90 @@ pub async fn chapter_page_status(
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Filters for [`list_history`]. `None` widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct AnalysisHistoryFilter<'a> {
|
||||
/// Exact analysis status (`done` or `failed`).
|
||||
pub status: Option<&'a str>,
|
||||
/// Only NSFW-flagged pages.
|
||||
pub nsfw_only: bool,
|
||||
/// Case-insensitive manga-title substring.
|
||||
pub search: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Paginated, newest-first analysis history — every completed/failed page
|
||||
/// analysis resolved to its manga/chapter/page context. Reads the
|
||||
/// persistent `page_analysis` table, so unlike the crawler job history it
|
||||
/// is not bounded by job reaping. Returns the page slice plus the filtered
|
||||
/// total. `pending` rows are excluded — history is terminal outcomes only.
|
||||
pub async fn list_history(
|
||||
pool: &PgPool,
|
||||
filter: AnalysisHistoryFilter<'_>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<AnalysisHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, AnalysisHistoryRow>(
|
||||
r#"
|
||||
SELECT
|
||||
pa.page_id,
|
||||
p.page_number,
|
||||
p.chapter_id,
|
||||
c.number AS chapter_number,
|
||||
c.manga_id,
|
||||
m.title AS manga_title,
|
||||
pa.status,
|
||||
pa.is_nsfw,
|
||||
pa.model,
|
||||
pa.error,
|
||||
pa.analyzed_at,
|
||||
pa.duration_ms
|
||||
FROM page_analysis pa
|
||||
JOIN pages p ON p.id = pa.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
ORDER BY pa.analyzed_at DESC NULLS LAST, pa.page_id
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
.bind(filter.nsfw_only)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*)
|
||||
FROM page_analysis pa
|
||||
JOIN pages p ON p.id = pa.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
.bind(filter.nsfw_only)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// Full analysis detail for one page. `None` when the page itself doesn't
|
||||
/// exist; an existing-but-unanalyzed page returns `status = "none"` with
|
||||
/// empty OCR/tags/warnings so the UI can show "not analyzed yet".
|
||||
@@ -331,6 +417,66 @@ pub async fn mark_failed(pool: &PgPool, page_id: Uuid, error: &str) -> AppResult
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record the wall-clock the worker spent on a page. Best-effort: updates the
|
||||
/// existing `page_analysis` row (written by `persist_analysis`/`mark_failed`);
|
||||
/// a transient failure with no row yet simply updates nothing.
|
||||
pub async fn record_duration(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
duration_ms: i64,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query("UPDATE page_analysis SET duration_ms = $2 WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.bind(duration_ms)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Aggregate analysis timing + outcome roll-up over an optional time window
|
||||
/// (`since = None` → all time). Mean duration ignores NULL `duration_ms`
|
||||
/// (pre-tracking rows); `by_model` breaks the mean out per model.
|
||||
pub async fn analysis_metrics(
|
||||
pool: &PgPool,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> AppResult<AnalysisMetrics> {
|
||||
let (n, ok, failed, avg_ms): (i64, i64, i64, Option<f64>) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE status = 'done') AS ok,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||
-- Mean over successful pages only, so it reconciles with the
|
||||
-- by-model breakdown below (which is done-only). Failed /
|
||||
-- timed-out durations would otherwise skew the headline up.
|
||||
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
|
||||
FROM page_analysis
|
||||
WHERE status <> 'pending'
|
||||
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
let by_model = sqlx::query_as::<_, ModelDuration>(
|
||||
r#"
|
||||
SELECT model, AVG(duration_ms)::float8 AS avg_ms, COUNT(*) AS n
|
||||
FROM page_analysis
|
||||
WHERE status = 'done'
|
||||
AND duration_ms IS NOT NULL
|
||||
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
|
||||
GROUP BY model
|
||||
ORDER BY n DESC
|
||||
"#,
|
||||
)
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
||||
}
|
||||
|
||||
/// Persist a completed analysis for a page, replacing any previous result.
|
||||
///
|
||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||
|
||||
@@ -192,6 +192,11 @@ impl CrawlerSettings {
|
||||
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||
retention_days: self.retention_days,
|
||||
// Not a runtime-editable setting; re-read the env default so a
|
||||
// settings reload keeps whatever CRAWL_METRICS_RETENTION_DAYS was
|
||||
// configured at boot.
|
||||
metrics_retention_days: crate::config::env_u64("CRAWL_METRICS_RETENTION_DAYS", 90)
|
||||
as u32,
|
||||
start_url,
|
||||
rate_ms: self.rate_ms,
|
||||
cdn_host,
|
||||
|
||||
@@ -158,6 +158,16 @@ async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
|
||||
|
||||
// The worker stamps how long the dispatch took onto the existing row
|
||||
// (recorded after ack; shutdown has awaited the in-flight job).
|
||||
let dur: Option<i64> =
|
||||
sqlx::query_scalar("SELECT duration_ms FROM page_analysis WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(dur.is_some(), "worker should record analysis duration_ms");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -601,3 +601,255 @@ async fn coverage_requires_admin(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis history: terminal-outcome log, searchable + filterable.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga → chapter → one page, then write a terminal `page_analysis`
|
||||
/// row for that page. Returns the page id.
|
||||
async fn seed_analyzed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
is_nsfw: bool,
|
||||
error: Option<&str>,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, is_nsfw, model, error, analyzed_at) \
|
||||
VALUES ($1, $2, $3, 'vis', $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(is_nsfw)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_analyzed_page(&pool, "Alpha", "done", false, None).await;
|
||||
seed_analyzed_page(&pool, "Beta", "failed", false, Some("boom")).await;
|
||||
seed_analyzed_page(&pool, "Gamma", "done", true, None).await;
|
||||
// A still-pending row must NOT appear in history (terminal outcomes only).
|
||||
let pending_page = seed_analyzed_page(&pool, "Delta", "pending", false, None).await;
|
||||
let _ = pending_page;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: the three terminal rows, pending excluded.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
|
||||
// Filter by status=failed.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?status=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Beta");
|
||||
assert_eq!(body["items"][0]["error"], "boom");
|
||||
|
||||
// NSFW-only.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?nsfw=true",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Gamma");
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?search=Alpha",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Alpha");
|
||||
assert_eq!(body["items"][0]["status"], "done");
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis metrics: durations + averages + by-model + success.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a manga→chapter→page + a terminal page_analysis row carrying a
|
||||
/// duration and model. Returns the page id.
|
||||
async fn seed_timed_page(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
status: &str,
|
||||
model: &str,
|
||||
duration_ms: i64,
|
||||
) -> uuid::Uuid {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, model, duration_ms, analyzed_at) \
|
||||
VALUES ($1, $2, $3, $4, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(status)
|
||||
.bind(model)
|
||||
.bind(duration_ms)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
page_id
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_metrics_over_http(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2000).await;
|
||||
seed_timed_page(&pool, "Beta", "done", "qwen", 4000).await;
|
||||
seed_timed_page(&pool, "Gamma", "done", "llava", 3000).await;
|
||||
seed_timed_page(&pool, "Delta", "failed", "qwen", 1000).await;
|
||||
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/metrics",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["n"], 4);
|
||||
assert_eq!(body["ok"], 3);
|
||||
assert_eq!(body["failed"], 1);
|
||||
// Headline mean is done-only (2000 + 4000 + 3000)/3 — the failed row's
|
||||
// 1000ms is excluded so it reconciles with the by-model breakdown.
|
||||
assert_eq!(body["avg_ms"], 3000.0);
|
||||
|
||||
// by_model: qwen has the done rows 2000 + 4000 → avg 3000, n 2
|
||||
// (the failed qwen row is excluded — done only).
|
||||
let by_model = body["by_model"].as_array().unwrap();
|
||||
let qwen = by_model.iter().find(|m| m["model"] == "qwen").unwrap();
|
||||
assert_eq!(qwen["n"], 2);
|
||||
assert_eq!(qwen["avg_ms"], 3000.0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn analysis_history_carries_duration(pool: PgPool) {
|
||||
seed_timed_page(&pool, "Alpha", "done", "qwen", 2500).await;
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/history",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 2500);
|
||||
}
|
||||
|
||||
@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
|
||||
assert_eq!(payload["requeued"], 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job history: unified, searchable, filterable list over all states/kinds.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Seed a chapter-content job in a given state and return its (manga, chapter).
|
||||
async fn seed_history_chapter_job(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
number: i32,
|
||||
state: &str,
|
||||
) -> (Uuid, Uuid) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
/// Seed a `done` analyze_page job pointing at a real page so the history
|
||||
/// row resolves manga/chapter/page-number through the page breadcrumb.
|
||||
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
let page_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'k', 'image/png')",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(chapter_id)
|
||||
.bind(page_number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
|
||||
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
|
||||
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
|
||||
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
|
||||
seed_history_analyze_job(&pool, "One Piece", 7).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Unfiltered: all four jobs.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Filter by state.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?state=done",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
// Naruto (sync_chapter done) + One Piece (analyze_page done).
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
|
||||
// Filter by kind.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=analyze_page",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "One Piece");
|
||||
assert_eq!(body["items"][0]["page_number"], 7);
|
||||
assert_eq!(body["items"][0]["chapter_number"], 3);
|
||||
|
||||
// Search by manga title.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?search=Bleach",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Bleach");
|
||||
assert_eq!(body["items"][0]["state"], "running");
|
||||
|
||||
// Pagination: limit clamps the page slice but total reflects the full set.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?limit=1",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["page"]["total"], 4);
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operation metrics: durations, averages, recent-ops log.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn record_metric(
|
||||
pool: &PgPool,
|
||||
op: &str,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
outcome: &str,
|
||||
duration_ms: i64,
|
||||
items: Option<i32>,
|
||||
) {
|
||||
mangalord::repo::crawl_metrics::record(
|
||||
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
|
||||
// Two chapter ops (one failed) + a cover op.
|
||||
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
|
||||
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
|
||||
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Summary: per-type averages + success split.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let summary = body["summary"].as_array().unwrap();
|
||||
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
|
||||
assert_eq!(chapter["n"], 2);
|
||||
assert_eq!(chapter["ok"], 1);
|
||||
assert_eq!(chapter["failed"], 1);
|
||||
assert_eq!(chapter["avg_ms"], 7000.0);
|
||||
|
||||
// Recent-ops log + outcome filter.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["op"], "chapter");
|
||||
assert_eq!(body["items"][0]["outcome"], "failed");
|
||||
|
||||
// Admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_history_includes_chapter_duration(pool: PgPool) {
|
||||
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
|
||||
// A chapter metric for that chapter → history row should surface its duration.
|
||||
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie(
|
||||
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["items"][0]["duration_ms"], 6100);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
|
||||
140
backend/tests/crawl_metrics.rs
Normal file
140
backend/tests/crawl_metrics.rs
Normal file
@@ -0,0 +1,140 @@
|
||||
//! Repo-level integration tests for the crawl_metrics timing log.
|
||||
|
||||
use chrono::Utc;
|
||||
use mangalord::repo::crawl_metrics::{
|
||||
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
|
||||
|
||||
// Two chapter ops (one ok, one failed) + one cover op.
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(
|
||||
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter.n, 2);
|
||||
assert_eq!(chapter.ok, 1);
|
||||
assert_eq!(chapter.failed, 1);
|
||||
assert_eq!(chapter.avg_ms, Some(7000.0));
|
||||
assert_eq!(chapter.avg_items, Some(15.0));
|
||||
|
||||
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
|
||||
assert_eq!(cover.n, 1);
|
||||
assert_eq!(cover.avg_ms, Some(500.0));
|
||||
assert_eq!(cover.avg_items, None);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn summary_window_excludes_old_rows(pool: PgPool) {
|
||||
let (m, _c) = seed_manga_chapter(&pool, "X").await;
|
||||
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// Backdate it two days.
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let since = Utc::now() - chrono::Duration::days(1);
|
||||
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
|
||||
assert!(windowed.is_empty(), "row older than the window must be excluded");
|
||||
|
||||
let all = crawl_metrics::summary(&pool, None).await.unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
|
||||
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
|
||||
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
|
||||
.await
|
||||
.unwrap();
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Unfiltered: both, newest-first, with labels resolved.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(rows.len(), 2);
|
||||
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
|
||||
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
|
||||
assert_eq!(chapter_row.chapter_number, Some(12));
|
||||
assert_eq!(chapter_row.items, Some(20));
|
||||
|
||||
// Filter by op.
|
||||
let (rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(rows[0].outcome, "failed");
|
||||
assert_eq!(rows[0].error.as_deref(), Some("oops"));
|
||||
|
||||
// Filter by outcome.
|
||||
let (_rows, total) = crawl_metrics::list_ops(
|
||||
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
// Pagination: limit clamps the slice; total reflects the full set.
|
||||
let (rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
|
||||
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 0 disables — nothing removed.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
|
||||
// 90-day retention removes the 100-day-old row.
|
||||
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
|
||||
let (_rows, total) =
|
||||
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
|
||||
assert_eq!(total, 0);
|
||||
}
|
||||
@@ -39,6 +39,7 @@ fn make_cfg(
|
||||
daily_at: far_future_daily_at(),
|
||||
tz: Tz::UTC,
|
||||
retention_days: 7,
|
||||
metrics_retention_days: 90,
|
||||
session_expired,
|
||||
status: mangalord::crawler::status::StatusHandle::new(workers),
|
||||
job_timeout: Duration::from_secs(60),
|
||||
|
||||
@@ -155,6 +155,48 @@ async function mockAdmin(page: Page, cap: Captured) {
|
||||
})
|
||||
);
|
||||
|
||||
// Analysis history (terminal-outcome log). Registered before the
|
||||
// page-detail routes are matched by their more-specific globs.
|
||||
await page.route('**/api/v1/admin/analysis/history**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
page_id: pageDone,
|
||||
page_number: 1,
|
||||
chapter_id: chapterId,
|
||||
chapter_number: 1,
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
status: 'done',
|
||||
is_nsfw: true,
|
||||
model: 'test-model',
|
||||
error: null,
|
||||
analyzed_at: '2026-06-13T12:00:00Z',
|
||||
duration_ms: 2400
|
||||
}
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
await page.route('**/api/v1/admin/analysis/metrics**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
n: 3204,
|
||||
ok: 3159,
|
||||
failed: 45,
|
||||
avg_ms: 2400,
|
||||
by_model: [{ model: 'qwen2-vl-7b', avg_ms: 2400, n: 3100 }]
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
// Default: keep the SSE connection pending (no events) so tests that
|
||||
// don't care about live updates don't trigger reconnect churn. The
|
||||
// live-updates test overrides this with a fulfilling stream.
|
||||
@@ -240,14 +282,18 @@ test.describe('/admin/analysis', () => {
|
||||
kind: 'started',
|
||||
page_id: pageNone,
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: chapterId,
|
||||
chapter_number: 1,
|
||||
page_number: 2
|
||||
})}\n\n` +
|
||||
`event: analysis\ndata: ${JSON.stringify({
|
||||
kind: 'completed',
|
||||
page_id: pageNone,
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: chapterId,
|
||||
chapter_number: 1,
|
||||
page_number: 2
|
||||
})}\n\n`;
|
||||
let served = false;
|
||||
@@ -268,10 +314,81 @@ test.describe('/admin/analysis', () => {
|
||||
// (The mocked stream closes after its body, so the live pill flips
|
||||
// back to "Reconnecting…" — the ticker is the durable signal.)
|
||||
const tick = page.getByTestId('admin-analysis-ticker');
|
||||
await expect(tick).toContainText('Analyzing page 2');
|
||||
await expect(tick).toContainText('Analyzing Berserk');
|
||||
await expect(tick).toContainText('Analyzed page 2');
|
||||
});
|
||||
|
||||
test('now-analyzing banner appears from SSE and Jump expands the chapter', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
// A single `started` frame, then hang so the banner stays put.
|
||||
const frame = `event: analysis\ndata: ${JSON.stringify({
|
||||
kind: 'started',
|
||||
page_id: pageDone,
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: chapterId,
|
||||
chapter_number: 1,
|
||||
page_number: 1
|
||||
})}\n\n`;
|
||||
let served = false;
|
||||
await page.route('**/api/v1/admin/analysis/status/stream', (r) => {
|
||||
if (served) return new Promise(() => {});
|
||||
served = true;
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
body: frame
|
||||
});
|
||||
});
|
||||
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
const banner = page.getByTestId('admin-analysis-now');
|
||||
await expect(banner).toContainText('Now analyzing');
|
||||
await expect(banner).toContainText('Berserk · Ch 1 · Page 1');
|
||||
|
||||
// Jump expands the manga + chapter and the analyzing chip appears.
|
||||
await page.getByTestId('admin-analysis-now-jump').click();
|
||||
await expect(page.getByTestId(`admin-analysis-page-${pageDone}`)).toBeVisible();
|
||||
});
|
||||
|
||||
test('history tab lists terminal analyses and opens the detail modal', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
await page.getByTestId('admin-analysis-tab-history').click();
|
||||
const row = page.getByTestId(`analysis-history-row-${pageDone}`);
|
||||
await expect(row).toContainText('Berserk · Ch 1 · p1');
|
||||
await expect(row).toContainText('NSFW');
|
||||
await expect(row).toContainText('2.4s'); // duration column
|
||||
|
||||
await row.click();
|
||||
await expect(page.getByTestId('admin-analysis-detail')).toBeVisible();
|
||||
await expect(page.getByTestId('admin-analysis-detail-status')).toContainText(
|
||||
'Analyzed'
|
||||
);
|
||||
});
|
||||
|
||||
test('metrics tab shows aggregate timing + by-model', async ({ page }) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
await page.getByTestId('admin-analysis-tab-metrics').click();
|
||||
await expect(page.getByTestId('analysis-metrics-n')).toContainText('3204');
|
||||
await expect(page.getByTestId('analysis-metrics-avg')).toContainText('2.4s');
|
||||
await expect(page.getByTestId('analysis-metrics')).toContainText('qwen2-vl-7b');
|
||||
});
|
||||
|
||||
test('queue an unanalyzed page from its detail modal', async ({ page }) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
|
||||
220
frontend/e2e/admin-crawler.spec.ts
Normal file
220
frontend/e2e/admin-crawler.spec.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// E2E for the admin Crawler "History" tab: the Live/History toggle, the
|
||||
// searchable/filterable job log, and inline requeue of a dead job. The
|
||||
// live status + SSE stream are left pending so the test focuses on history
|
||||
// (the live dashboard is exercised by its own status mocks elsewhere).
|
||||
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const deadJobId = 'd1111111-1111-1111-1111-111111111111';
|
||||
const doneJobId = 'd2222222-2222-2222-2222-222222222222';
|
||||
|
||||
const adminUser = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'admin',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: true
|
||||
};
|
||||
|
||||
type Captured = { requeue: Record<string, unknown> | null };
|
||||
|
||||
async function mockAdmin(page: Page, cap: Captured) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: adminUser })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/system', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
disk: null,
|
||||
memory: { total_bytes: 1, used_bytes: 0, percent_used: 0 },
|
||||
cpu: { percent_used: 0 },
|
||||
alerts: []
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
// Leave the live status fetch + SSE stream pending so the Live view
|
||||
// stays in its "Loading…" state and doesn't churn; History is what we test.
|
||||
await page.route('**/api/v1/admin/crawler', () => new Promise(() => {}));
|
||||
await page.route('**/api/v1/admin/crawler/stream', () => new Promise(() => {}));
|
||||
await page.route('**/api/v1/admin/crawler/dead-jobs**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 20, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
|
||||
await page.route('**/api/v1/admin/crawler/history**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
id: deadJobId,
|
||||
state: 'dead',
|
||||
kind: 'sync_chapter_content',
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Naruto',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 700,
|
||||
page_number: null,
|
||||
source_key: null,
|
||||
attempts: 5,
|
||||
max_attempts: 5,
|
||||
last_error: 'boom: upstream 500',
|
||||
updated_at: '2026-06-15T00:00:00Z'
|
||||
},
|
||||
{
|
||||
id: doneJobId,
|
||||
state: 'done',
|
||||
kind: 'analyze_page',
|
||||
manga_id: 'm-2',
|
||||
manga_title: 'Bleach',
|
||||
chapter_id: 'c-2',
|
||||
chapter_number: 3,
|
||||
page_number: 7,
|
||||
source_key: null,
|
||||
attempts: 1,
|
||||
max_attempts: 5,
|
||||
last_error: null,
|
||||
updated_at: '2026-06-15T00:00:00Z'
|
||||
}
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 2 }
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
// Summary registered first (broad glob); the more-specific /ops route is
|
||||
// registered after so it wins for the ops URL (Playwright: last match wins).
|
||||
await page.route('**/api/v1/admin/crawler/metrics**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
summary: [
|
||||
{ op: 'manga_list', avg_ms: 42000, n: 8, ok: 8, failed: 0, avg_items: 50 },
|
||||
{ op: 'manga_detail', avg_ms: 1300, n: 210, ok: 205, failed: 5, avg_items: null },
|
||||
{ op: 'manga_cover', avg_ms: 480, n: 180, ok: 176, failed: 4, avg_items: null },
|
||||
{ op: 'chapter', avg_ms: 6800, n: 430, ok: 421, failed: 9, avg_items: 32 }
|
||||
]
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/crawler/metrics/ops**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
id: 'op-1',
|
||||
op: 'chapter',
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 12,
|
||||
outcome: 'ok',
|
||||
duration_ms: 6100,
|
||||
items: 20,
|
||||
error: null,
|
||||
finished_at: '2026-06-15T00:00:00Z'
|
||||
}
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
await page.route('**/api/v1/admin/crawler/dead-jobs/requeue', (r) => {
|
||||
cap.requeue = JSON.parse(r.request().postData() ?? '{}');
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ requeued: 1 })
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('/admin/crawler history', () => {
|
||||
test('toggle to History lists jobs and requeues a dead job inline', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { requeue: null };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/crawler');
|
||||
|
||||
await page.getByTestId('crawler-tab-history').click();
|
||||
|
||||
const deadRow = page.getByTestId(`crawler-history-row-${deadJobId}`);
|
||||
await expect(deadRow).toContainText('Naruto · Ch 700');
|
||||
await expect(deadRow).toContainText('dead');
|
||||
|
||||
const doneRow = page.getByTestId(`crawler-history-row-${doneJobId}`);
|
||||
await expect(doneRow).toContainText('Bleach · Ch 3 · p7');
|
||||
|
||||
// Click the dead row to expand its error detail.
|
||||
await deadRow.click();
|
||||
await expect(
|
||||
page.getByTestId('crawler-history').locator('.errrow')
|
||||
).toContainText('boom: upstream 500');
|
||||
|
||||
// Inline requeue posts scope=job.
|
||||
await page.getByTestId(`crawler-history-requeue-${deadJobId}`).click();
|
||||
await expect.poll(() => cap.requeue).toEqual({ scope: 'job', job_id: deadJobId });
|
||||
});
|
||||
|
||||
test('Metrics tab shows averages by type, derived per-page, and the ops log', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { requeue: null };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/crawler');
|
||||
|
||||
await page.getByTestId('crawler-tab-metrics').click();
|
||||
|
||||
// Per-type averages.
|
||||
await expect(page.getByTestId('crawler-metrics-row-chapter')).toContainText('6.8s');
|
||||
await expect(page.getByTestId('crawler-metrics-row-manga_cover')).toContainText(
|
||||
'480ms'
|
||||
);
|
||||
// Derived per-page row: 6800ms ÷ 32 ≈ 213ms.
|
||||
await expect(page.getByTestId('crawler-metrics-perpage')).toContainText('213ms');
|
||||
|
||||
// Recent-ops log shows individual durations.
|
||||
await expect(page.getByTestId('crawler-op-op-1')).toContainText('6.1s');
|
||||
await expect(page.getByTestId('crawler-op-op-1')).toContainText('Berserk · Ch 12');
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.82.0",
|
||||
"version": "0.84.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -35,7 +35,12 @@ import {
|
||||
getAnalysisChapterCoverage,
|
||||
getAnalysisChapterPages,
|
||||
getAnalysisPageDetail,
|
||||
analysisStatusStreamUrl
|
||||
analysisStatusStreamUrl,
|
||||
listCrawlerJobHistory,
|
||||
listAnalysisHistory,
|
||||
getCrawlerMetrics,
|
||||
listCrawlerOps,
|
||||
getAnalysisMetrics
|
||||
} from './admin';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -651,4 +656,152 @@ describe('admin crawler api client', () => {
|
||||
expect(url).toMatch(/\/v1\/admin\/storage\/backfill$/);
|
||||
expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST');
|
||||
});
|
||||
|
||||
// ---- job history ----
|
||||
|
||||
it('listCrawlerJobHistory forwards state/kind/search/limit/offset and parses items', async () => {
|
||||
const row = {
|
||||
id: 'j-1',
|
||||
state: 'done',
|
||||
kind: 'analyze_page',
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 12,
|
||||
page_number: 7,
|
||||
source_key: null,
|
||||
attempts: 1,
|
||||
max_attempts: 5,
|
||||
last_error: null,
|
||||
updated_at: '2026-06-15T00:00:00Z'
|
||||
};
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [row], page: { limit: 50, offset: 0, total: 1 } })
|
||||
);
|
||||
const page = await listCrawlerJobHistory({
|
||||
state: 'done',
|
||||
kind: 'analyze_page',
|
||||
search: 'Ber',
|
||||
offset: 10
|
||||
});
|
||||
expect(page.items[0]).toEqual(row);
|
||||
expect(page.page.total).toBe(1);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('/v1/admin/crawler/history?');
|
||||
expect(url).toContain('state=done');
|
||||
expect(url).toContain('kind=analyze_page');
|
||||
expect(url).toContain('search=Ber');
|
||||
expect(url).toContain('offset=10');
|
||||
});
|
||||
|
||||
it('listCrawlerJobHistory omits unset filters', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
);
|
||||
await listCrawlerJobHistory();
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler\/history$/);
|
||||
});
|
||||
|
||||
it('listAnalysisHistory forwards status/nsfw/search and parses items', async () => {
|
||||
const row = {
|
||||
page_id: 'p-1',
|
||||
page_number: 3,
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 700,
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Naruto',
|
||||
status: 'failed',
|
||||
is_nsfw: false,
|
||||
model: null,
|
||||
error: 'boom',
|
||||
analyzed_at: '2026-06-15T00:00:00Z'
|
||||
};
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [row], page: { limit: 25, offset: 0, total: 1 } })
|
||||
);
|
||||
const page = await listAnalysisHistory({ status: 'failed', nsfw: true, search: 'Nar' });
|
||||
expect(page.items[0]).toEqual(row);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('/v1/admin/analysis/history?');
|
||||
expect(url).toContain('status=failed');
|
||||
expect(url).toContain('nsfw=true');
|
||||
expect(url).toContain('search=Nar');
|
||||
});
|
||||
|
||||
it('listAnalysisHistory omits nsfw when false', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 25, offset: 0, total: 0 } })
|
||||
);
|
||||
await listAnalysisHistory({ status: 'done', nsfw: false });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('status=done');
|
||||
expect(url).not.toContain('nsfw');
|
||||
});
|
||||
|
||||
// ---- operation metrics ----
|
||||
|
||||
it('getCrawlerMetrics omits days when 0 and parses the summary', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
summary: [
|
||||
{ op: 'chapter', avg_ms: 7000, n: 2, ok: 1, failed: 1, avg_items: 15 }
|
||||
]
|
||||
})
|
||||
);
|
||||
const r = await getCrawlerMetrics(0);
|
||||
expect(r.summary[0].avg_ms).toBe(7000);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler\/metrics$/);
|
||||
});
|
||||
|
||||
it('getCrawlerMetrics forwards a positive days window', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ summary: [] }));
|
||||
await getCrawlerMetrics(7);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('days=7');
|
||||
});
|
||||
|
||||
it('listCrawlerOps forwards op/outcome/days and parses the page', async () => {
|
||||
const row = {
|
||||
id: 'o-1',
|
||||
op: 'chapter',
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Berserk',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 12,
|
||||
outcome: 'ok',
|
||||
duration_ms: 6100,
|
||||
items: 20,
|
||||
error: null,
|
||||
finished_at: '2026-06-15T00:00:00Z'
|
||||
};
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [row], page: { limit: 50, offset: 0, total: 1 } })
|
||||
);
|
||||
const page = await listCrawlerOps({ op: 'chapter', outcome: 'ok', days: 30 });
|
||||
expect(page.items[0]).toEqual(row);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('/v1/admin/crawler/metrics/ops?');
|
||||
expect(url).toContain('op=chapter');
|
||||
expect(url).toContain('outcome=ok');
|
||||
expect(url).toContain('days=30');
|
||||
});
|
||||
|
||||
it('getAnalysisMetrics parses aggregate + by-model', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
n: 4,
|
||||
ok: 3,
|
||||
failed: 1,
|
||||
avg_ms: 2500,
|
||||
by_model: [{ model: 'qwen', avg_ms: 3000, n: 2 }]
|
||||
})
|
||||
);
|
||||
const m = await getAnalysisMetrics(7);
|
||||
expect(m.avg_ms).toBe(2500);
|
||||
expect(m.by_model[0].model).toBe('qwen');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('/v1/admin/analysis/metrics?days=7');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -436,6 +436,128 @@ export async function listActiveJobs(
|
||||
);
|
||||
}
|
||||
|
||||
/** Queue states a crawler job can be in. */
|
||||
export type CrawlerJobState = 'pending' | 'running' | 'done' | 'dead';
|
||||
|
||||
/** Job kinds the crawler/analysis queue carries. */
|
||||
export type CrawlerJobKind =
|
||||
| 'sync_manga'
|
||||
| 'sync_chapter_list'
|
||||
| 'sync_chapter_content'
|
||||
| 'analyze_page';
|
||||
|
||||
/** One row in the crawler job-history table — any state/kind, resolved to
|
||||
* its manga/chapter/page context (best-effort `null`s for kinds that don't
|
||||
* carry that reference, e.g. a bootstrap `sync_manga`). */
|
||||
export type CrawlerHistoryRow = {
|
||||
id: string;
|
||||
state: CrawlerJobState;
|
||||
kind: CrawlerJobKind | null;
|
||||
manga_id: string | null;
|
||||
manga_title: string | null;
|
||||
chapter_id: string | null;
|
||||
chapter_number: number | null;
|
||||
page_number: number | null;
|
||||
source_key: string | null;
|
||||
attempts: number;
|
||||
max_attempts: number;
|
||||
last_error: string | null;
|
||||
updated_at: string;
|
||||
/** Recorded duration for a chapter / analyze job; `null` otherwise. */
|
||||
duration_ms: number | null;
|
||||
};
|
||||
|
||||
export type CrawlerHistoryPage = { items: CrawlerHistoryRow[]; page: Page };
|
||||
|
||||
/** GET /v1/admin/crawler/history — unified, searchable/filterable job log.
|
||||
* History depth is bounded by the done-job reaper (recent window); `dead`
|
||||
* jobs persist until requeued. */
|
||||
export async function listCrawlerJobHistory(
|
||||
opts?: {
|
||||
state?: CrawlerJobState;
|
||||
kind?: CrawlerJobKind;
|
||||
search?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<CrawlerHistoryPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.state) params.set('state', opts.state);
|
||||
if (opts?.kind) params.set('kind', opts.kind);
|
||||
if (opts?.search) params.set('search', opts.search);
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<CrawlerHistoryPage>(
|
||||
`/v1/admin/crawler/history${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
// ---- operation metrics (durations / averages) ------------------------------
|
||||
|
||||
/** Crawl operation kinds timed in `crawl_metrics`. */
|
||||
export type CrawlOp = 'manga_list' | 'manga_detail' | 'manga_cover' | 'chapter';
|
||||
|
||||
/** Per-op average roll-up over the selected window. */
|
||||
export type OpSummary = {
|
||||
op: CrawlOp;
|
||||
avg_ms: number | null;
|
||||
n: number;
|
||||
ok: number;
|
||||
failed: number;
|
||||
avg_items: number | null;
|
||||
};
|
||||
|
||||
/** One timed operation in the recent-ops log. */
|
||||
export type OpRow = {
|
||||
id: string;
|
||||
op: CrawlOp;
|
||||
manga_id: string | null;
|
||||
manga_title: string | null;
|
||||
chapter_id: string | null;
|
||||
chapter_number: number | null;
|
||||
outcome: 'ok' | 'failed';
|
||||
duration_ms: number;
|
||||
items: number | null;
|
||||
error: string | null;
|
||||
finished_at: string;
|
||||
};
|
||||
|
||||
export type CrawlerOpsPage = { items: OpRow[]; page: Page };
|
||||
|
||||
/** GET /v1/admin/crawler/metrics — per-type average durations + success.
|
||||
* `days` windows the rows (0/omitted = all time). */
|
||||
export async function getCrawlerMetrics(days = 0): Promise<{ summary: OpSummary[] }> {
|
||||
const qs = days > 0 ? `?days=${days}` : '';
|
||||
return request<{ summary: OpSummary[] }>(`/v1/admin/crawler/metrics${qs}`);
|
||||
}
|
||||
|
||||
/** GET /v1/admin/crawler/metrics/ops — paginated recent timed-operations log. */
|
||||
export async function listCrawlerOps(
|
||||
opts?: {
|
||||
op?: CrawlOp;
|
||||
outcome?: 'ok' | 'failed';
|
||||
days?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<CrawlerOpsPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.op) params.set('op', opts.op);
|
||||
if (opts?.outcome) params.set('outcome', opts.outcome);
|
||||
if (opts?.days != null && opts.days > 0) params.set('days', String(opts.days));
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<CrawlerOpsPage>(
|
||||
`/v1/admin/crawler/metrics/ops${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
/** A manga queued for a cover fetch (no cover yet + a live source). */
|
||||
export type MissingCover = { manga_id: string; manga_title: string };
|
||||
export type MissingCoversPage = { items: MissingCover[]; page: Page };
|
||||
@@ -588,6 +710,71 @@ export async function getAnalysisPageDetail(
|
||||
);
|
||||
}
|
||||
|
||||
/** One row in the analysis history table — a terminal (done/failed)
|
||||
* analysis pass resolved to its page/chapter/manga context. Persists
|
||||
* indefinitely (sourced from `page_analysis`, not the reaped job queue). */
|
||||
export type AnalysisHistoryRow = {
|
||||
page_id: string;
|
||||
page_number: number;
|
||||
chapter_id: string;
|
||||
chapter_number: number;
|
||||
manga_id: string;
|
||||
manga_title: string;
|
||||
status: 'done' | 'failed';
|
||||
is_nsfw: boolean;
|
||||
model: string | null;
|
||||
error: string | null;
|
||||
analyzed_at: string | null;
|
||||
/** Wall-clock the worker spent; `null` for pre-tracking rows. */
|
||||
duration_ms: number | null;
|
||||
};
|
||||
|
||||
export type AnalysisHistoryPage = { items: AnalysisHistoryRow[]; page: Page };
|
||||
|
||||
/** GET /v1/admin/analysis/history — searchable/filterable log of completed
|
||||
* and failed page analyses, newest first. */
|
||||
export async function listAnalysisHistory(
|
||||
opts?: {
|
||||
status?: 'done' | 'failed';
|
||||
nsfw?: boolean;
|
||||
search?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<AnalysisHistoryPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.status) params.set('status', opts.status);
|
||||
if (opts?.nsfw) params.set('nsfw', 'true');
|
||||
if (opts?.search) params.set('search', opts.search);
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<AnalysisHistoryPage>(
|
||||
`/v1/admin/analysis/history${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
/** Per-model average analysis duration. */
|
||||
export type ModelDuration = { model: string | null; avg_ms: number | null; n: number };
|
||||
|
||||
/** Aggregate analysis timing/outcome roll-up over the selected window. */
|
||||
export type AnalysisMetrics = {
|
||||
n: number;
|
||||
ok: number;
|
||||
failed: number;
|
||||
avg_ms: number | null;
|
||||
by_model: ModelDuration[];
|
||||
};
|
||||
|
||||
/** GET /v1/admin/analysis/metrics — avg duration, success, by-model.
|
||||
* `days` windows the rows (0/omitted = all time). */
|
||||
export async function getAnalysisMetrics(days = 0): Promise<AnalysisMetrics> {
|
||||
const qs = days > 0 ? `?days=${days}` : '';
|
||||
return request<AnalysisMetrics>(`/v1/admin/analysis/metrics${qs}`);
|
||||
}
|
||||
|
||||
/** One live analysis event from the SSE stream. */
|
||||
export type AnalysisEvent =
|
||||
| {
|
||||
@@ -600,7 +787,9 @@ export type AnalysisEvent =
|
||||
kind: 'started' | 'completed' | 'failed';
|
||||
page_id: string;
|
||||
manga_id: string;
|
||||
manga_title: string;
|
||||
chapter_id: string;
|
||||
chapter_number: number;
|
||||
page_number: number;
|
||||
};
|
||||
|
||||
|
||||
229
frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte
Normal file
229
frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte
Normal file
@@ -0,0 +1,229 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import {
|
||||
listAnalysisHistory,
|
||||
type AnalysisHistoryRow
|
||||
} from '$lib/api/admin';
|
||||
|
||||
// Row click opens the parent's existing page-detail modal — history
|
||||
// reuses the same drill-down inspector as the coverage grid.
|
||||
let {
|
||||
onOpenDetail
|
||||
}: {
|
||||
onOpenDetail: (pageId: string) => void;
|
||||
} = $props();
|
||||
|
||||
const LIMIT = 25;
|
||||
|
||||
let rows = $state<AnalysisHistoryRow[]>([]);
|
||||
let total = $state(0);
|
||||
let page = $state(1);
|
||||
let statusFilter = $state<'' | 'done' | 'failed'>('');
|
||||
let nsfwOnly = $state(false);
|
||||
let search = $state('');
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
|
||||
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const resp = await listAnalysisHistory({
|
||||
status: statusFilter || undefined,
|
||||
nsfw: nsfwOnly,
|
||||
search: search.trim() || undefined,
|
||||
limit: LIMIT,
|
||||
offset: (page - 1) * LIMIT
|
||||
});
|
||||
rows = resp.items;
|
||||
total = resp.page.total ?? resp.items.length;
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : 'Failed to load history.';
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
|
||||
function applyFilters() {
|
||||
page = 1;
|
||||
load();
|
||||
}
|
||||
function onPageChange(p: number) {
|
||||
page = p;
|
||||
load();
|
||||
}
|
||||
function onSearchKey(e: KeyboardEvent) {
|
||||
if (e.key === 'Enter') applyFilters();
|
||||
}
|
||||
|
||||
function fmtAgo(iso: string | null): string {
|
||||
if (!iso) return '—';
|
||||
const secs = Math.round((Date.now() - new Date(iso).getTime()) / 1000);
|
||||
if (secs < 45) return 'just now';
|
||||
const mins = Math.round(secs / 60);
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hrs = Math.round(mins / 60);
|
||||
if (hrs < 24) return `${hrs}h ago`;
|
||||
return new Date(iso).toLocaleDateString();
|
||||
}
|
||||
</script>
|
||||
|
||||
<section class="history" data-testid="analysis-history">
|
||||
<div class="toolbar">
|
||||
<select
|
||||
bind:value={statusFilter}
|
||||
onchange={applyFilters}
|
||||
aria-label="Filter by status"
|
||||
data-testid="analysis-history-status"
|
||||
>
|
||||
<option value="">All statuses</option>
|
||||
<option value="done">done</option>
|
||||
<option value="failed">failed</option>
|
||||
</select>
|
||||
<label class="nsfw">
|
||||
<input
|
||||
type="checkbox"
|
||||
bind:checked={nsfwOnly}
|
||||
onchange={applyFilters}
|
||||
data-testid="analysis-history-nsfw"
|
||||
/>
|
||||
NSFW only
|
||||
</label>
|
||||
<input
|
||||
class="search"
|
||||
type="text"
|
||||
bind:value={search}
|
||||
placeholder="Search manga title…"
|
||||
onkeydown={onSearchKey}
|
||||
data-testid="analysis-history-search"
|
||||
/>
|
||||
<button type="button" onclick={applyFilters}>Search</button>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if loading}
|
||||
<p class="muted" data-testid="analysis-history-loading">Loading…</p>
|
||||
{:else if rows.length === 0}
|
||||
<p class="muted" data-testid="analysis-history-empty">No analyses match.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Page</th>
|
||||
<th>Model</th>
|
||||
<th class="num">Dur</th>
|
||||
<th>Analyzed</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each rows as r (r.page_id)}
|
||||
<tr
|
||||
class="clickable"
|
||||
onclick={() => onOpenDetail(r.page_id)}
|
||||
data-testid={`analysis-history-row-${r.page_id}`}
|
||||
title={r.error ?? ''}
|
||||
>
|
||||
<td><span class="status-pill {r.status}">{r.status}</span></td>
|
||||
<td>
|
||||
{r.manga_title} · Ch {r.chapter_number} · p{r.page_number}
|
||||
{#if r.is_nsfw}<span class="nsfw-tag">⚠ NSFW</span>{/if}
|
||||
</td>
|
||||
<td class="muted">{r.model ?? '—'}</td>
|
||||
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
||||
<td title={r.analyzed_at ? new Date(r.analyzed_at).toLocaleString() : ''}
|
||||
>{fmtAgo(r.analyzed_at)}</td
|
||||
>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pager {page} {totalPages} onChange={onPageChange} testid="analysis-history-pager" />
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.toolbar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
select,
|
||||
.search {
|
||||
height: 36px;
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.nsfw {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-1);
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
max-width: 52rem;
|
||||
}
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-2);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.num {
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
tr.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
tr.clickable:hover {
|
||||
background: var(--surface);
|
||||
}
|
||||
.status-pill {
|
||||
font-weight: var(--weight-semibold);
|
||||
font-size: var(--font-xs);
|
||||
padding: 1px var(--space-2);
|
||||
border-radius: var(--radius-pill);
|
||||
text-transform: uppercase;
|
||||
background: var(--surface-elevated);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.status-pill.done {
|
||||
background: color-mix(in srgb, #2e7d32 16%, transparent);
|
||||
color: #2e7d32;
|
||||
}
|
||||
.status-pill.failed {
|
||||
background: color-mix(in srgb, var(--danger) 16%, transparent);
|
||||
color: var(--danger);
|
||||
}
|
||||
.nsfw-tag {
|
||||
font-size: var(--font-xs);
|
||||
color: #b85e1a;
|
||||
margin-left: var(--space-1);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger);
|
||||
}
|
||||
</style>
|
||||
171
frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte
Normal file
171
frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte
Normal file
@@ -0,0 +1,171 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import { getAnalysisMetrics, type AnalysisMetrics } from '$lib/api/admin';
|
||||
|
||||
let days = $state(7);
|
||||
let metrics = $state<AnalysisMetrics | null>(null);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
|
||||
const successPct = $derived(
|
||||
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
|
||||
);
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
metrics = await getAnalysisMetrics(days);
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : 'Failed to load metrics.';
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
</script>
|
||||
|
||||
<section data-testid="analysis-metrics">
|
||||
<div class="winrow">
|
||||
<label>
|
||||
Window
|
||||
<select bind:value={days} onchange={load} data-testid="analysis-metrics-window">
|
||||
<option value={1}>24 hours</option>
|
||||
<option value={7}>7 days</option>
|
||||
<option value={30}>30 days</option>
|
||||
<option value={0}>All time</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{:else if loading}
|
||||
<p class="muted">Loading…</p>
|
||||
{:else if metrics}
|
||||
<div class="tiles">
|
||||
<div class="tile">
|
||||
<span class="label">Pages analyzed</span>
|
||||
<span class="value" data-testid="analysis-metrics-n">{metrics.n}</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="label">Avg duration</span>
|
||||
<span class="value" data-testid="analysis-metrics-avg"
|
||||
>{fmtDuration(metrics.avg_ms)}</span
|
||||
>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="label">Success</span>
|
||||
<span class="value">{successPct == null ? '—' : `${successPct}%`}</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="label">Failed</span>
|
||||
<span class="value">{metrics.failed}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2>By model</h2>
|
||||
{#if metrics.by_model.length === 0}
|
||||
<p class="muted">No completed analyses in this window.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Model</th>
|
||||
<th class="num">Avg</th>
|
||||
<th class="num">N</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each metrics.by_model as m (m.model)}
|
||||
<tr>
|
||||
<td>{m.model ?? '(unknown)'}</td>
|
||||
<td class="num">{fmtDuration(m.avg_ms)}</td>
|
||||
<td class="num">{m.n}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
{/if}
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.winrow {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
label {
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
select {
|
||||
height: 32px;
|
||||
margin-left: var(--space-1);
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.tiles {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
|
||||
gap: var(--space-3);
|
||||
max-width: 44rem;
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
.tile {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
}
|
||||
.tile .label {
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.tile .value {
|
||||
font-size: var(--font-lg);
|
||||
font-weight: var(--weight-semibold);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
h2 {
|
||||
margin: 0 0 var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
max-width: 32rem;
|
||||
}
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-2);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.num {
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger);
|
||||
}
|
||||
</style>
|
||||
293
frontend/src/lib/components/crawler/CrawlerHistoryTable.svelte
Normal file
293
frontend/src/lib/components/crawler/CrawlerHistoryTable.svelte
Normal file
@@ -0,0 +1,293 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import SearchBar from './SearchBar.svelte';
|
||||
import {
|
||||
listCrawlerJobHistory,
|
||||
type CrawlerHistoryRow,
|
||||
type CrawlerJobState,
|
||||
type CrawlerJobKind,
|
||||
type RequeueScope
|
||||
} from '$lib/api/admin';
|
||||
|
||||
// Requeue is owned by the parent (shared with the dead-jobs table); we
|
||||
// call it then reload so the row's new state shows. `busy` disables the
|
||||
// inline button during a parent-driven action.
|
||||
let {
|
||||
onRequeue,
|
||||
busy = false
|
||||
}: {
|
||||
onRequeue: (scope: RequeueScope) => Promise<void>;
|
||||
busy?: boolean;
|
||||
} = $props();
|
||||
|
||||
const LIMIT = 25;
|
||||
|
||||
let rows = $state<CrawlerHistoryRow[]>([]);
|
||||
let total = $state(0);
|
||||
let page = $state(1);
|
||||
let stateFilter = $state<'' | CrawlerJobState>('');
|
||||
let kindFilter = $state<'' | CrawlerJobKind>('');
|
||||
let search = $state('');
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let expanded = $state<string | null>(null);
|
||||
|
||||
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const resp = await listCrawlerJobHistory({
|
||||
state: stateFilter || undefined,
|
||||
kind: kindFilter || undefined,
|
||||
search: search.trim() || undefined,
|
||||
limit: LIMIT,
|
||||
offset: (page - 1) * LIMIT
|
||||
});
|
||||
rows = resp.items;
|
||||
total = resp.page.total ?? resp.items.length;
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : 'Failed to load history.';
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
|
||||
function applyFilters() {
|
||||
page = 1;
|
||||
load();
|
||||
}
|
||||
function onPageChange(p: number) {
|
||||
page = p;
|
||||
load();
|
||||
}
|
||||
async function requeue(scope: RequeueScope) {
|
||||
await onRequeue(scope);
|
||||
await load();
|
||||
}
|
||||
|
||||
/** A human "Manga · Ch N · pP" target, or the source key / em-dash. */
|
||||
function target(r: CrawlerHistoryRow): string {
|
||||
if (r.manga_title) {
|
||||
let s = r.manga_title;
|
||||
if (r.chapter_number != null) s += ` · Ch ${r.chapter_number}`;
|
||||
if (r.page_number != null) s += ` · p${r.page_number}`;
|
||||
return s;
|
||||
}
|
||||
return r.source_key ?? '—';
|
||||
}
|
||||
|
||||
function fmtAgo(iso: string): string {
|
||||
const then = new Date(iso).getTime();
|
||||
const secs = Math.round((Date.now() - then) / 1000);
|
||||
if (secs < 45) return 'just now';
|
||||
if (secs < 90) return '1m ago';
|
||||
const mins = Math.round(secs / 60);
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hrs = Math.round(mins / 60);
|
||||
if (hrs < 24) return `${hrs}h ago`;
|
||||
return new Date(iso).toLocaleDateString();
|
||||
}
|
||||
|
||||
const KINDS: { value: '' | CrawlerJobKind; label: string }[] = [
|
||||
{ value: '', label: 'All types' },
|
||||
{ value: 'sync_manga', label: 'sync_manga' },
|
||||
{ value: 'sync_chapter_list', label: 'sync_chapter_list' },
|
||||
{ value: 'sync_chapter_content', label: 'sync_chapter' },
|
||||
{ value: 'analyze_page', label: 'analyze_page' }
|
||||
];
|
||||
const STATES: { value: '' | CrawlerJobState; label: string }[] = [
|
||||
{ value: '', label: 'All states' },
|
||||
{ value: 'done', label: 'done' },
|
||||
{ value: 'dead', label: 'dead' },
|
||||
{ value: 'running', label: 'running' },
|
||||
{ value: 'pending', label: 'pending' }
|
||||
];
|
||||
</script>
|
||||
|
||||
<section class="history" data-testid="crawler-history">
|
||||
<div class="toolbar">
|
||||
<select
|
||||
bind:value={stateFilter}
|
||||
onchange={applyFilters}
|
||||
aria-label="Filter by state"
|
||||
data-testid="crawler-history-state"
|
||||
>
|
||||
{#each STATES as s (s.value)}<option value={s.value}>{s.label}</option>{/each}
|
||||
</select>
|
||||
<select
|
||||
bind:value={kindFilter}
|
||||
onchange={applyFilters}
|
||||
aria-label="Filter by type"
|
||||
data-testid="crawler-history-kind"
|
||||
>
|
||||
{#each KINDS as k (k.value)}<option value={k.value}>{k.label}</option>{/each}
|
||||
</select>
|
||||
<SearchBar
|
||||
bind:value={search}
|
||||
placeholder="Search manga / chapter…"
|
||||
onSearch={applyFilters}
|
||||
/>
|
||||
</div>
|
||||
<p class="muted note">
|
||||
Showing recent jobs — completed jobs are pruned after the retention
|
||||
window; dead jobs persist until requeued.
|
||||
</p>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
{#if loading}
|
||||
<p class="muted" data-testid="crawler-history-loading">Loading…</p>
|
||||
{:else if rows.length === 0}
|
||||
<p class="muted" data-testid="crawler-history-empty">No jobs match.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>State</th>
|
||||
<th>Type</th>
|
||||
<th>Target</th>
|
||||
<th class="num">Dur</th>
|
||||
<th>Att.</th>
|
||||
<th>Updated</th>
|
||||
<th class="actions"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each rows as r (r.id)}
|
||||
<tr
|
||||
class:clickable={!!r.last_error}
|
||||
onclick={() =>
|
||||
r.last_error && (expanded = expanded === r.id ? null : r.id)}
|
||||
data-testid={`crawler-history-row-${r.id}`}
|
||||
>
|
||||
<td>
|
||||
<span class="badge state-{r.state}">{r.state}</span>
|
||||
</td>
|
||||
<td class="kind">{r.kind ?? '—'}</td>
|
||||
<td>{target(r)}</td>
|
||||
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
||||
<td>{r.attempts}/{r.max_attempts}</td>
|
||||
<td title={new Date(r.updated_at).toLocaleString()}
|
||||
>{fmtAgo(r.updated_at)}</td
|
||||
>
|
||||
<td class="actions">
|
||||
{#if r.state === 'dead'}
|
||||
<button
|
||||
type="button"
|
||||
disabled={busy}
|
||||
onclick={(e) => {
|
||||
e.stopPropagation();
|
||||
requeue({ scope: 'job', job_id: r.id });
|
||||
}}
|
||||
data-testid={`crawler-history-requeue-${r.id}`}
|
||||
>
|
||||
⤴ Requeue
|
||||
</button>
|
||||
{/if}
|
||||
</td>
|
||||
</tr>
|
||||
{#if expanded === r.id && r.last_error}
|
||||
<tr class="errrow">
|
||||
<td colspan="7"><code>{r.last_error}</code></td>
|
||||
</tr>
|
||||
{/if}
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pager {page} {totalPages} onChange={onPageChange} testid="crawler-history-pager" />
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.toolbar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
select {
|
||||
height: 36px;
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.note {
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-2);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.kind {
|
||||
color: var(--text-muted);
|
||||
font-family: var(--font-mono, monospace);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.actions {
|
||||
text-align: right;
|
||||
}
|
||||
.num {
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
tr.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
tr.clickable:hover {
|
||||
background: var(--surface);
|
||||
}
|
||||
.errrow td {
|
||||
background: var(--surface);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.errrow code {
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger, #dc2626);
|
||||
}
|
||||
/* State dots reuse the page's shared badge palette. */
|
||||
.badge {
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.state-done {
|
||||
background: #dcfce7;
|
||||
color: #166534;
|
||||
border-color: #86efac;
|
||||
}
|
||||
.state-dead {
|
||||
background: color-mix(in srgb, var(--danger, #dc2626) 16%, transparent);
|
||||
color: var(--danger, #dc2626);
|
||||
border-color: color-mix(in srgb, var(--danger, #dc2626) 45%, transparent);
|
||||
}
|
||||
.state-running,
|
||||
.state-pending {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
border-color: #fcd34d;
|
||||
}
|
||||
</style>
|
||||
330
frontend/src/lib/components/crawler/CrawlerMetricsPanel.svelte
Normal file
330
frontend/src/lib/components/crawler/CrawlerMetricsPanel.svelte
Normal file
@@ -0,0 +1,330 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import {
|
||||
getCrawlerMetrics,
|
||||
listCrawlerOps,
|
||||
type OpSummary,
|
||||
type OpRow,
|
||||
type CrawlOp
|
||||
} from '$lib/api/admin';
|
||||
|
||||
const LIMIT = 25;
|
||||
|
||||
let days = $state(7);
|
||||
let summary = $state<OpSummary[]>([]);
|
||||
let summaryLoading = $state(true);
|
||||
|
||||
let rows = $state<OpRow[]>([]);
|
||||
let total = $state(0);
|
||||
let page = $state(1);
|
||||
let opFilter = $state<'' | CrawlOp>('');
|
||||
let outcomeFilter = $state<'' | 'ok' | 'failed'>('');
|
||||
let opsLoading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let expanded = $state<string | null>(null);
|
||||
|
||||
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
||||
|
||||
// Human labels + canonical order for the summary table.
|
||||
const OP_LABELS: Record<CrawlOp, string> = {
|
||||
manga_list: 'manga list walk',
|
||||
manga_detail: 'manga detail',
|
||||
manga_cover: 'manga cover',
|
||||
chapter: 'whole chapter'
|
||||
};
|
||||
const OP_ORDER: CrawlOp[] = ['manga_list', 'manga_detail', 'manga_cover', 'chapter'];
|
||||
const orderedSummary = $derived(
|
||||
OP_ORDER.map((op) => summary.find((s) => s.op === op)).filter(
|
||||
(s): s is OpSummary => s != null
|
||||
)
|
||||
);
|
||||
// Derived per-page crawl time from the chapter roll-up (Σms ÷ Σpages ≈
|
||||
// avg_ms ÷ avg_items). Shown as an indented sub-row under "whole chapter".
|
||||
const chapter = $derived(summary.find((s) => s.op === 'chapter'));
|
||||
const perPageMs = $derived(
|
||||
chapter && chapter.avg_ms != null && chapter.avg_items && chapter.avg_items > 0
|
||||
? chapter.avg_ms / chapter.avg_items
|
||||
: null
|
||||
);
|
||||
|
||||
function successPct(s: OpSummary): string {
|
||||
if (s.n === 0) return '—';
|
||||
return `${Math.round((s.ok / s.n) * 100)}%`;
|
||||
}
|
||||
|
||||
async function loadSummary() {
|
||||
summaryLoading = true;
|
||||
try {
|
||||
summary = (await getCrawlerMetrics(days)).summary;
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : 'Failed to load metrics.';
|
||||
} finally {
|
||||
summaryLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadOps() {
|
||||
opsLoading = true;
|
||||
try {
|
||||
const resp = await listCrawlerOps({
|
||||
op: opFilter || undefined,
|
||||
outcome: outcomeFilter || undefined,
|
||||
days,
|
||||
limit: LIMIT,
|
||||
offset: (page - 1) * LIMIT
|
||||
});
|
||||
rows = resp.items;
|
||||
total = resp.page.total ?? resp.items.length;
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : 'Failed to load operations.';
|
||||
} finally {
|
||||
opsLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
loadSummary();
|
||||
loadOps();
|
||||
});
|
||||
|
||||
function onWindowChange() {
|
||||
page = 1;
|
||||
loadSummary();
|
||||
loadOps();
|
||||
}
|
||||
function onOpsFilter() {
|
||||
page = 1;
|
||||
loadOps();
|
||||
}
|
||||
function onPageChange(p: number) {
|
||||
page = p;
|
||||
loadOps();
|
||||
}
|
||||
|
||||
function opLabel(op: CrawlOp): string {
|
||||
return OP_LABELS[op] ?? op;
|
||||
}
|
||||
function fmtAgo(iso: string): string {
|
||||
const secs = Math.round((Date.now() - new Date(iso).getTime()) / 1000);
|
||||
if (secs < 45) return 'just now';
|
||||
const mins = Math.round(secs / 60);
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hrs = Math.round(mins / 60);
|
||||
if (hrs < 24) return `${hrs}h ago`;
|
||||
return new Date(iso).toLocaleDateString();
|
||||
}
|
||||
</script>
|
||||
|
||||
<section data-testid="crawler-metrics">
|
||||
<div class="winrow">
|
||||
<label>
|
||||
Window
|
||||
<select
|
||||
bind:value={days}
|
||||
onchange={onWindowChange}
|
||||
data-testid="crawler-metrics-window"
|
||||
>
|
||||
<option value={1}>24 hours</option>
|
||||
<option value={7}>7 days</option>
|
||||
<option value={30}>30 days</option>
|
||||
<option value={0}>All time</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
<h2>Average durations by type</h2>
|
||||
{#if summaryLoading}
|
||||
<p class="muted">Loading…</p>
|
||||
{:else if orderedSummary.length === 0}
|
||||
<p class="muted" data-testid="crawler-metrics-empty">No operations recorded yet.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th class="num">Avg</th>
|
||||
<th class="num">N</th>
|
||||
<th class="num">OK / Fail</th>
|
||||
<th class="num">Success</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each orderedSummary as s (s.op)}
|
||||
<tr data-testid={`crawler-metrics-row-${s.op}`}>
|
||||
<td>{opLabel(s.op)}</td>
|
||||
<td class="num">{fmtDuration(s.avg_ms)}</td>
|
||||
<td class="num">{s.n}</td>
|
||||
<td class="num">{s.ok} / {s.failed}</td>
|
||||
<td class="num">{successPct(s)}</td>
|
||||
</tr>
|
||||
{#if s.op === 'chapter' && perPageMs != null}
|
||||
<tr class="derived" data-testid="crawler-metrics-perpage">
|
||||
<td>└ per page (≈)</td>
|
||||
<td class="num">{fmtDuration(perPageMs)}</td>
|
||||
<td class="num" colspan="3"
|
||||
>≈{Math.round(chapter?.avg_items ?? 0)} pages/chapter</td
|
||||
>
|
||||
</tr>
|
||||
{/if}
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
{/if}
|
||||
|
||||
<h2>Recent operations</h2>
|
||||
<div class="toolbar">
|
||||
<select bind:value={opFilter} onchange={onOpsFilter} aria-label="Filter by type">
|
||||
<option value="">All types</option>
|
||||
{#each OP_ORDER as op (op)}<option value={op}>{opLabel(op)}</option>{/each}
|
||||
</select>
|
||||
<select
|
||||
bind:value={outcomeFilter}
|
||||
onchange={onOpsFilter}
|
||||
aria-label="Filter by outcome"
|
||||
>
|
||||
<option value="">All outcomes</option>
|
||||
<option value="ok">ok</option>
|
||||
<option value="failed">failed</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{#if opsLoading}
|
||||
<p class="muted">Loading…</p>
|
||||
{:else if rows.length === 0}
|
||||
<p class="muted" data-testid="crawler-metrics-ops-empty">No operations match.</p>
|
||||
{:else}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Target</th>
|
||||
<th class="num">Dur</th>
|
||||
<th>Outcome</th>
|
||||
<th>When</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each rows as r (r.id)}
|
||||
<tr
|
||||
class:clickable={!!r.error}
|
||||
onclick={() => r.error && (expanded = expanded === r.id ? null : r.id)}
|
||||
data-testid={`crawler-op-${r.id}`}
|
||||
>
|
||||
<td>{opLabel(r.op)}</td>
|
||||
<td>
|
||||
{r.manga_title
|
||||
? `${r.manga_title}${r.chapter_number != null ? ` · Ch ${r.chapter_number}` : ''}`
|
||||
: '—'}
|
||||
</td>
|
||||
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
||||
<td>
|
||||
<span class="dot {r.outcome}"></span>{r.outcome}
|
||||
</td>
|
||||
<td title={new Date(r.finished_at).toLocaleString()}
|
||||
>{fmtAgo(r.finished_at)}</td
|
||||
>
|
||||
</tr>
|
||||
{#if expanded === r.id && r.error}
|
||||
<tr class="errrow"><td colspan="5"><code>{r.error}</code></td></tr>
|
||||
{/if}
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pager {page} {totalPages} onChange={onPageChange} testid="crawler-metrics-pager" />
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.winrow {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
label {
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
select {
|
||||
height: 32px;
|
||||
margin-left: var(--space-1);
|
||||
padding: 0 var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
h2 {
|
||||
margin: var(--space-4) 0 var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.toolbar {
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
max-width: 48rem;
|
||||
}
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-2);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.num {
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.derived td {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
tr.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
tr.clickable:hover {
|
||||
background: var(--surface);
|
||||
}
|
||||
.errrow td {
|
||||
background: var(--surface);
|
||||
}
|
||||
.errrow code {
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.dot {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
margin-right: 6px;
|
||||
vertical-align: middle;
|
||||
}
|
||||
.dot.ok {
|
||||
background: #2e7d32;
|
||||
}
|
||||
.dot.failed {
|
||||
background: var(--danger);
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger);
|
||||
}
|
||||
</style>
|
||||
29
frontend/src/lib/format.test.ts
Normal file
29
frontend/src/lib/format.test.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { fmtDuration } from './format';
|
||||
|
||||
describe('fmtDuration', () => {
|
||||
it('renders sub-second as ms', () => {
|
||||
expect(fmtDuration(0)).toBe('0ms');
|
||||
expect(fmtDuration(420)).toBe('420ms');
|
||||
expect(fmtDuration(999)).toBe('999ms');
|
||||
});
|
||||
|
||||
it('renders seconds with one decimal under 10s, none above', () => {
|
||||
expect(fmtDuration(6100)).toBe('6.1s');
|
||||
expect(fmtDuration(42000)).toBe('42s');
|
||||
});
|
||||
|
||||
it('renders minutes + zero-padded seconds', () => {
|
||||
expect(fmtDuration(62000)).toBe('1m 02s');
|
||||
expect(fmtDuration(125000)).toBe('2m 05s');
|
||||
});
|
||||
|
||||
it('rounds the seconds carry into minutes (no "1m 60s")', () => {
|
||||
expect(fmtDuration(119600)).toBe('2m 00s');
|
||||
});
|
||||
|
||||
it('renders null/undefined as an em-dash', () => {
|
||||
expect(fmtDuration(null)).toBe('—');
|
||||
expect(fmtDuration(undefined)).toBe('—');
|
||||
});
|
||||
});
|
||||
14
frontend/src/lib/format.ts
Normal file
14
frontend/src/lib/format.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
/** Human-readable duration from milliseconds.
|
||||
* `420 → "420ms"`, `6100 → "6.1s"`, `62000 → "1m 02s"`, `null → "—"`. */
|
||||
export function fmtDuration(ms: number | null | undefined): string {
|
||||
if (ms == null) return '—';
|
||||
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||
const secs = ms / 1000;
|
||||
if (secs < 60) return `${secs.toFixed(secs < 10 ? 1 : 0)}s`;
|
||||
// Round to whole seconds *before* splitting so e.g. 119_600ms renders
|
||||
// "2m 00s", not "1m 60s".
|
||||
const whole = Math.round(secs);
|
||||
const mins = Math.floor(whole / 60);
|
||||
const rem = whole % 60;
|
||||
return `${mins}m ${String(rem).padStart(2, '0')}s`;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import { onMount, onDestroy, tick } from 'svelte';
|
||||
import { ApiError } from '$lib/api/client';
|
||||
import {
|
||||
reenqueueAnalysis,
|
||||
@@ -19,9 +19,16 @@
|
||||
import { chapterLabel } from '$lib/api/chapters';
|
||||
import CoverageBadge from '$lib/components/CoverageBadge.svelte';
|
||||
import Modal from '$lib/components/Modal.svelte';
|
||||
import AnalysisHistoryTable from '$lib/components/analysis/AnalysisHistoryTable.svelte';
|
||||
import AnalysisMetricsPanel from '$lib/components/analysis/AnalysisMetricsPanel.svelte';
|
||||
import SegmentedControl from '$lib/components/SegmentedControl.svelte';
|
||||
|
||||
const LIMIT = 25;
|
||||
|
||||
// Segmented Live/History/Metrics view. Live = the SSE coverage
|
||||
// dashboard; History = a searchable log; Metrics = durations/averages.
|
||||
let view = $state<'live' | 'history' | 'metrics'>('live');
|
||||
|
||||
// Global enqueue toggle.
|
||||
let includeAnalyzed = $state(false);
|
||||
|
||||
@@ -65,6 +72,44 @@
|
||||
let source: EventSource | null = null;
|
||||
let sseErrors = 0;
|
||||
|
||||
// Global "now analyzing" pointer, driven by the `started` SSE event so
|
||||
// the operator sees what the worker is on even when nothing is
|
||||
// expanded. `phase` flips to done/failed for a moment on the matching
|
||||
// completed/failed event, then the banner clears.
|
||||
type NowAnalyzing = {
|
||||
manga_id: string;
|
||||
manga_title: string;
|
||||
chapter_id: string;
|
||||
chapter_number: number;
|
||||
page_id: string;
|
||||
page_number: number;
|
||||
phase: 'analyzing' | 'done' | 'failed';
|
||||
};
|
||||
let nowAnalyzing = $state<NowAnalyzing | null>(null);
|
||||
let clearTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
function scheduleClear(pageId: string) {
|
||||
if (clearTimer) clearTimeout(clearTimer);
|
||||
clearTimer = setTimeout(() => {
|
||||
// Don't clear if a newer page took over the banner.
|
||||
if (nowAnalyzing?.page_id === pageId) nowAnalyzing = null;
|
||||
}, 4000);
|
||||
}
|
||||
|
||||
/** Switch to Live, expand the now-analyzing chapter, scroll its chip. */
|
||||
async function jumpToNowAnalyzing() {
|
||||
const t = nowAnalyzing;
|
||||
if (!t) return;
|
||||
view = 'live';
|
||||
await tick();
|
||||
if (expandedManga !== t.manga_id) await toggleManga(t.manga_id);
|
||||
if (expandedChapter !== t.chapter_id) await toggleChapter(t.chapter_id);
|
||||
await tick();
|
||||
document
|
||||
.querySelector(`[data-testid="admin-analysis-page-${t.page_id}"]`)
|
||||
?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
void loadCoverage(true);
|
||||
openStream();
|
||||
@@ -73,6 +118,7 @@
|
||||
onDestroy(() => {
|
||||
source?.close();
|
||||
source = null;
|
||||
if (clearTimer) clearTimeout(clearTimer);
|
||||
});
|
||||
|
||||
function pushTicker(text: string) {
|
||||
@@ -155,16 +201,34 @@
|
||||
}
|
||||
if (ev.kind === 'started') {
|
||||
liveStatus = { ...liveStatus, [ev.page_id]: 'analyzing' };
|
||||
pushTicker(`Analyzing page ${ev.page_number}`);
|
||||
if (clearTimer) clearTimeout(clearTimer);
|
||||
nowAnalyzing = {
|
||||
manga_id: ev.manga_id,
|
||||
manga_title: ev.manga_title,
|
||||
chapter_id: ev.chapter_id,
|
||||
chapter_number: ev.chapter_number,
|
||||
page_id: ev.page_id,
|
||||
page_number: ev.page_number,
|
||||
phase: 'analyzing'
|
||||
};
|
||||
pushTicker(`Analyzing ${ev.manga_title} · p${ev.page_number}`);
|
||||
} else if (ev.kind === 'completed') {
|
||||
liveStatus = { ...liveStatus, [ev.page_id]: 'done' };
|
||||
if (!counted.has(ev.page_id)) {
|
||||
counted.add(ev.page_id);
|
||||
bumpCoverage(ev.manga_id, ev.chapter_id);
|
||||
}
|
||||
if (nowAnalyzing?.page_id === ev.page_id) {
|
||||
nowAnalyzing = { ...nowAnalyzing, phase: 'done' };
|
||||
scheduleClear(ev.page_id);
|
||||
}
|
||||
pushTicker(`✓ Analyzed page ${ev.page_number}`);
|
||||
} else if (ev.kind === 'failed') {
|
||||
liveStatus = { ...liveStatus, [ev.page_id]: 'failed' };
|
||||
if (nowAnalyzing?.page_id === ev.page_id) {
|
||||
nowAnalyzing = { ...nowAnalyzing, phase: 'failed' };
|
||||
scheduleClear(ev.page_id);
|
||||
}
|
||||
pushTicker(`✗ Failed page ${ev.page_number}`);
|
||||
}
|
||||
}
|
||||
@@ -393,6 +457,60 @@
|
||||
page's result. Updates stream live as pages are queued and processed.
|
||||
</p>
|
||||
|
||||
<div class="viewtabs">
|
||||
<SegmentedControl
|
||||
options={[
|
||||
{ label: 'Live', value: 'live' },
|
||||
{ label: 'History', value: 'history' },
|
||||
{ label: 'Metrics', value: 'metrics' }
|
||||
]}
|
||||
value={view}
|
||||
onchange={(v) => (view = v)}
|
||||
ariaLabel="Analysis view"
|
||||
testid="admin-analysis-tab"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{#if nowAnalyzing}
|
||||
<div
|
||||
class="now-analyzing {nowAnalyzing.phase}"
|
||||
data-testid="admin-analysis-now"
|
||||
aria-live="polite"
|
||||
>
|
||||
<span class="now-icon">
|
||||
{nowAnalyzing.phase === 'done' ? '✓' : nowAnalyzing.phase === 'failed' ? '✗' : '⟳'}
|
||||
</span>
|
||||
<span class="now-label">
|
||||
<strong
|
||||
>{nowAnalyzing.phase === 'done'
|
||||
? 'Analyzed'
|
||||
: nowAnalyzing.phase === 'failed'
|
||||
? 'Failed'
|
||||
: 'Now analyzing'}</strong
|
||||
>
|
||||
{nowAnalyzing.manga_title} · Ch {nowAnalyzing.chapter_number} · Page
|
||||
{nowAnalyzing.page_number}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
class="now-jump"
|
||||
onclick={jumpToNowAnalyzing}
|
||||
data-testid="admin-analysis-now-jump"
|
||||
>
|
||||
Jump →
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if view === 'history'}
|
||||
<AnalysisHistoryTable onOpenDetail={openDetail} />
|
||||
{/if}
|
||||
|
||||
{#if view === 'metrics'}
|
||||
<AnalysisMetricsPanel />
|
||||
{/if}
|
||||
|
||||
{#if view === 'live'}
|
||||
{#if ticker.length > 0}
|
||||
<ul class="ticker" data-testid="admin-analysis-ticker" aria-live="polite">
|
||||
{#each ticker as t (t.id)}
|
||||
@@ -586,6 +704,7 @@
|
||||
{#if error}
|
||||
<p class="error" role="alert" data-testid="admin-analysis-error">{error}</p>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<Modal
|
||||
open={detailOpen}
|
||||
@@ -708,6 +827,58 @@
|
||||
.live-pill.on .live-dot {
|
||||
animation: pulse 1.6s ease-in-out infinite;
|
||||
}
|
||||
.viewtabs {
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.now-analyzing {
|
||||
position: sticky;
|
||||
top: var(--space-2);
|
||||
z-index: 5;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
max-width: 44rem;
|
||||
margin-bottom: var(--space-3);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid color-mix(in srgb, #d4762a 45%, transparent);
|
||||
border-radius: var(--radius-md);
|
||||
background: color-mix(in srgb, #d4762a 12%, transparent);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.now-analyzing.done {
|
||||
border-color: color-mix(in srgb, #2e7d32 45%, transparent);
|
||||
background: color-mix(in srgb, #2e7d32 12%, transparent);
|
||||
}
|
||||
.now-analyzing.failed {
|
||||
border-color: color-mix(in srgb, var(--danger) 45%, transparent);
|
||||
background: color-mix(in srgb, var(--danger) 12%, transparent);
|
||||
}
|
||||
.now-icon {
|
||||
font-size: var(--font-lg);
|
||||
color: #b85e1a;
|
||||
}
|
||||
.now-analyzing.analyzing .now-icon {
|
||||
animation: spin 1.4s linear infinite;
|
||||
display: inline-block;
|
||||
}
|
||||
.now-analyzing.done .now-icon {
|
||||
color: #2e7d32;
|
||||
}
|
||||
.now-analyzing.failed .now-icon {
|
||||
color: var(--danger);
|
||||
}
|
||||
.now-label {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
.now-jump {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
@keyframes spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
.ticker {
|
||||
list-style: none;
|
||||
margin: 0 0 var(--space-3);
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
import ActiveJobsTable from '$lib/components/crawler/ActiveJobsTable.svelte';
|
||||
import MissingCoversTable from '$lib/components/crawler/MissingCoversTable.svelte';
|
||||
import DeadJobsTable from '$lib/components/crawler/DeadJobsTable.svelte';
|
||||
import CrawlerHistoryTable from '$lib/components/crawler/CrawlerHistoryTable.svelte';
|
||||
import CrawlerMetricsPanel from '$lib/components/crawler/CrawlerMetricsPanel.svelte';
|
||||
import SegmentedControl from '$lib/components/SegmentedControl.svelte';
|
||||
import SessionModal from '$lib/components/crawler/SessionModal.svelte';
|
||||
import RestartConfirmModal from '$lib/components/crawler/RestartConfirmModal.svelte';
|
||||
import RequeueAllConfirmModal from '$lib/components/crawler/RequeueAllConfirmModal.svelte';
|
||||
@@ -27,6 +30,10 @@
|
||||
type RequeueScope
|
||||
} from '$lib/api/admin';
|
||||
|
||||
// Segmented Live/History/Metrics view. Live keeps the SSE-driven
|
||||
// dashboard; History is a searchable job log; Metrics shows durations.
|
||||
let view = $state<'live' | 'history' | 'metrics'>('live');
|
||||
|
||||
let status: CrawlerStatus | null = $state(null);
|
||||
let error: string | null = $state(null);
|
||||
let notice: string | null = $state(null);
|
||||
@@ -387,6 +394,20 @@
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="viewtabs">
|
||||
<SegmentedControl
|
||||
options={[
|
||||
{ label: 'Live', value: 'live' },
|
||||
{ label: 'History', value: 'history' },
|
||||
{ label: 'Metrics', value: 'metrics' }
|
||||
]}
|
||||
value={view}
|
||||
onchange={(v) => (view = v)}
|
||||
ariaLabel="Crawler view"
|
||||
testid="crawler-tab"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
{/if}
|
||||
@@ -394,6 +415,11 @@
|
||||
<p class="notice" role="status">{notice}</p>
|
||||
{/if}
|
||||
|
||||
{#if view === 'history'}
|
||||
<CrawlerHistoryTable onRequeue={requeue} {busy} />
|
||||
{:else if view === 'metrics'}
|
||||
<CrawlerMetricsPanel />
|
||||
{:else}
|
||||
{#if status}
|
||||
<CrawlerHero {status} />
|
||||
|
||||
@@ -446,6 +472,7 @@
|
||||
onRequeue={requeue}
|
||||
onRequeueAll={() => (requeueAllModalOpen = true)}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<RestartConfirmModal
|
||||
open={restartModalOpen}
|
||||
@@ -488,6 +515,9 @@
|
||||
.livedot.on {
|
||||
color: var(--success, #0a7d2c);
|
||||
}
|
||||
.viewtabs {
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
.notice {
|
||||
color: var(--success, #0a7d2c);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
|
||||
Reference in New Issue
Block a user