fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(admin): observability — job history, live now-analyzing, durations & metrics (0.84.0) #6

Merged
fabi merged 1 commits from feat/observability-job-history into main 2026-06-16 12:21:13 +00:00
Conversation 0 Commits 1 Files Changed 41 +3655 -21
41 changed files with 3655 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 5fa4442904 - Show all commits

2
backend/Cargo.lock generated
View File

@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mangalord"
version = "0.82.0"
version = "0.84.0"
dependencies = [
"anyhow",
"argon2",

2
backend/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "mangalord"
version = "0.82.0"
version = "0.84.0"
edition = "2021"
default-run = "mangalord"

39
backend/migrations/0028_operation_metrics.sql Normal file
View File

@@ -0,0 +1,39 @@
-- Durable timing log for crawler operations + a duration column for page
-- analysis. The dashboard's job history shows *what* ran; this records *how
-- long it took* so operators can see per-operation durations and averages
-- broken down by type/granularity (manga list walk, manga detail, cover,
-- whole chapter — per-page crawl timing is derived from the chapter row's
-- `items` count rather than stored per image).
--
-- This table is durable on purpose: it outlives the `crawler_jobs` done-job
-- reaper so averages stay meaningful, and it captures the INLINE operations
-- (list walk, manga detail, cover) that are not queue jobs at all. Volume is
-- low (one row per manga / chapter / pass, not per image), so a generous
-- retention reaper is hygiene rather than a necessity.
CREATE TABLE crawl_metrics (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-- Operation granularity. analyze_page is intentionally absent — analysis
-- duration lives on page_analysis.duration_ms (one row per page already).
op text NOT NULL
CHECK (op IN ('manga_list', 'manga_detail', 'manga_cover', 'chapter')),
-- Best-effort target context for labeling/drill. SET NULL so deleting a
-- manga/chapter doesn't erase its historical timings.
manga_id uuid REFERENCES mangas(id) ON DELETE SET NULL,
chapter_id uuid REFERENCES chapters(id) ON DELETE SET NULL,
outcome text NOT NULL CHECK (outcome IN ('ok', 'failed')),
duration_ms bigint NOT NULL,
-- Unit count for the op: chapter = pages stored; manga_list = mangas
-- discovered. Drives the derived "per page" average. NULL when N/A.
items integer,
error text,
finished_at timestamptz NOT NULL DEFAULT now()
);
-- Per-type averages + the recent-ops log both filter by op and order by time.
CREATE INDEX crawl_metrics_op_time_idx ON crawl_metrics (op, finished_at DESC);
-- The window filter (last 24h / 7d / 30d) and the retention reaper scan by time.
CREATE INDEX crawl_metrics_time_idx ON crawl_metrics (finished_at);
-- Wall-clock the analysis worker spent on a page (vision dispatch). NULL for
-- pre-existing rows and any page analyzed before this migration.
ALTER TABLE page_analysis ADD COLUMN duration_ms bigint;

48
backend/src/analysis/daemon.rs
View File

@@ -216,16 +216,23 @@ impl WorkerContext {
}
}
// Resolve the page breadcrumb once so live events carry the
// manga/chapter/number the dashboard keys on. A missing breadcrumb
// (page deleted) just suppresses events — the dispatch still runs
// and acks normally.
let breadcrumb = repo::page::locate(&self.pool, page_id).await.ok().flatten();
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
// Resolve the labeled page breadcrumb once so live events carry the
// manga/chapter/number AND human labels the dashboard's "now
// analyzing" banner names. A missing breadcrumb (page deleted) just
// suppresses events — the dispatch still runs and acks normally.
let breadcrumb = repo::page::locate_labeled(&self.pool, page_id)
.await
.ok()
.flatten();
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
breadcrumb.clone()
{
self.events.publish(AnalysisEvent::Started {
page_id,
manga_id,
manga_title,
chapter_id,
chapter_number,
page_number,
});
}
@@ -246,8 +253,10 @@ impl WorkerContext {
})
};
let started = std::time::Instant::now();
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(page_id)).catch_unwind();
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
let elapsed_ms = started.elapsed().as_millis() as i64;
heartbeat.abort();
// Flatten timeout / panic / dispatch error into one Result + message.
@@ -258,11 +267,27 @@ impl WorkerContext {
Ok(Ok(Ok(()))) => Ok(()),
};
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
if let Some((manga_id, manga_title, chapter_id, chapter_number, page_number)) =
breadcrumb
{
let event = if result.is_ok() {
AnalysisEvent::Completed { page_id, manga_id, chapter_id, page_number }
AnalysisEvent::Completed {
page_id,
manga_id,
manga_title,
chapter_id,
chapter_number,
page_number,
}
} else {
AnalysisEvent::Failed { page_id, manga_id, chapter_id, page_number }
AnalysisEvent::Failed {
page_id,
manga_id,
manga_title,
chapter_id,
chapter_number,
page_number,
}
};
self.events.publish(event);
}
@@ -294,6 +319,11 @@ impl WorkerContext {
}
}
}
// Stamp how long the vision dispatch took. Best-effort and ordered
// after the row is written (persist on success / mark_failed on
// terminal failure); a transient failure with no row updates nothing.
let _ = repo::page_analysis::record_duration(&self.pool, page_id, elapsed_ms).await;
}
}

32
backend/src/analysis/events.rs
View File

@@ -25,25 +25,33 @@ pub enum AnalysisEvent {
manga_id: Option<Uuid>,
chapter_id: Option<Uuid>,
},
/// The worker began analyzing a page.
/// The worker began analyzing a page. Carries the manga title and
/// chapter number (alongside the ids) so the dashboard's "now
/// analyzing" banner can name the target without a lookup.
Started {
page_id: Uuid,
manga_id: Uuid,
manga_title: String,
chapter_id: Uuid,
chapter_number: i32,
page_number: i32,
},
/// The worker finished a page successfully.
Completed {
page_id: Uuid,
manga_id: Uuid,
manga_title: String,
chapter_id: Uuid,
chapter_number: i32,
page_number: i32,
},
/// The worker failed a page (this attempt).
Failed {
page_id: Uuid,
manga_id: Uuid,
manga_title: String,
chapter_id: Uuid,
chapter_number: i32,
page_number: i32,
},
}
@@ -76,3 +84,25 @@ impl Default for AnalysisEvents {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn started_serializes_kind_and_labels() {
let ev = AnalysisEvent::Started {
page_id: Uuid::nil(),
manga_id: Uuid::nil(),
manga_title: "Berserk".into(),
chapter_id: Uuid::nil(),
chapter_number: 12,
page_number: 7,
};
let v = serde_json::to_value(&ev).unwrap();
assert_eq!(v["kind"], "started");
assert_eq!(v["manga_title"], "Berserk");
assert_eq!(v["chapter_number"], 12);
assert_eq!(v["page_number"], 7);
}
}

78
backend/src/api/admin/analysis.rs
View File

@@ -26,7 +26,8 @@ use crate::api::pagination::PagedResponse;
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::domain::page_analysis::{
ChapterCoverage, MangaCoverage, PageAnalysisDetail, PageStatusItem,
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, MangaCoverage, PageAnalysisDetail,
PageStatusItem,
};
use crate::error::{AppError, AppResult};
use crate::repo;
@@ -41,6 +42,8 @@ pub fn routes() -> Router<AppState> {
.route("/admin/analysis/mangas/:id/chapters", get(coverage_chapters))
.route("/admin/analysis/chapters/:id/pages", get(chapter_pages))
.route("/admin/analysis/pages/:id", get(page_detail))
.route("/admin/analysis/history", get(list_history))
.route("/admin/analysis/metrics", get(metrics))
.route("/admin/analysis/status/stream", get(stream_status))
}
@@ -149,6 +152,79 @@ struct ItemsResponse<T> {
items: Vec<T>,
}
/// Status/NSFW filters + pagination/search for the analysis history list.
#[derive(Debug, Deserialize, Default)]
pub struct HistoryParams {
#[serde(default)]
pub status: Option<String>,
#[serde(default)]
pub nsfw: bool,
#[serde(default)]
pub search: Option<String>,
#[serde(default = "default_coverage_limit")]
pub limit: i64,
#[serde(default)]
pub offset: i64,
}
async fn list_history(
State(state): State<AppState>,
_admin: RequireAdmin,
Query(params): Query<HistoryParams>,
) -> AppResult<Json<PagedResponse<AnalysisHistoryRow>>> {
let limit = params.limit.clamp(1, 100);
let offset = params.offset.max(0);
let status = params
.status
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty());
let search = params
.search
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty());
let (items, total) = repo::page_analysis::list_history(
&state.db,
repo::page_analysis::AnalysisHistoryFilter {
status,
nsfw_only: params.nsfw,
search,
},
limit,
offset,
)
.await?;
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
}
/// `?days=` window for the metrics endpoints. `0` (or absent) = all time.
#[derive(Debug, Deserialize, Default)]
pub struct MetricsParams {
#[serde(default)]
pub days: i64,
}
/// Convert a `days` window param into a lower-bound timestamp. `<= 0` → all
/// time (`None`); otherwise `now - days` (capped at a year).
pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
if days <= 0 {
None
} else {
Some(chrono::Utc::now() - chrono::Duration::days(days.min(365)))
}
}
async fn metrics(
State(state): State<AppState>,
_admin: RequireAdmin,
Query(params): Query<MetricsParams>,
) -> AppResult<Json<AnalysisMetrics>> {
let since = window_since(params.days);
let m = repo::page_analysis::analysis_metrics(&state.db, since).await?;
Ok(Json(m))
}
#[derive(Debug, Deserialize)]
pub struct ReenqueueBody {
/// Skip pages that already have a `done` analysis row. Defaults to

63
backend/src/api/admin/crawler/history.rs Normal file
View File

@@ -0,0 +1,63 @@
//! GET /admin/crawler/history — unified, searchable, filterable job log.
//!
//! A pure DB-derived read over the `crawler_jobs` table across every state
//! and kind. Drives the "History" tab on the crawler dashboard. Depth is
//! bounded by the done-job reaper (recent window); `dead` jobs persist.
use axum::extract::{Query, State};
use axum::routing::get;
use axum::{Json, Router};
use serde::Deserialize;
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::error::AppResult;
use crate::repo;
use crate::repo::crawler::{JobHistoryFilter, JobHistoryRow};
use super::default_limit;
pub(super) fn routes() -> Router<AppState> {
Router::new().route("/admin/crawler/history", get(list_history))
}
/// State + kind filters and pagination/search for the history list.
#[derive(Debug, Deserialize, Default)]
struct HistoryParams {
#[serde(default)]
state: Option<String>,
#[serde(default)]
kind: Option<String>,
#[serde(default)]
search: Option<String>,
#[serde(default = "default_limit")]
limit: i64,
#[serde(default)]
offset: i64,
}
async fn list_history(
State(state): State<AppState>,
_admin: RequireAdmin,
Query(params): Query<HistoryParams>,
) -> AppResult<Json<crate::api::pagination::PagedResponse<JobHistoryRow>>> {
let limit = params.limit.clamp(1, 200);
let offset = params.offset.max(0);
let state_f = params.state.filter(|s| !s.trim().is_empty());
let kind_f = params.kind.filter(|s| !s.trim().is_empty());
let search_f = params.search.filter(|s| !s.trim().is_empty());
let (items, total) = repo::crawler::list_job_history(
&state.db,
JobHistoryFilter {
state: state_f.as_deref(),
kind: kind_f.as_deref(),
search: search_f.as_deref(),
},
limit,
offset,
)
.await?;
Ok(Json(crate::api::pagination::PagedResponse::with_total(
items, limit, offset, total,
)))
}

88
backend/src/api/admin/crawler/metrics.rs Normal file
View File

@@ -0,0 +1,88 @@
//! GET /admin/crawler/metrics — per-type average durations + success
//! GET /admin/crawler/metrics/ops — paginated recent timed-operations log
//!
//! Pure DB reads over the durable `crawl_metrics` table. Drive the "Metrics"
//! tab on the crawler dashboard. `days` windows the rows (`0`/absent = all).
use axum::extract::{Query, State};
use axum::routing::get;
use axum::{Json, Router};
use serde::{Deserialize, Serialize};
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::domain::crawl_metrics::{OpRow, OpSummary};
use crate::error::AppResult;
use crate::repo;
use crate::repo::crawl_metrics::OpFilter;
use super::default_limit;
// Reuse the analysis handler's window helper so both endpoints clamp `days`
// identically.
use crate::api::admin::analysis::window_since;
pub(super) fn routes() -> Router<AppState> {
Router::new()
.route("/admin/crawler/metrics", get(summary))
.route("/admin/crawler/metrics/ops", get(list_ops))
}
#[derive(Debug, Deserialize, Default)]
struct SummaryParams {
#[serde(default)]
days: i64,
}
#[derive(Debug, Serialize)]
struct SummaryResponse {
summary: Vec<OpSummary>,
}
async fn summary(
State(state): State<AppState>,
_admin: RequireAdmin,
Query(params): Query<SummaryParams>,
) -> AppResult<Json<SummaryResponse>> {
let since = window_since(params.days);
let summary = repo::crawl_metrics::summary(&state.db, since).await?;
Ok(Json(SummaryResponse { summary }))
}
#[derive(Debug, Deserialize, Default)]
struct OpsParams {
#[serde(default)]
op: Option<String>,
#[serde(default)]
outcome: Option<String>,
#[serde(default)]
days: i64,
#[serde(default = "default_limit")]
limit: i64,
#[serde(default)]
offset: i64,
}
async fn list_ops(
State(state): State<AppState>,
_admin: RequireAdmin,
Query(params): Query<OpsParams>,
) -> AppResult<Json<crate::api::pagination::PagedResponse<OpRow>>> {
let limit = params.limit.clamp(1, 200);
let offset = params.offset.max(0);
let op = params.op.filter(|s| !s.trim().is_empty());
let outcome = params.outcome.filter(|s| !s.trim().is_empty());
let (items, total) = repo::crawl_metrics::list_ops(
&state.db,
OpFilter {
op: op.as_deref(),
outcome: outcome.as_deref(),
since: window_since(params.days),
},
limit,
offset,
)
.await?;
Ok(Json(crate::api::pagination::PagedResponse::with_total(
items, limit, offset, total,
)))
}

4
backend/src/api/admin/crawler/mod.rs
View File

@@ -16,6 +16,8 @@
mod backlog;
mod control;
mod dead_jobs;
mod history;
mod metrics;
mod status;
use axum::Router;
@@ -29,6 +31,8 @@ pub fn routes() -> Router<AppState> {
.merge(control::routes())
.merge(dead_jobs::routes())
.merge(backlog::routes())
.merge(history::routes())
.merge(metrics::routes())
}
/// Default page size for the backlog list endpoints.

1
backend/src/app.rs
View File

@@ -666,6 +666,7 @@ async fn spawn_crawler_daemon(
daily_at: cfg.daily_at,
tz: cfg.tz,
retention_days: cfg.retention_days,
metrics_retention_days: cfg.metrics_retention_days,
session_expired,
status: status.clone(),
job_timeout: cfg.job_timeout,

8
backend/src/config.rs
View File

@@ -321,6 +321,10 @@ pub struct CrawlerConfig {
pub idle_timeout: Duration,
pub chapter_workers: usize,
pub retention_days: u32,
/// Days to keep `crawl_metrics` timing rows before reaping. `0`
/// disables the reaper. Defaults to 90 (volume is low — one row per
/// manga/chapter/pass, not per image). `CRAWL_METRICS_RETENTION_DAYS`.
pub metrics_retention_days: u32,
pub start_url: Option<String>,
pub rate_ms: u64,
pub cdn_host: Option<String>,
@@ -379,6 +383,7 @@ impl Default for CrawlerConfig {
idle_timeout: Duration::from_secs(600),
chapter_workers: 1,
retention_days: 7,
metrics_retention_days: 90,
start_url: None,
rate_ms: 1000,
cdn_host: None,
@@ -500,6 +505,7 @@ impl CrawlerConfig {
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
metrics_retention_days: env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32,
start_url,
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
cdn_host,
@@ -583,7 +589,7 @@ fn build_download_allowlist(
allow
}
fn env_u64(name: &str, default: u64) -> u64 {
pub(crate) fn env_u64(name: &str, default: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|s| s.parse().ok())

61
backend/src/crawler/content.rs
View File

@@ -197,8 +197,69 @@ where
/// On any failure the chapter stays at `page_count = 0` (no partial
/// rows) and the blobs already written are deleted best-effort by
/// [`cleanup_orphans`], so a retry starts clean.
/// Timing wrapper over [`sync_chapter_content_inner`]: records one `chapter`
/// row in `crawl_metrics` with the wall-clock and outcome. Best-effort — a
/// metrics-write failure never fails the chapter sync. `Skipped` /
/// `SessionExpired` perform no fetch, so they aren't timed.
#[allow(clippy::too_many_arguments)]
pub async fn sync_chapter_content(
browser: &chromiumoxide::Browser,
db: &PgPool,
storage: &dyn Storage,
http: &reqwest::Client,
rate: &HostRateLimiters,
chapter_id: Uuid,
manga_id: Uuid,
source_url: &str,
force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
tor: Option<&crate::crawler::tor::TorController>,
progress: Option<&crate::crawler::status::StatusHandle>,
enqueue_analysis: bool,
) -> anyhow::Result<SyncOutcome> {
let started = std::time::Instant::now();
let result = sync_chapter_content_inner(
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
)
.await;
let duration_ms = started.elapsed().as_millis() as i64;
match &result {
Ok(SyncOutcome::Fetched { pages }) => {
let _ = crate::repo::crawl_metrics::record(
db,
crate::repo::crawl_metrics::OP_CHAPTER,
Some(manga_id),
Some(chapter_id),
"ok",
duration_ms,
Some(*pages as i32),
None,
)
.await;
}
Err(e) => {
let _ = crate::repo::crawl_metrics::record(
db,
crate::repo::crawl_metrics::OP_CHAPTER,
Some(manga_id),
Some(chapter_id),
"failed",
duration_ms,
None,
Some(&format!("{e:#}")),
)
.await;
}
// Skipped / SessionExpired: no fetch performed → not a timed op.
Ok(_) => {}
}
result
}
#[allow(clippy::too_many_arguments)]
async fn sync_chapter_content_inner(
browser: &chromiumoxide::Browser,
db: &PgPool,
storage: &dyn Storage,

9
backend/src/crawler/daemon.rs
View File

@@ -86,6 +86,7 @@ pub struct DaemonConfig {
pub daily_at: NaiveTime,
pub tz: Tz,
pub retention_days: u32,
pub metrics_retention_days: u32,
pub session_expired: Arc<AtomicBool>,
/// Live status surface updated by the cron + workers.
pub status: StatusHandle,
@@ -139,6 +140,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
daily_at,
tz,
retention_days,
metrics_retention_days,
session_expired,
status,
job_timeout,
@@ -152,6 +154,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
daily_at,
tz,
retention_days,
metrics_retention_days,
metadata,
status: status.clone(),
};
@@ -190,6 +193,7 @@ struct CronContext {
daily_at: NaiveTime,
tz: Tz,
retention_days: u32,
metrics_retention_days: u32,
metadata: Arc<dyn MetadataPass>,
status: StatusHandle,
}
@@ -271,6 +275,7 @@ impl CronContext {
let metadata = &self.metadata;
let pool = &self.pool;
let retention_days = self.retention_days;
let metrics_retention_days = self.metrics_retention_days;
let status = &self.status;
let body = async move {
match metadata.run().await {
@@ -290,6 +295,10 @@ impl CronContext {
Ok(n) => tracing::info!(reaped = n, "cron: done-job reaper finished"),
Err(e) => tracing::error!(?e, "cron: done-job reaper failed"),
}
match crate::repo::crawl_metrics::reap(pool, metrics_retention_days).await {
Ok(n) => tracing::info!(reaped = n, "cron: crawl-metrics reaper finished"),
Err(e) => tracing::error!(?e, "cron: crawl-metrics reaper failed"),
}
if let Err(e) = write_last_tick(pool, Utc::now()).await {
tracing::warn!(?e, "cron: persist last_metadata_tick_at failed");
}

70
backend/src/crawler/pipeline.rs
View File

@@ -118,6 +118,7 @@ pub async fn run_metadata_pass(
status: Option<&crate::crawler::status::StatusHandle>,
tor: Option<&crate::crawler::tor::TorController>,
) -> anyhow::Result<MetadataStats> {
let pass_started = std::time::Instant::now();
let lease = browser_manager
.acquire()
.await
@@ -243,6 +244,7 @@ pub async fn run_metadata_pass(
key = %r.source_manga_key,
"fetching metadata"
);
let detail_started = std::time::Instant::now();
let manga = match source.fetch_manga(&ctx, &r).await {
Ok(m) => {
consecutive_failures = 0;
@@ -255,6 +257,19 @@ pub async fn run_metadata_pass(
error = ?e,
"fetch_manga failed"
);
// Detail-fetch timing (failed). manga_id is unknown — the
// manga was never upserted.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
None,
None,
"failed",
detail_started.elapsed().as_millis() as i64,
None,
Some(&format!("{e:#}")),
)
.await;
stats.mangas_failed += 1;
consecutive_failures += 1;
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
@@ -270,6 +285,9 @@ pub async fn run_metadata_pass(
continue;
}
};
// Detail fetch succeeded; stash its duration to record once the
// manga_id is known (after upsert).
let detail_ms = detail_started.elapsed().as_millis() as i64;
// Partial-render guard: an empty chapter list paired with a
// prior count > 0 is overwhelmingly a chromium snapshot
@@ -330,6 +348,18 @@ pub async fn run_metadata_pass(
}
};
stats.upserted += 1;
// Detail-fetch timing (ok), now that we have the manga_id.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_DETAIL,
Some(upsert.manga_id),
None,
"ok",
detail_ms,
None,
None,
)
.await;
// Record success in the dedup set. Cover and chapter-sync
// failures below are non-fatal and don't roll this back —
// metadata is the durable source of truth for the dedup.
@@ -358,6 +388,7 @@ pub async fn run_metadata_pass(
manga_title: manga.title.clone(),
})
});
let cover_started = std::time::Instant::now();
let cover_result = download_and_store_cover(
db,
storage,
@@ -370,6 +401,18 @@ pub async fn run_metadata_pass(
max_image_bytes,
)
.await;
let cover_ms = cover_started.elapsed().as_millis() as i64;
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_COVER,
Some(upsert.manga_id),
None,
if cover_result.is_ok() { "ok" } else { "failed" },
cover_ms,
None,
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
)
.await;
match cover_result {
Ok(()) => stats.covers_fetched += 1,
Err(e) => tracing::warn!(
@@ -458,6 +501,21 @@ pub async fn run_metadata_pass(
"metadata pass complete"
);
// Record the whole-list-walk timing (best-effort). `items` = mangas
// discovered this pass; outcome is `failed` only when the failure breaker
// tripped (a degraded walk), else `ok`.
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_LIST,
None,
None,
if hit_failure_breaker { "failed" } else { "ok" },
pass_started.elapsed().as_millis() as i64,
Some(stats.discovered as i32),
None,
)
.await;
drop(lease);
Ok(stats)
}
@@ -694,6 +752,7 @@ pub async fn backfill_missing_covers(
manga_title: manga.title.clone(),
})
});
let cover_started = std::time::Instant::now();
let cover_result = download_and_store_cover(
db,
storage,
@@ -706,6 +765,17 @@ pub async fn backfill_missing_covers(
max_image_bytes,
)
.await;
let _ = repo::crawl_metrics::record(
db,
repo::crawl_metrics::OP_MANGA_COVER,
Some(entry.manga_id),
None,
if cover_result.is_ok() { "ok" } else { "failed" },
cover_started.elapsed().as_millis() as i64,
None,
cover_result.as_ref().err().map(|e| format!("{e:#}")).as_deref(),
)
.await;
match cover_result {
Ok(()) => stats.fetched += 1,
Err(e) => {

43
backend/src/domain/crawl_metrics.rs Normal file
View File

@@ -0,0 +1,43 @@
//! Timing metrics for crawler operations (see `crawl_metrics` table, 0028).
//!
//! One [`OpRow`] per completed operation (manga list walk, manga detail,
//! cover, whole chapter); [`OpSummary`] is the per-type average roll-up that
//! drives the admin Metrics tab. Per-page crawl timing is derived on the
//! client from the `chapter` summary's `avg_items` (pages/chapter), not
//! stored per image. Analysis duration lives on `page_analysis`, not here.
use chrono::{DateTime, Utc};
use serde::Serialize;
use sqlx::FromRow;
use uuid::Uuid;
/// Per-operation-type average roll-up over a time window.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct OpSummary {
/// `manga_list` | `manga_detail` | `manga_cover` | `chapter`.
pub op: String,
/// Mean duration in milliseconds (`NULL`→absent only when `n = 0`).
pub avg_ms: Option<f64>,
/// Total operations in the window.
pub n: i64,
pub ok: i64,
pub failed: i64,
/// Mean `items` (pages/chapter, mangas/list-walk); `None` when unused.
pub avg_items: Option<f64>,
}
/// One timed operation, resolved to manga/chapter labels for the recent-ops log.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct OpRow {
pub id: Uuid,
pub op: String,
pub manga_id: Option<Uuid>,
pub manga_title: Option<String>,
pub chapter_id: Option<Uuid>,
pub chapter_number: Option<i32>,
pub outcome: String,
pub duration_ms: i64,
pub items: Option<i32>,
pub error: Option<String>,
pub finished_at: DateTime<Utc>,
}

2
backend/src/domain/mod.rs
View File

@@ -4,6 +4,7 @@ pub mod author;
pub mod bookmark;
pub mod chapter;
pub mod collection;
pub mod crawl_metrics;
pub mod genre;
pub mod manga;
pub mod page;
@@ -25,6 +26,7 @@ pub use author::{Author, AuthorRef, AuthorWithCount};
pub use bookmark::{Bookmark, BookmarkSummary};
pub use chapter::Chapter;
pub use collection::{Collection, CollectionPageItem, CollectionSummary};
pub use crawl_metrics::{OpRow, OpSummary};
pub use genre::{Genre, GenreRef};
pub use manga::{Manga, MangaCard, MangaDetail};
pub use page::Page;

40
backend/src/domain/page_analysis.rs
View File

@@ -157,6 +157,46 @@ pub struct PageStatusItem {
pub status: String,
}
/// One row in the admin analysis-history table: a completed or failed
/// analysis pass resolved to its page/chapter/manga context. Sourced from
/// the persistent `page_analysis` table (not the reaped job queue), so it
/// survives indefinitely. `status` is `done` | `failed`.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct AnalysisHistoryRow {
pub page_id: Uuid,
pub page_number: i32,
pub chapter_id: Uuid,
pub chapter_number: i32,
pub manga_id: Uuid,
pub manga_title: String,
pub status: String,
pub is_nsfw: bool,
pub model: Option<String>,
pub error: Option<String>,
pub analyzed_at: Option<DateTime<Utc>>,
/// Wall-clock the worker spent on this page; `None` for rows analyzed
/// before duration tracking landed.
pub duration_ms: Option<i64>,
}
/// Per-model average analysis duration (admin Metrics tab).
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct ModelDuration {
pub model: Option<String>,
pub avg_ms: Option<f64>,
pub n: i64,
}
/// Aggregate analysis timing/outcome roll-up over a time window.
#[derive(Debug, Clone, Serialize)]
pub struct AnalysisMetrics {
pub n: i64,
pub ok: i64,
pub failed: i64,
pub avg_ms: Option<f64>,
pub by_model: Vec<ModelDuration>,
}
/// One OCR line in the page-detail view.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct OcrLine {

158
backend/src/repo/crawl_metrics.rs Normal file
View File

@@ -0,0 +1,158 @@
//! DB access for the `crawl_metrics` timing log (migration 0028).
//!
//! `record` is the best-effort write called from each crawl operation;
//! `summary` and `list_ops` are the admin Metrics-tab reads. Mirrors the
//! plain-fn + `query_as` style used across `repo`.
use chrono::{DateTime, Utc};
use sqlx::PgPool;
use uuid::Uuid;
use crate::domain::crawl_metrics::{OpRow, OpSummary};
/// Operation kinds, kept as consts so call sites and the CHECK constraint
/// don't drift from a free-typed literal.
pub const OP_MANGA_LIST: &str = "manga_list";
pub const OP_MANGA_DETAIL: &str = "manga_detail";
pub const OP_MANGA_COVER: &str = "manga_cover";
pub const OP_CHAPTER: &str = "chapter";
/// Insert one completed-operation timing row. Best-effort: callers log and
/// continue on error so a metrics-write failure never fails the actual crawl
/// work. `outcome` is `"ok"` or `"failed"`.
#[allow(clippy::too_many_arguments)]
pub async fn record(
pool: &PgPool,
op: &str,
manga_id: Option<Uuid>,
chapter_id: Option<Uuid>,
outcome: &str,
duration_ms: i64,
items: Option<i32>,
error: Option<&str>,
) -> sqlx::Result<()> {
sqlx::query(
"INSERT INTO crawl_metrics \
(op, manga_id, chapter_id, outcome, duration_ms, items, error) \
VALUES ($1, $2, $3, $4, $5, $6, $7)",
)
.bind(op)
.bind(manga_id)
.bind(chapter_id)
.bind(outcome)
.bind(duration_ms)
.bind(items)
.bind(error)
.execute(pool)
.await?;
Ok(())
}
/// Per-op average roll-up over an optional time window (`since = None` → all
/// time). One row per `op` that has any metric in the window, with mean
/// duration, counts, success split, and mean `items`.
pub async fn summary(
pool: &PgPool,
since: Option<DateTime<Utc>>,
) -> sqlx::Result<Vec<OpSummary>> {
sqlx::query_as::<_, OpSummary>(
r#"
SELECT
op,
AVG(duration_ms)::float8 AS avg_ms,
COUNT(*) AS n,
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
AVG(items)::float8 AS avg_items
FROM crawl_metrics
WHERE ($1::timestamptz IS NULL OR finished_at >= $1)
GROUP BY op
ORDER BY op
"#,
)
.bind(since)
.fetch_all(pool)
.await
}
/// Filters for [`list_ops`]. `None`/empty widens the scope.
#[derive(Debug, Default, Clone)]
pub struct OpFilter<'a> {
pub op: Option<&'a str>,
pub outcome: Option<&'a str>,
pub since: Option<DateTime<Utc>>,
}
/// Paginated, newest-first recent-operations log resolved to manga/chapter
/// labels. Returns the page slice plus the filtered total.
pub async fn list_ops(
pool: &PgPool,
filter: OpFilter<'_>,
limit: i64,
offset: i64,
) -> sqlx::Result<(Vec<OpRow>, i64)> {
let items = sqlx::query_as::<_, OpRow>(
r#"
SELECT
cm.id,
cm.op,
cm.manga_id,
m.title AS manga_title,
cm.chapter_id,
c.number AS chapter_number,
cm.outcome,
cm.duration_ms,
cm.items,
cm.error,
cm.finished_at
FROM crawl_metrics cm
LEFT JOIN mangas m ON m.id = cm.manga_id
LEFT JOIN chapters c ON c.id = cm.chapter_id
WHERE ($1::text IS NULL OR cm.op = $1)
AND ($2::text IS NULL OR cm.outcome = $2)
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
ORDER BY cm.finished_at DESC
LIMIT $4 OFFSET $5
"#,
)
.bind(filter.op)
.bind(filter.outcome)
.bind(filter.since)
.bind(limit)
.bind(offset)
.fetch_all(pool)
.await?;
let (total,): (i64,) = sqlx::query_as(
r#"
SELECT COUNT(*)
FROM crawl_metrics cm
WHERE ($1::text IS NULL OR cm.op = $1)
AND ($2::text IS NULL OR cm.outcome = $2)
AND ($3::timestamptz IS NULL OR cm.finished_at >= $3)
"#,
)
.bind(filter.op)
.bind(filter.outcome)
.bind(filter.since)
.fetch_one(pool)
.await?;
Ok((items, total))
}
/// Delete metric rows older than `retention_days`. `0` disables the reaper
/// (returns 0 without touching the table). Mirrors `jobs::reap_done`.
pub async fn reap(pool: &PgPool, retention_days: u32) -> sqlx::Result<u64> {
if retention_days == 0 {
return Ok(0);
}
let result = sqlx::query(
"DELETE FROM crawl_metrics \
WHERE finished_at < now() - ($1::bigint || ' days')::interval",
)
.bind(retention_days as i64)
.execute(pool)
.await?;
Ok(result.rows_affected())
}

133
backend/src/repo/crawler.rs
View File

@@ -774,6 +774,139 @@ pub async fn list_active_jobs(
Ok((items, total))
}
// ---------------------------------------------------------------------------
// Job history: unified, searchable, filterable view over the queue table.
// ---------------------------------------------------------------------------
/// A `crawler_jobs` row resolved to human context for the admin history
/// table. Works across all job kinds: the target columns are best-effort
/// `Option`s resolved through whichever payload reference the kind carries
/// (`chapter_id`, `manga_id`, or `page_id` via the page breadcrumb), so a
/// `sync_manga` job (which has neither yet) simply leaves them `None` and
/// falls back to `source_key`.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct JobHistoryRow {
pub id: Uuid,
/// `pending` | `running` | `done` | `dead` (the live queue states).
pub state: String,
/// `sync_manga` | `sync_chapter_list` | `sync_chapter_content` | `analyze_page`.
pub kind: Option<String>,
pub manga_id: Option<Uuid>,
pub manga_title: Option<String>,
pub chapter_id: Option<Uuid>,
pub chapter_number: Option<i32>,
/// Set only for `analyze_page` (resolved through the page breadcrumb).
pub page_number: Option<i32>,
/// Source-side key, the only target a `sync_manga` job carries.
pub source_key: Option<String>,
pub attempts: i32,
pub max_attempts: i32,
pub last_error: Option<String>,
pub updated_at: DateTime<Utc>,
/// How long the job's work took, when a timing was recorded: the latest
/// `chapter` crawl-metric for a chapter job, else the page's analysis
/// duration for an `analyze_page` job. `None` for kinds we don't time or
/// jobs not yet completed.
pub duration_ms: Option<i64>,
}
/// Filters for [`list_job_history`]. All optional; `None` widens the scope.
#[derive(Debug, Default, Clone)]
pub struct JobHistoryFilter<'a> {
/// Exact queue state (`done`, `dead`, `running`, `pending`).
pub state: Option<&'a str>,
/// Exact payload `kind`.
pub kind: Option<&'a str>,
/// Case-insensitive manga-title substring.
pub search: Option<&'a str>,
}
/// Paginated, newest-first view of the job queue across every state and
/// kind — the searchable/filterable history surface. Joins each job to its
/// manga/chapter/page context (best-effort) so the table can label rows.
/// Returns the page slice plus the filtered total for pagination.
///
/// History depth is bounded by the done-job reaper (`reap_done`): completed
/// jobs older than the retention window are gone. Terminal `dead` jobs
/// persist until requeued.
pub async fn list_job_history(
pool: &PgPool,
filter: JobHistoryFilter<'_>,
limit: i64,
offset: i64,
) -> sqlx::Result<(Vec<JobHistoryRow>, i64)> {
let search_pat = filter
.search
.map(|s| format!("%{}%", s.trim()))
.filter(|p| p.len() > 2);
// The same FROM/JOIN/WHERE drives both the page slice and the count, so
// they stay in lockstep. `pg` resolves analyze_page → page → chapter;
// `COALESCE` lets one set of joins serve every kind.
let items: Vec<JobHistoryRow> = sqlx::query_as(
r#"
SELECT
cj.id,
cj.state,
cj.payload->>'kind' AS kind,
COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid) AS manga_id,
m.title AS manga_title,
COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id) AS chapter_id,
c.number AS chapter_number,
pg.page_number AS page_number,
cj.payload->>'source_manga_key' AS source_key,
cj.attempts,
cj.max_attempts,
cj.last_error,
cj.updated_at,
COALESCE(cm.duration_ms, pa.duration_ms) AS duration_ms
FROM crawler_jobs cj
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
LEFT JOIN page_analysis pa ON pa.page_id = (cj.payload->>'page_id')::uuid
LEFT JOIN LATERAL (
SELECT duration_ms FROM crawl_metrics
WHERE op = 'chapter'
AND chapter_id = (cj.payload->>'chapter_id')::uuid
ORDER BY finished_at DESC LIMIT 1
) cm ON true
WHERE ($1::text IS NULL OR cj.state = $1)
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
AND ($3::text IS NULL OR m.title ILIKE $3)
ORDER BY cj.updated_at DESC
LIMIT $4 OFFSET $5
"#,
)
.bind(filter.state)
.bind(filter.kind)
.bind(&search_pat)
.bind(limit)
.bind(offset)
.fetch_all(pool)
.await?;
let total: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM crawler_jobs cj
LEFT JOIN pages pg ON pg.id = (cj.payload->>'page_id')::uuid
LEFT JOIN chapters c ON c.id = COALESCE((cj.payload->>'chapter_id')::uuid, pg.chapter_id)
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
WHERE ($1::text IS NULL OR cj.state = $1)
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
AND ($3::text IS NULL OR m.title ILIKE $3)
"#,
)
.bind(filter.state)
.bind(filter.kind)
.bind(&search_pat)
.fetch_one(pool)
.await?;
Ok((items, total))
}
/// A manga whose cover is still missing (queued for cover fetch).
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct MissingCoverRow {

1
backend/src/repo/mod.rs
View File

@@ -6,6 +6,7 @@ pub mod author;
pub mod bookmark;
pub mod chapter;
pub mod collection;
pub mod crawl_metrics;
pub mod crawler;
pub mod genre;
pub mod manga;

21
backend/src/repo/page.rs
View File

@@ -60,6 +60,27 @@ pub async fn locate(
Ok(row)
}
/// Like [`locate`] but also resolves the manga title and chapter number, so
/// live analysis events can carry human labels for the "now analyzing"
/// banner without the dashboard having to look them up. Returns
/// `(manga_id, manga_title, chapter_id, chapter_number, page_number)`.
pub async fn locate_labeled(
pool: &PgPool,
page_id: Uuid,
) -> AppResult<Option<(Uuid, String, Uuid, i32, i32)>> {
let row: Option<(Uuid, String, Uuid, i32, i32)> = sqlx::query_as(
"SELECT c.manga_id, m.title, p.chapter_id, c.number, p.page_number \
FROM pages p \
JOIN chapters c ON c.id = p.chapter_id \
JOIN mangas m ON m.id = c.manga_id \
WHERE p.id = $1",
)
.bind(page_id)
.fetch_optional(pool)
.await?;
Ok(row)
}
pub async fn list_for_chapter(pool: &PgPool, chapter_id: Uuid) -> AppResult<Vec<Page>> {
let rows = sqlx::query_as::<_, Page>(
r#"

150
backend/src/repo/page_analysis.rs
View File

@@ -16,9 +16,11 @@ use uuid::Uuid;
use crate::crawler::jobs::{self, JobPayload};
use crate::domain::page_analysis::{
ChapterCoverage, ContentWarning, MangaCoverage, OcrKind, OcrLine, PageAnalysis,
PageAnalysisDetail, PageSearchItem, PageStatusItem, VisionAnalysis,
AnalysisHistoryRow, AnalysisMetrics, ChapterCoverage, ContentWarning, MangaCoverage,
ModelDuration, OcrKind, OcrLine, PageAnalysis, PageAnalysisDetail, PageSearchItem,
PageStatusItem, VisionAnalysis,
};
use chrono::{DateTime, Utc};
use crate::error::AppResult;
/// Filter set for [`page_search`]. `tags` are AND-ed (a page must carry
@@ -212,6 +214,90 @@ pub async fn chapter_page_status(
Ok(rows)
}
/// Filters for [`list_history`]. `None` widens the scope.
#[derive(Debug, Default, Clone)]
pub struct AnalysisHistoryFilter<'a> {
/// Exact analysis status (`done` or `failed`).
pub status: Option<&'a str>,
/// Only NSFW-flagged pages.
pub nsfw_only: bool,
/// Case-insensitive manga-title substring.
pub search: Option<&'a str>,
}
/// Paginated, newest-first analysis history — every completed/failed page
/// analysis resolved to its manga/chapter/page context. Reads the
/// persistent `page_analysis` table, so unlike the crawler job history it
/// is not bounded by job reaping. Returns the page slice plus the filtered
/// total. `pending` rows are excluded — history is terminal outcomes only.
pub async fn list_history(
pool: &PgPool,
filter: AnalysisHistoryFilter<'_>,
limit: i64,
offset: i64,
) -> AppResult<(Vec<AnalysisHistoryRow>, i64)> {
let search_pat = filter
.search
.map(|s| format!("%{}%", s.trim()))
.filter(|p| p.len() > 2);
let items = sqlx::query_as::<_, AnalysisHistoryRow>(
r#"
SELECT
pa.page_id,
p.page_number,
p.chapter_id,
c.number AS chapter_number,
c.manga_id,
m.title AS manga_title,
pa.status,
pa.is_nsfw,
pa.model,
pa.error,
pa.analyzed_at,
pa.duration_ms
FROM page_analysis pa
JOIN pages p ON p.id = pa.page_id
JOIN chapters c ON c.id = p.chapter_id
JOIN mangas m ON m.id = c.manga_id
WHERE pa.status <> 'pending'
AND ($1::text IS NULL OR pa.status = $1)
AND ($2::bool IS FALSE OR pa.is_nsfw)
AND ($3::text IS NULL OR m.title ILIKE $3)
ORDER BY pa.analyzed_at DESC NULLS LAST, pa.page_id
LIMIT $4 OFFSET $5
"#,
)
.bind(filter.status)
.bind(filter.nsfw_only)
.bind(&search_pat)
.bind(limit)
.bind(offset)
.fetch_all(pool)
.await?;
let (total,): (i64,) = sqlx::query_as(
r#"
SELECT count(*)
FROM page_analysis pa
JOIN pages p ON p.id = pa.page_id
JOIN chapters c ON c.id = p.chapter_id
JOIN mangas m ON m.id = c.manga_id
WHERE pa.status <> 'pending'
AND ($1::text IS NULL OR pa.status = $1)
AND ($2::bool IS FALSE OR pa.is_nsfw)
AND ($3::text IS NULL OR m.title ILIKE $3)
"#,
)
.bind(filter.status)
.bind(filter.nsfw_only)
.bind(&search_pat)
.fetch_one(pool)
.await?;
Ok((items, total))
}
/// Full analysis detail for one page. `None` when the page itself doesn't
/// exist; an existing-but-unanalyzed page returns `status = "none"` with
/// empty OCR/tags/warnings so the UI can show "not analyzed yet".
@@ -331,6 +417,66 @@ pub async fn mark_failed(pool: &PgPool, page_id: Uuid, error: &str) -> AppResult
Ok(())
}
/// Record the wall-clock the worker spent on a page. Best-effort: updates the
/// existing `page_analysis` row (written by `persist_analysis`/`mark_failed`);
/// a transient failure with no row yet simply updates nothing.
pub async fn record_duration(
pool: &PgPool,
page_id: Uuid,
duration_ms: i64,
) -> AppResult<()> {
sqlx::query("UPDATE page_analysis SET duration_ms = $2 WHERE page_id = $1")
.bind(page_id)
.bind(duration_ms)
.execute(pool)
.await?;
Ok(())
}
/// Aggregate analysis timing + outcome roll-up over an optional time window
/// (`since = None` → all time). Mean duration ignores NULL `duration_ms`
/// (pre-tracking rows); `by_model` breaks the mean out per model.
pub async fn analysis_metrics(
pool: &PgPool,
since: Option<DateTime<Utc>>,
) -> AppResult<AnalysisMetrics> {
let (n, ok, failed, avg_ms): (i64, i64, i64, Option<f64>) = sqlx::query_as(
r#"
SELECT
COUNT(*) AS n,
COUNT(*) FILTER (WHERE status = 'done') AS ok,
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
-- Mean over successful pages only, so it reconciles with the
-- by-model breakdown below (which is done-only). Failed /
-- timed-out durations would otherwise skew the headline up.
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
FROM page_analysis
WHERE status <> 'pending'
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
"#,
)
.bind(since)
.fetch_one(pool)
.await?;
let by_model = sqlx::query_as::<_, ModelDuration>(
r#"
SELECT model, AVG(duration_ms)::float8 AS avg_ms, COUNT(*) AS n
FROM page_analysis
WHERE status = 'done'
AND duration_ms IS NOT NULL
AND ($1::timestamptz IS NULL OR analyzed_at >= $1)
GROUP BY model
ORDER BY n DESC
"#,
)
.bind(since)
.fetch_all(pool)
.await?;
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
}
/// Persist a completed analysis for a page, replacing any previous result.
///
/// The model's free-form `kind` / `content_type` strings are mapped onto

5
backend/src/settings.rs
View File

@@ -192,6 +192,11 @@ impl CrawlerSettings {
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
chapter_workers: (self.chapter_workers as usize).max(1),
retention_days: self.retention_days,
// Not a runtime-editable setting; re-read the env default so a
// settings reload keeps whatever CRAWL_METRICS_RETENTION_DAYS was
// configured at boot.
metrics_retention_days: crate::config::env_u64("CRAWL_METRICS_RETENTION_DAYS", 90)
as u32,
start_url,
rate_ms: self.rate_ms,
cdn_host,

10
backend/tests/analysis_worker.rs
View File

@@ -158,6 +158,16 @@ async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
handle.shutdown().await;
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
// The worker stamps how long the dispatch took onto the existing row
// (recorded after ack; shutdown has awaited the in-flight job).
let dur: Option<i64> =
sqlx::query_scalar("SELECT duration_ms FROM page_analysis WHERE page_id = $1")
.bind(page_id)
.fetch_one(&pool)
.await
.unwrap();
assert!(dur.is_some(), "worker should record analysis duration_ms");
}
#[sqlx::test(migrations = "./migrations")]

252
backend/tests/api_admin_analysis.rs
View File

@@ -601,3 +601,255 @@ async fn coverage_requires_admin(pool: PgPool) {
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Analysis history: terminal-outcome log, searchable + filterable.
// ---------------------------------------------------------------------------
/// Seed a manga → chapter → one page, then write a terminal `page_analysis`
/// row for that page. Returns the page id.
async fn seed_analyzed_page(
pool: &PgPool,
title: &str,
status: &str,
is_nsfw: bool,
error: Option<&str>,
) -> uuid::Uuid {
let manga_id: uuid::Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
let page_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO page_analysis (page_id, status, is_nsfw, model, error, analyzed_at) \
VALUES ($1, $2, $3, 'vis', $4, now())",
)
.bind(page_id)
.bind(status)
.bind(is_nsfw)
.bind(error)
.execute(pool)
.await
.unwrap();
page_id
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_lists_filters_and_paginates(pool: PgPool) {
seed_analyzed_page(&pool, "Alpha", "done", false, None).await;
seed_analyzed_page(&pool, "Beta", "failed", false, Some("boom")).await;
seed_analyzed_page(&pool, "Gamma", "done", true, None).await;
// A still-pending row must NOT appear in history (terminal outcomes only).
let pending_page = seed_analyzed_page(&pool, "Delta", "pending", false, None).await;
let _ = pending_page;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Unfiltered: the three terminal rows, pending excluded.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 3);
// Filter by status=failed.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?status=failed",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Beta");
assert_eq!(body["items"][0]["error"], "boom");
// NSFW-only.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?nsfw=true",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Gamma");
// Search by manga title.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?search=Alpha",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Alpha");
assert_eq!(body["items"][0]["status"], "done");
// Pagination: limit clamps the slice; total reflects the full set.
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history?limit=1",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["items"].as_array().unwrap().len(), 1);
assert_eq!(body["page"]["total"], 3);
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_requires_admin(pool: PgPool) {
let h = common::harness(pool.clone());
let (_u, cookie) = common::register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Analysis metrics: durations + averages + by-model + success.
// ---------------------------------------------------------------------------
/// Seed a manga→chapter→page + a terminal page_analysis row carrying a
/// duration and model. Returns the page id.
async fn seed_timed_page(
pool: &PgPool,
title: &str,
status: &str,
model: &str,
duration_ms: i64,
) -> uuid::Uuid {
let manga_id: uuid::Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
let page_id: uuid::Uuid = sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'k', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO page_analysis (page_id, status, model, duration_ms, analyzed_at) \
VALUES ($1, $2, $3, $4, now())",
)
.bind(page_id)
.bind(status)
.bind(model)
.bind(duration_ms)
.execute(pool)
.await
.unwrap();
page_id
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_metrics_over_http(pool: PgPool) {
seed_timed_page(&pool, "Alpha", "done", "qwen", 2000).await;
seed_timed_page(&pool, "Beta", "done", "qwen", 4000).await;
seed_timed_page(&pool, "Gamma", "done", "llava", 3000).await;
seed_timed_page(&pool, "Delta", "failed", "qwen", 1000).await;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/metrics",
&cookie,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = common::body_json(resp).await;
assert_eq!(body["n"], 4);
assert_eq!(body["ok"], 3);
assert_eq!(body["failed"], 1);
// Headline mean is done-only (2000 + 4000 + 3000)/3 — the failed row's
// 1000ms is excluded so it reconciles with the by-model breakdown.
assert_eq!(body["avg_ms"], 3000.0);
// by_model: qwen has the done rows 2000 + 4000 → avg 3000, n 2
// (the failed qwen row is excluded — done only).
let by_model = body["by_model"].as_array().unwrap();
let qwen = by_model.iter().find(|m| m["model"] == "qwen").unwrap();
assert_eq!(qwen["n"], 2);
assert_eq!(qwen["avg_ms"], 3000.0);
}
#[sqlx::test(migrations = "./migrations")]
async fn analysis_history_carries_duration(pool: PgPool) {
seed_timed_page(&pool, "Alpha", "done", "qwen", 2500).await;
let h = common::harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(common::get_with_cookie(
"/api/v1/admin/analysis/history",
&cookie,
))
.await
.unwrap();
let body = common::body_json(resp).await;
assert_eq!(body["items"][0]["duration_ms"], 2500);
}

259
backend/tests/api_admin_crawler.rs
View File

@@ -612,6 +612,265 @@ async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
assert_eq!(payload["requeued"], 1);
}
// ---------------------------------------------------------------------------
// Job history: unified, searchable, filterable list over all states/kinds.
// ---------------------------------------------------------------------------
/// Seed a chapter-content job in a given state and return its (manga, chapter).
async fn seed_history_chapter_job(
pool: &PgPool,
title: &str,
number: i32,
state: &str,
) -> (Uuid, Uuid) {
let manga_id = Uuid::new_v4();
let chapter_id = Uuid::new_v4();
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
.bind(manga_id)
.bind(title)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
.bind(chapter_id)
.bind(manga_id)
.bind(number)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
.bind(Uuid::new_v4())
.bind(json!({
"kind": "sync_chapter_content",
"source_id": "target",
"chapter_id": chapter_id,
"source_chapter_key": "k",
}))
.bind(state)
.execute(pool)
.await
.unwrap();
(manga_id, chapter_id)
}
/// Seed a `done` analyze_page job pointing at a real page so the history
/// row resolves manga/chapter/page-number through the page breadcrumb.
async fn seed_history_analyze_job(pool: &PgPool, title: &str, page_number: i32) {
let manga_id = Uuid::new_v4();
let chapter_id = Uuid::new_v4();
let page_id = Uuid::new_v4();
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
.bind(manga_id)
.bind(title)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 3)")
.bind(chapter_id)
.bind(manga_id)
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO pages (id, chapter_id, page_number, storage_key, content_type) \
VALUES ($1, $2, $3, 'k', 'image/png')",
)
.bind(page_id)
.bind(chapter_id)
.bind(page_number)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, 'done')")
.bind(Uuid::new_v4())
.bind(json!({ "kind": "analyze_page", "page_id": page_id, "force": false }))
.execute(pool)
.await
.unwrap();
}
#[sqlx::test(migrations = "./migrations")]
async fn job_history_lists_filters_and_paginates(pool: PgPool) {
seed_history_chapter_job(&pool, "Naruto", 1, "done").await;
seed_history_chapter_job(&pool, "Bleach", 2, "running").await;
seed_dead_job(&pool, "Vinland").await; // dead sync_chapter_content
seed_history_analyze_job(&pool, "One Piece", 7).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Unfiltered: all four jobs.
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &cookie))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 4);
// Filter by state.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?state=done",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
// Naruto (sync_chapter done) + One Piece (analyze_page done).
assert_eq!(body["page"]["total"], 2);
// Filter by kind.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?kind=analyze_page",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "One Piece");
assert_eq!(body["items"][0]["page_number"], 7);
assert_eq!(body["items"][0]["chapter_number"], 3);
// Search by manga title.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?search=Bleach",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["manga_title"], "Bleach");
assert_eq!(body["items"][0]["state"], "running");
// Pagination: limit clamps the page slice but total reflects the full set.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?limit=1",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["items"].as_array().unwrap().len(), 1);
assert_eq!(body["page"]["total"], 4);
// Admin-gated.
let (_u, plain) = register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/history", &plain))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
// ---------------------------------------------------------------------------
// Operation metrics: durations, averages, recent-ops log.
// ---------------------------------------------------------------------------
async fn record_metric(
pool: &PgPool,
op: &str,
manga_id: Option<Uuid>,
chapter_id: Option<Uuid>,
outcome: &str,
duration_ms: i64,
items: Option<i32>,
) {
mangalord::repo::crawl_metrics::record(
pool, op, manga_id, chapter_id, outcome, duration_ms, items, None,
)
.await
.unwrap();
}
#[sqlx::test(migrations = "./migrations")]
async fn crawler_metrics_summary_and_ops_over_http(pool: PgPool) {
// Two chapter ops (one failed) + a cover op.
record_metric(&pool, "chapter", None, None, "ok", 6000, Some(20)).await;
record_metric(&pool, "chapter", None, None, "failed", 8000, Some(10)).await;
record_metric(&pool, "manga_cover", None, None, "ok", 500, None).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
// Summary: per-type averages + success split.
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &cookie))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = body_json(resp).await;
let summary = body["summary"].as_array().unwrap();
let chapter = summary.iter().find(|s| s["op"] == "chapter").unwrap();
assert_eq!(chapter["n"], 2);
assert_eq!(chapter["ok"], 1);
assert_eq!(chapter["failed"], 1);
assert_eq!(chapter["avg_ms"], 7000.0);
// Recent-ops log + outcome filter.
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/metrics/ops?outcome=failed",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["page"]["total"], 1);
assert_eq!(body["items"][0]["op"], "chapter");
assert_eq!(body["items"][0]["outcome"], "failed");
// Admin-gated.
let (_u, plain) = register_user(&h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie("/api/v1/admin/crawler/metrics", &plain))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
#[sqlx::test(migrations = "./migrations")]
async fn job_history_includes_chapter_duration(pool: PgPool) {
let (_m, chapter_id) = seed_history_chapter_job(&pool, "Berserk", 12, "done").await;
// A chapter metric for that chapter → history row should surface its duration.
record_metric(&pool, "chapter", None, Some(chapter_id), "ok", 6100, Some(20)).await;
let h = harness(pool.clone());
let cookie = seed_admin(&pool, &h.app).await;
let resp = h
.app
.clone()
.oneshot(get_with_cookie(
"/api/v1/admin/crawler/history?kind=sync_chapter_content",
&cookie,
))
.await
.unwrap();
let body = body_json(resp).await;
assert_eq!(body["items"][0]["duration_ms"], 6100);
}
#[sqlx::test(migrations = "./migrations")]
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
let job_id = seed_dead_job(&pool, "Bleach").await;

140
backend/tests/crawl_metrics.rs Normal file
View File

@@ -0,0 +1,140 @@
//! Repo-level integration tests for the crawl_metrics timing log.
use chrono::Utc;
use mangalord::repo::crawl_metrics::{
self, OpFilter, OP_CHAPTER, OP_MANGA_COVER, OP_MANGA_DETAIL, OP_MANGA_LIST,
};
use sqlx::PgPool;
use uuid::Uuid;
async fn seed_manga_chapter(pool: &PgPool, title: &str) -> (Uuid, Uuid) {
let manga_id: Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
.bind(title)
.fetch_one(pool)
.await
.unwrap();
let chapter_id: Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 12) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
(manga_id, chapter_id)
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_rolls_up_avg_counts_and_success(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Berserk").await;
// Two chapter ops (one ok, one failed) + one cover op.
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(
&pool, OP_CHAPTER, Some(m), Some(c), "failed", 8000, Some(10), Some("boom"),
)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_COVER, Some(m), None, "ok", 500, None, None)
.await
.unwrap();
let summary = crawl_metrics::summary(&pool, None).await.unwrap();
let chapter = summary.iter().find(|s| s.op == OP_CHAPTER).unwrap();
assert_eq!(chapter.n, 2);
assert_eq!(chapter.ok, 1);
assert_eq!(chapter.failed, 1);
assert_eq!(chapter.avg_ms, Some(7000.0));
assert_eq!(chapter.avg_items, Some(15.0));
let cover = summary.iter().find(|s| s.op == OP_MANGA_COVER).unwrap();
assert_eq!(cover.n, 1);
assert_eq!(cover.avg_ms, Some(500.0));
assert_eq!(cover.avg_items, None);
}
#[sqlx::test(migrations = "./migrations")]
async fn summary_window_excludes_old_rows(pool: PgPool) {
let (m, _c) = seed_manga_chapter(&pool, "X").await;
crawl_metrics::record(&pool, OP_MANGA_DETAIL, Some(m), None, "ok", 1000, None, None)
.await
.unwrap();
// Backdate it two days.
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '2 days'")
.execute(&pool)
.await
.unwrap();
let since = Utc::now() - chrono::Duration::days(1);
let windowed = crawl_metrics::summary(&pool, Some(since)).await.unwrap();
assert!(windowed.is_empty(), "row older than the window must be excluded");
let all = crawl_metrics::summary(&pool, None).await.unwrap();
assert_eq!(all.len(), 1);
}
#[sqlx::test(migrations = "./migrations")]
async fn list_ops_filters_paginates_and_labels(pool: PgPool) {
let (m, c) = seed_manga_chapter(&pool, "Bleach").await;
crawl_metrics::record(&pool, OP_CHAPTER, Some(m), Some(c), "ok", 6000, Some(20), None)
.await
.unwrap();
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "failed", 42000, Some(8), Some("oops"))
.await
.unwrap();
// Unfiltered: both, newest-first, with labels resolved.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 2);
assert_eq!(rows.len(), 2);
let chapter_row = rows.iter().find(|r| r.op == OP_CHAPTER).unwrap();
assert_eq!(chapter_row.manga_title.as_deref(), Some("Bleach"));
assert_eq!(chapter_row.chapter_number, Some(12));
assert_eq!(chapter_row.items, Some(20));
// Filter by op.
let (rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { op: Some(OP_MANGA_LIST), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
assert_eq!(rows[0].outcome, "failed");
assert_eq!(rows[0].error.as_deref(), Some("oops"));
// Filter by outcome.
let (_rows, total) = crawl_metrics::list_ops(
&pool, OpFilter { outcome: Some("ok"), ..Default::default() }, 50, 0,
)
.await
.unwrap();
assert_eq!(total, 1);
// Pagination: limit clamps the slice; total reflects the full set.
let (rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 1, 0).await.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(total, 2);
}
#[sqlx::test(migrations = "./migrations")]
async fn reap_deletes_old_rows_and_zero_disables(pool: PgPool) {
crawl_metrics::record(&pool, OP_MANGA_LIST, None, None, "ok", 1000, Some(5), None)
.await
.unwrap();
sqlx::query("UPDATE crawl_metrics SET finished_at = now() - interval '100 days'")
.execute(&pool)
.await
.unwrap();
// 0 disables — nothing removed.
assert_eq!(crawl_metrics::reap(&pool, 0).await.unwrap(), 0);
// 90-day retention removes the 100-day-old row.
assert_eq!(crawl_metrics::reap(&pool, 90).await.unwrap(), 1);
let (_rows, total) =
crawl_metrics::list_ops(&pool, OpFilter::default(), 50, 0).await.unwrap();
assert_eq!(total, 0);
}

1
backend/tests/crawler_daemon.rs
View File

@@ -39,6 +39,7 @@ fn make_cfg(
daily_at: far_future_daily_at(),
tz: Tz::UTC,
retention_days: 7,
metrics_retention_days: 90,
session_expired,
status: mangalord::crawler::status::StatusHandle::new(workers),
job_timeout: Duration::from_secs(60),

119
frontend/e2e/admin-analysis.spec.ts
View File

@@ -155,6 +155,48 @@ async function mockAdmin(page: Page, cap: Captured) {
})
);
// Analysis history (terminal-outcome log). Registered before the
// page-detail routes are matched by their more-specific globs.
await page.route('**/api/v1/admin/analysis/history**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
items: [
{
page_id: pageDone,
page_number: 1,
chapter_id: chapterId,
chapter_number: 1,
manga_id: mangaId,
manga_title: 'Berserk',
status: 'done',
is_nsfw: true,
model: 'test-model',
error: null,
analyzed_at: '2026-06-13T12:00:00Z',
duration_ms: 2400
}
],
page: { limit: 25, offset: 0, total: 1 }
})
})
);
await page.route('**/api/v1/admin/analysis/metrics**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
n: 3204,
ok: 3159,
failed: 45,
avg_ms: 2400,
by_model: [{ model: 'qwen2-vl-7b', avg_ms: 2400, n: 3100 }]
})
})
);
// Default: keep the SSE connection pending (no events) so tests that
// don't care about live updates don't trigger reconnect churn. The
// live-updates test overrides this with a fulfilling stream.
@@ -240,14 +282,18 @@ test.describe('/admin/analysis', () => {
kind: 'started',
page_id: pageNone,
manga_id: mangaId,
manga_title: 'Berserk',
chapter_id: chapterId,
chapter_number: 1,
page_number: 2
})}\n\n` +
`event: analysis\ndata: ${JSON.stringify({
kind: 'completed',
page_id: pageNone,
manga_id: mangaId,
manga_title: 'Berserk',
chapter_id: chapterId,
chapter_number: 1,
page_number: 2
})}\n\n`;
let served = false;
@@ -268,10 +314,81 @@ test.describe('/admin/analysis', () => {
// (The mocked stream closes after its body, so the live pill flips
// back to "Reconnecting…" — the ticker is the durable signal.)
const tick = page.getByTestId('admin-analysis-ticker');
await expect(tick).toContainText('Analyzing page 2');
await expect(tick).toContainText('Analyzing Berserk');
await expect(tick).toContainText('Analyzed page 2');
});
test('now-analyzing banner appears from SSE and Jump expands the chapter', async ({
page
}) => {
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
await mockAdmin(page, cap);
// A single `started` frame, then hang so the banner stays put.
const frame = `event: analysis\ndata: ${JSON.stringify({
kind: 'started',
page_id: pageDone,
manga_id: mangaId,
manga_title: 'Berserk',
chapter_id: chapterId,
chapter_number: 1,
page_number: 1
})}\n\n`;
let served = false;
await page.route('**/api/v1/admin/analysis/status/stream', (r) => {
if (served) return new Promise(() => {});
served = true;
return r.fulfill({
status: 200,
headers: { 'content-type': 'text/event-stream' },
body: frame
});
});
await page.setViewportSize(DESKTOP);
await page.goto('/admin/analysis');
const banner = page.getByTestId('admin-analysis-now');
await expect(banner).toContainText('Now analyzing');
await expect(banner).toContainText('Berserk · Ch 1 · Page 1');
// Jump expands the manga + chapter and the analyzing chip appears.
await page.getByTestId('admin-analysis-now-jump').click();
await expect(page.getByTestId(`admin-analysis-page-${pageDone}`)).toBeVisible();
});
test('history tab lists terminal analyses and opens the detail modal', async ({
page
}) => {
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
await mockAdmin(page, cap);
await page.setViewportSize(DESKTOP);
await page.goto('/admin/analysis');
await page.getByTestId('admin-analysis-tab-history').click();
const row = page.getByTestId(`analysis-history-row-${pageDone}`);
await expect(row).toContainText('Berserk · Ch 1 · p1');
await expect(row).toContainText('NSFW');
await expect(row).toContainText('2.4s'); // duration column
await row.click();
await expect(page.getByTestId('admin-analysis-detail')).toBeVisible();
await expect(page.getByTestId('admin-analysis-detail-status')).toContainText(
'Analyzed'
);
});
test('metrics tab shows aggregate timing + by-model', async ({ page }) => {
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
await mockAdmin(page, cap);
await page.setViewportSize(DESKTOP);
await page.goto('/admin/analysis');
await page.getByTestId('admin-analysis-tab-metrics').click();
await expect(page.getByTestId('analysis-metrics-n')).toContainText('3204');
await expect(page.getByTestId('analysis-metrics-avg')).toContainText('2.4s');
await expect(page.getByTestId('analysis-metrics')).toContainText('qwen2-vl-7b');
});
test('queue an unanalyzed page from its detail modal', async ({ page }) => {
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
await mockAdmin(page, cap);

220
frontend/e2e/admin-crawler.spec.ts Normal file
View File

@@ -0,0 +1,220 @@
import { test, expect, type Page } from '@playwright/test';
// E2E for the admin Crawler "History" tab: the Live/History toggle, the
// searchable/filterable job log, and inline requeue of a dead job. The
// live status + SSE stream are left pending so the test focuses on history
// (the live dashboard is exercised by its own status mocks elsewhere).
const DESKTOP = { width: 1280, height: 720 } as const;
const deadJobId = 'd1111111-1111-1111-1111-111111111111';
const doneJobId = 'd2222222-2222-2222-2222-222222222222';
const adminUser = {
id: 'u11111111-1111-1111-1111-111111111111',
username: 'admin',
created_at: '2026-01-01T00:00:00Z',
is_admin: true
};
type Captured = { requeue: Record<string, unknown> | null };
async function mockAdmin(page: Page, cap: Captured) {
await page.route('**/api/v1/auth/config', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
})
);
await page.route('**/api/v1/auth/me', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ user: adminUser })
})
);
await page.route('**/api/v1/auth/me/preferences', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
})
);
await page.route('**/api/v1/me/bookmarks*', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
})
);
await page.route('**/api/v1/admin/system', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
disk: null,
memory: { total_bytes: 1, used_bytes: 0, percent_used: 0 },
cpu: { percent_used: 0 },
alerts: []
})
})
);
// Leave the live status fetch + SSE stream pending so the Live view
// stays in its "Loading…" state and doesn't churn; History is what we test.
await page.route('**/api/v1/admin/crawler', () => new Promise(() => {}));
await page.route('**/api/v1/admin/crawler/stream', () => new Promise(() => {}));
await page.route('**/api/v1/admin/crawler/dead-jobs**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ items: [], page: { limit: 20, offset: 0, total: 0 } })
})
);
await page.route('**/api/v1/admin/crawler/history**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
items: [
{
id: deadJobId,
state: 'dead',
kind: 'sync_chapter_content',
manga_id: 'm-1',
manga_title: 'Naruto',
chapter_id: 'c-1',
chapter_number: 700,
page_number: null,
source_key: null,
attempts: 5,
max_attempts: 5,
last_error: 'boom: upstream 500',
updated_at: '2026-06-15T00:00:00Z'
},
{
id: doneJobId,
state: 'done',
kind: 'analyze_page',
manga_id: 'm-2',
manga_title: 'Bleach',
chapter_id: 'c-2',
chapter_number: 3,
page_number: 7,
source_key: null,
attempts: 1,
max_attempts: 5,
last_error: null,
updated_at: '2026-06-15T00:00:00Z'
}
],
page: { limit: 25, offset: 0, total: 2 }
})
})
);
// Summary registered first (broad glob); the more-specific /ops route is
// registered after so it wins for the ops URL (Playwright: last match wins).
await page.route('**/api/v1/admin/crawler/metrics**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
summary: [
{ op: 'manga_list', avg_ms: 42000, n: 8, ok: 8, failed: 0, avg_items: 50 },
{ op: 'manga_detail', avg_ms: 1300, n: 210, ok: 205, failed: 5, avg_items: null },
{ op: 'manga_cover', avg_ms: 480, n: 180, ok: 176, failed: 4, avg_items: null },
{ op: 'chapter', avg_ms: 6800, n: 430, ok: 421, failed: 9, avg_items: 32 }
]
})
})
);
await page.route('**/api/v1/admin/crawler/metrics/ops**', (r) =>
r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
items: [
{
id: 'op-1',
op: 'chapter',
manga_id: 'm-1',
manga_title: 'Berserk',
chapter_id: 'c-1',
chapter_number: 12,
outcome: 'ok',
duration_ms: 6100,
items: 20,
error: null,
finished_at: '2026-06-15T00:00:00Z'
}
],
page: { limit: 25, offset: 0, total: 1 }
})
})
);
await page.route('**/api/v1/admin/crawler/dead-jobs/requeue', (r) => {
cap.requeue = JSON.parse(r.request().postData() ?? '{}');
return r.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ requeued: 1 })
});
});
}
test.describe('/admin/crawler history', () => {
test('toggle to History lists jobs and requeues a dead job inline', async ({
page
}) => {
const cap: Captured = { requeue: null };
await mockAdmin(page, cap);
await page.setViewportSize(DESKTOP);
await page.goto('/admin/crawler');
await page.getByTestId('crawler-tab-history').click();
const deadRow = page.getByTestId(`crawler-history-row-${deadJobId}`);
await expect(deadRow).toContainText('Naruto · Ch 700');
await expect(deadRow).toContainText('dead');
const doneRow = page.getByTestId(`crawler-history-row-${doneJobId}`);
await expect(doneRow).toContainText('Bleach · Ch 3 · p7');
// Click the dead row to expand its error detail.
await deadRow.click();
await expect(
page.getByTestId('crawler-history').locator('.errrow')
).toContainText('boom: upstream 500');
// Inline requeue posts scope=job.
await page.getByTestId(`crawler-history-requeue-${deadJobId}`).click();
await expect.poll(() => cap.requeue).toEqual({ scope: 'job', job_id: deadJobId });
});
test('Metrics tab shows averages by type, derived per-page, and the ops log', async ({
page
}) => {
const cap: Captured = { requeue: null };
await mockAdmin(page, cap);
await page.setViewportSize(DESKTOP);
await page.goto('/admin/crawler');
await page.getByTestId('crawler-tab-metrics').click();
// Per-type averages.
await expect(page.getByTestId('crawler-metrics-row-chapter')).toContainText('6.8s');
await expect(page.getByTestId('crawler-metrics-row-manga_cover')).toContainText(
'480ms'
);
// Derived per-page row: 6800ms ÷ 32 ≈ 213ms.
await expect(page.getByTestId('crawler-metrics-perpage')).toContainText('213ms');
// Recent-ops log shows individual durations.
await expect(page.getByTestId('crawler-op-op-1')).toContainText('6.1s');
await expect(page.getByTestId('crawler-op-op-1')).toContainText('Berserk · Ch 12');
});
});

2
frontend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "mangalord-frontend",
"version": "0.82.0",
"version": "0.84.0",
"private": true,
"type": "module",
"scripts": {

155
frontend/src/lib/api/admin.test.ts
View File

@@ -35,7 +35,12 @@ import {
getAnalysisChapterCoverage,
getAnalysisChapterPages,
getAnalysisPageDetail,
analysisStatusStreamUrl
analysisStatusStreamUrl,
listCrawlerJobHistory,
listAnalysisHistory,
getCrawlerMetrics,
listCrawlerOps,
getAnalysisMetrics
} from './admin';
function ok(body: unknown, status = 200): Response {
@@ -651,4 +656,152 @@ describe('admin crawler api client', () => {
expect(url).toMatch(/\/v1\/admin\/storage\/backfill$/);
expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST');
});
// ---- job history ----
it('listCrawlerJobHistory forwards state/kind/search/limit/offset and parses items', async () => {
const row = {
id: 'j-1',
state: 'done',
kind: 'analyze_page',
manga_id: 'm-1',
manga_title: 'Berserk',
chapter_id: 'c-1',
chapter_number: 12,
page_number: 7,
source_key: null,
attempts: 1,
max_attempts: 5,
last_error: null,
updated_at: '2026-06-15T00:00:00Z'
};
fetchSpy.mockResolvedValueOnce(
ok({ items: [row], page: { limit: 50, offset: 0, total: 1 } })
);
const page = await listCrawlerJobHistory({
state: 'done',
kind: 'analyze_page',
search: 'Ber',
offset: 10
});
expect(page.items[0]).toEqual(row);
expect(page.page.total).toBe(1);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('/v1/admin/crawler/history?');
expect(url).toContain('state=done');
expect(url).toContain('kind=analyze_page');
expect(url).toContain('search=Ber');
expect(url).toContain('offset=10');
});
it('listCrawlerJobHistory omits unset filters', async () => {
fetchSpy.mockResolvedValueOnce(
ok({ items: [], page: { limit: 50, offset: 0, total: 0 } })
);
await listCrawlerJobHistory();
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/crawler\/history$/);
});
it('listAnalysisHistory forwards status/nsfw/search and parses items', async () => {
const row = {
page_id: 'p-1',
page_number: 3,
chapter_id: 'c-1',
chapter_number: 700,
manga_id: 'm-1',
manga_title: 'Naruto',
status: 'failed',
is_nsfw: false,
model: null,
error: 'boom',
analyzed_at: '2026-06-15T00:00:00Z'
};
fetchSpy.mockResolvedValueOnce(
ok({ items: [row], page: { limit: 25, offset: 0, total: 1 } })
);
const page = await listAnalysisHistory({ status: 'failed', nsfw: true, search: 'Nar' });
expect(page.items[0]).toEqual(row);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('/v1/admin/analysis/history?');
expect(url).toContain('status=failed');
expect(url).toContain('nsfw=true');
expect(url).toContain('search=Nar');
});
it('listAnalysisHistory omits nsfw when false', async () => {
fetchSpy.mockResolvedValueOnce(
ok({ items: [], page: { limit: 25, offset: 0, total: 0 } })
);
await listAnalysisHistory({ status: 'done', nsfw: false });
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('status=done');
expect(url).not.toContain('nsfw');
});
// ---- operation metrics ----
it('getCrawlerMetrics omits days when 0 and parses the summary', async () => {
fetchSpy.mockResolvedValueOnce(
ok({
summary: [
{ op: 'chapter', avg_ms: 7000, n: 2, ok: 1, failed: 1, avg_items: 15 }
]
})
);
const r = await getCrawlerMetrics(0);
expect(r.summary[0].avg_ms).toBe(7000);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/crawler\/metrics$/);
});
it('getCrawlerMetrics forwards a positive days window', async () => {
fetchSpy.mockResolvedValueOnce(ok({ summary: [] }));
await getCrawlerMetrics(7);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('days=7');
});
it('listCrawlerOps forwards op/outcome/days and parses the page', async () => {
const row = {
id: 'o-1',
op: 'chapter',
manga_id: 'm-1',
manga_title: 'Berserk',
chapter_id: 'c-1',
chapter_number: 12,
outcome: 'ok',
duration_ms: 6100,
items: 20,
error: null,
finished_at: '2026-06-15T00:00:00Z'
};
fetchSpy.mockResolvedValueOnce(
ok({ items: [row], page: { limit: 50, offset: 0, total: 1 } })
);
const page = await listCrawlerOps({ op: 'chapter', outcome: 'ok', days: 30 });
expect(page.items[0]).toEqual(row);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('/v1/admin/crawler/metrics/ops?');
expect(url).toContain('op=chapter');
expect(url).toContain('outcome=ok');
expect(url).toContain('days=30');
});
it('getAnalysisMetrics parses aggregate + by-model', async () => {
fetchSpy.mockResolvedValueOnce(
ok({
n: 4,
ok: 3,
failed: 1,
avg_ms: 2500,
by_model: [{ model: 'qwen', avg_ms: 3000, n: 2 }]
})
);
const m = await getAnalysisMetrics(7);
expect(m.avg_ms).toBe(2500);
expect(m.by_model[0].model).toBe('qwen');
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toContain('/v1/admin/analysis/metrics?days=7');
});
});

189
frontend/src/lib/api/admin.ts
View File

@@ -436,6 +436,128 @@ export async function listActiveJobs(
);
}
/** Queue states a crawler job can be in. */
export type CrawlerJobState = 'pending' | 'running' | 'done' | 'dead';
/** Job kinds the crawler/analysis queue carries. */
export type CrawlerJobKind =
| 'sync_manga'
| 'sync_chapter_list'
| 'sync_chapter_content'
| 'analyze_page';
/** One row in the crawler job-history table — any state/kind, resolved to
* its manga/chapter/page context (best-effort `null`s for kinds that don't
* carry that reference, e.g. a bootstrap `sync_manga`). */
export type CrawlerHistoryRow = {
id: string;
state: CrawlerJobState;
kind: CrawlerJobKind | null;
manga_id: string | null;
manga_title: string | null;
chapter_id: string | null;
chapter_number: number | null;
page_number: number | null;
source_key: string | null;
attempts: number;
max_attempts: number;
last_error: string | null;
updated_at: string;
/** Recorded duration for a chapter / analyze job; `null` otherwise. */
duration_ms: number | null;
};
export type CrawlerHistoryPage = { items: CrawlerHistoryRow[]; page: Page };
/** GET /v1/admin/crawler/history — unified, searchable/filterable job log.
* History depth is bounded by the done-job reaper (recent window); `dead`
* jobs persist until requeued. */
export async function listCrawlerJobHistory(
opts?: {
state?: CrawlerJobState;
kind?: CrawlerJobKind;
search?: string;
limit?: number;
offset?: number;
},
init?: RequestInit
): Promise<CrawlerHistoryPage> {
const params = new URLSearchParams();
if (opts?.state) params.set('state', opts.state);
if (opts?.kind) params.set('kind', opts.kind);
if (opts?.search) params.set('search', opts.search);
if (opts?.limit != null) params.set('limit', String(opts.limit));
if (opts?.offset != null) params.set('offset', String(opts.offset));
const qs = params.toString();
return request<CrawlerHistoryPage>(
`/v1/admin/crawler/history${qs ? `?${qs}` : ''}`,
init
);
}
// ---- operation metrics (durations / averages) ------------------------------
/** Crawl operation kinds timed in `crawl_metrics`. */
export type CrawlOp = 'manga_list' | 'manga_detail' | 'manga_cover' | 'chapter';
/** Per-op average roll-up over the selected window. */
export type OpSummary = {
op: CrawlOp;
avg_ms: number | null;
n: number;
ok: number;
failed: number;
avg_items: number | null;
};
/** One timed operation in the recent-ops log. */
export type OpRow = {
id: string;
op: CrawlOp;
manga_id: string | null;
manga_title: string | null;
chapter_id: string | null;
chapter_number: number | null;
outcome: 'ok' | 'failed';
duration_ms: number;
items: number | null;
error: string | null;
finished_at: string;
};
export type CrawlerOpsPage = { items: OpRow[]; page: Page };
/** GET /v1/admin/crawler/metrics — per-type average durations + success.
* `days` windows the rows (0/omitted = all time). */
export async function getCrawlerMetrics(days = 0): Promise<{ summary: OpSummary[] }> {
const qs = days > 0 ? `?days=${days}` : '';
return request<{ summary: OpSummary[] }>(`/v1/admin/crawler/metrics${qs}`);
}
/** GET /v1/admin/crawler/metrics/ops — paginated recent timed-operations log. */
export async function listCrawlerOps(
opts?: {
op?: CrawlOp;
outcome?: 'ok' | 'failed';
days?: number;
limit?: number;
offset?: number;
},
init?: RequestInit
): Promise<CrawlerOpsPage> {
const params = new URLSearchParams();
if (opts?.op) params.set('op', opts.op);
if (opts?.outcome) params.set('outcome', opts.outcome);
if (opts?.days != null && opts.days > 0) params.set('days', String(opts.days));
if (opts?.limit != null) params.set('limit', String(opts.limit));
if (opts?.offset != null) params.set('offset', String(opts.offset));
const qs = params.toString();
return request<CrawlerOpsPage>(
`/v1/admin/crawler/metrics/ops${qs ? `?${qs}` : ''}`,
init
);
}
/** A manga queued for a cover fetch (no cover yet + a live source). */
export type MissingCover = { manga_id: string; manga_title: string };
export type MissingCoversPage = { items: MissingCover[]; page: Page };
@@ -588,6 +710,71 @@ export async function getAnalysisPageDetail(
);
}
/** One row in the analysis history table — a terminal (done/failed)
* analysis pass resolved to its page/chapter/manga context. Persists
* indefinitely (sourced from `page_analysis`, not the reaped job queue). */
export type AnalysisHistoryRow = {
page_id: string;
page_number: number;
chapter_id: string;
chapter_number: number;
manga_id: string;
manga_title: string;
status: 'done' | 'failed';
is_nsfw: boolean;
model: string | null;
error: string | null;
analyzed_at: string | null;
/** Wall-clock the worker spent; `null` for pre-tracking rows. */
duration_ms: number | null;
};
export type AnalysisHistoryPage = { items: AnalysisHistoryRow[]; page: Page };
/** GET /v1/admin/analysis/history — searchable/filterable log of completed
* and failed page analyses, newest first. */
export async function listAnalysisHistory(
opts?: {
status?: 'done' | 'failed';
nsfw?: boolean;
search?: string;
limit?: number;
offset?: number;
},
init?: RequestInit
): Promise<AnalysisHistoryPage> {
const params = new URLSearchParams();
if (opts?.status) params.set('status', opts.status);
if (opts?.nsfw) params.set('nsfw', 'true');
if (opts?.search) params.set('search', opts.search);
if (opts?.limit != null) params.set('limit', String(opts.limit));
if (opts?.offset != null) params.set('offset', String(opts.offset));
const qs = params.toString();
return request<AnalysisHistoryPage>(
`/v1/admin/analysis/history${qs ? `?${qs}` : ''}`,
init
);
}
/** Per-model average analysis duration. */
export type ModelDuration = { model: string | null; avg_ms: number | null; n: number };
/** Aggregate analysis timing/outcome roll-up over the selected window. */
export type AnalysisMetrics = {
n: number;
ok: number;
failed: number;
avg_ms: number | null;
by_model: ModelDuration[];
};
/** GET /v1/admin/analysis/metrics — avg duration, success, by-model.
* `days` windows the rows (0/omitted = all time). */
export async function getAnalysisMetrics(days = 0): Promise<AnalysisMetrics> {
const qs = days > 0 ? `?days=${days}` : '';
return request<AnalysisMetrics>(`/v1/admin/analysis/metrics${qs}`);
}
/** One live analysis event from the SSE stream. */
export type AnalysisEvent =
| {
@@ -600,7 +787,9 @@ export type AnalysisEvent =
kind: 'started' | 'completed' | 'failed';
page_id: string;
manga_id: string;
manga_title: string;
chapter_id: string;
chapter_number: number;
page_number: number;
};

229
frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte Normal file
View File

@@ -0,0 +1,229 @@
<script lang="ts">
import { onMount } from 'svelte';
import Pager from '$lib/components/Pager.svelte';
import { fmtDuration } from '$lib/format';
import {
listAnalysisHistory,
type AnalysisHistoryRow
} from '$lib/api/admin';
// Row click opens the parent's existing page-detail modal — history
// reuses the same drill-down inspector as the coverage grid.
let {
onOpenDetail
}: {
onOpenDetail: (pageId: string) => void;
} = $props();
const LIMIT = 25;
let rows = $state<AnalysisHistoryRow[]>([]);
let total = $state(0);
let page = $state(1);
let statusFilter = $state<'' | 'done' | 'failed'>('');
let nsfwOnly = $state(false);
let search = $state('');
let loading = $state(true);
let error = $state<string | null>(null);
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
async function load() {
loading = true;
error = null;
try {
const resp = await listAnalysisHistory({
status: statusFilter || undefined,
nsfw: nsfwOnly,
search: search.trim() || undefined,
limit: LIMIT,
offset: (page - 1) * LIMIT
});
rows = resp.items;
total = resp.page.total ?? resp.items.length;
} catch (e) {
error = e instanceof Error ? e.message : 'Failed to load history.';
} finally {
loading = false;
}
}
onMount(load);
function applyFilters() {
page = 1;
load();
}
function onPageChange(p: number) {
page = p;
load();
}
function onSearchKey(e: KeyboardEvent) {
if (e.key === 'Enter') applyFilters();
}
function fmtAgo(iso: string | null): string {
if (!iso) return '—';
const secs = Math.round((Date.now() - new Date(iso).getTime()) / 1000);
if (secs < 45) return 'just now';
const mins = Math.round(secs / 60);
if (mins < 60) return `${mins}m ago`;
const hrs = Math.round(mins / 60);
if (hrs < 24) return `${hrs}h ago`;
return new Date(iso).toLocaleDateString();
}
</script>
<section class="history" data-testid="analysis-history">
<div class="toolbar">
<select
bind:value={statusFilter}
onchange={applyFilters}
aria-label="Filter by status"
data-testid="analysis-history-status"
>
<option value="">All statuses</option>
<option value="done">done</option>
<option value="failed">failed</option>
</select>
<label class="nsfw">
<input
type="checkbox"
bind:checked={nsfwOnly}
onchange={applyFilters}
data-testid="analysis-history-nsfw"
/>
NSFW only
</label>
<input
class="search"
type="text"
bind:value={search}
placeholder="Search manga title…"
onkeydown={onSearchKey}
data-testid="analysis-history-search"
/>
<button type="button" onclick={applyFilters}>Search</button>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
{#if loading}
<p class="muted" data-testid="analysis-history-loading">Loading…</p>
{:else if rows.length === 0}
<p class="muted" data-testid="analysis-history-empty">No analyses match.</p>
{:else}
<table>
<thead>
<tr>
<th>Status</th>
<th>Page</th>
<th>Model</th>
<th class="num">Dur</th>
<th>Analyzed</th>
</tr>
</thead>
<tbody>
{#each rows as r (r.page_id)}
<tr
class="clickable"
onclick={() => onOpenDetail(r.page_id)}
data-testid={`analysis-history-row-${r.page_id}`}
title={r.error ?? ''}
>
<td><span class="status-pill {r.status}">{r.status}</span></td>
<td>
{r.manga_title} · Ch {r.chapter_number} · p{r.page_number}
{#if r.is_nsfw}<span class="nsfw-tag">⚠ NSFW</span>{/if}
</td>
<td class="muted">{r.model ?? '—'}</td>
<td class="num">{fmtDuration(r.duration_ms)}</td>
<td title={r.analyzed_at ? new Date(r.analyzed_at).toLocaleString() : ''}
>{fmtAgo(r.analyzed_at)}</td
>
</tr>
{/each}
</tbody>
</table>
<Pager {page} {totalPages} onChange={onPageChange} testid="analysis-history-pager" />
{/if}
</section>
<style>
.toolbar {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: var(--space-2);
margin-bottom: var(--space-3);
}
select,
.search {
height: 36px;
padding: 0 var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
.nsfw {
display: inline-flex;
align-items: center;
gap: var(--space-1);
font-size: var(--font-sm);
color: var(--text-muted);
}
.muted {
color: var(--text-muted);
}
table {
width: 100%;
border-collapse: collapse;
max-width: 52rem;
}
th,
td {
padding: var(--space-2);
text-align: left;
border-bottom: 1px solid var(--border);
font-size: var(--font-sm);
}
.num {
text-align: right;
font-variant-numeric: tabular-nums;
}
tr.clickable {
cursor: pointer;
}
tr.clickable:hover {
background: var(--surface);
}
.status-pill {
font-weight: var(--weight-semibold);
font-size: var(--font-xs);
padding: 1px var(--space-2);
border-radius: var(--radius-pill);
text-transform: uppercase;
background: var(--surface-elevated);
color: var(--text-muted);
}
.status-pill.done {
background: color-mix(in srgb, #2e7d32 16%, transparent);
color: #2e7d32;
}
.status-pill.failed {
background: color-mix(in srgb, var(--danger) 16%, transparent);
color: var(--danger);
}
.nsfw-tag {
font-size: var(--font-xs);
color: #b85e1a;
margin-left: var(--space-1);
}
.error {
color: var(--danger);
}
</style>

171
frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte Normal file
View File

@@ -0,0 +1,171 @@
<script lang="ts">
import { onMount } from 'svelte';
import { fmtDuration } from '$lib/format';
import { getAnalysisMetrics, type AnalysisMetrics } from '$lib/api/admin';
let days = $state(7);
let metrics = $state<AnalysisMetrics | null>(null);
let loading = $state(true);
let error = $state<string | null>(null);
const successPct = $derived(
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
);
async function load() {
loading = true;
error = null;
try {
metrics = await getAnalysisMetrics(days);
} catch (e) {
error = e instanceof Error ? e.message : 'Failed to load metrics.';
} finally {
loading = false;
}
}
onMount(load);
</script>
<section data-testid="analysis-metrics">
<div class="winrow">
<label>
Window
<select bind:value={days} onchange={load} data-testid="analysis-metrics-window">
<option value={1}>24 hours</option>
<option value={7}>7 days</option>
<option value={30}>30 days</option>
<option value={0}>All time</option>
</select>
</label>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
{:else if loading}
<p class="muted">Loading…</p>
{:else if metrics}
<div class="tiles">
<div class="tile">
<span class="label">Pages analyzed</span>
<span class="value" data-testid="analysis-metrics-n">{metrics.n}</span>
</div>
<div class="tile">
<span class="label">Avg duration</span>
<span class="value" data-testid="analysis-metrics-avg"
>{fmtDuration(metrics.avg_ms)}</span
>
</div>
<div class="tile">
<span class="label">Success</span>
<span class="value">{successPct == null ? '—' : `${successPct}%`}</span>
</div>
<div class="tile">
<span class="label">Failed</span>
<span class="value">{metrics.failed}</span>
</div>
</div>
<h2>By model</h2>
{#if metrics.by_model.length === 0}
<p class="muted">No completed analyses in this window.</p>
{:else}
<table>
<thead>
<tr>
<th>Model</th>
<th class="num">Avg</th>
<th class="num">N</th>
</tr>
</thead>
<tbody>
{#each metrics.by_model as m (m.model)}
<tr>
<td>{m.model ?? '(unknown)'}</td>
<td class="num">{fmtDuration(m.avg_ms)}</td>
<td class="num">{m.n}</td>
</tr>
{/each}
</tbody>
</table>
{/if}
{/if}
</section>
<style>
.winrow {
display: flex;
justify-content: flex-end;
margin-bottom: var(--space-3);
}
label {
font-size: var(--font-sm);
color: var(--text-muted);
}
select {
height: 32px;
margin-left: var(--space-1);
padding: 0 var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
.tiles {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
gap: var(--space-3);
max-width: 44rem;
margin-bottom: var(--space-4);
}
.tile {
display: flex;
flex-direction: column;
gap: 2px;
padding: var(--space-3);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
}
.tile .label {
font-size: var(--font-xs);
color: var(--text-muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.tile .value {
font-size: var(--font-lg);
font-weight: var(--weight-semibold);
font-variant-numeric: tabular-nums;
}
h2 {
margin: 0 0 var(--space-2);
font-size: var(--font-sm);
color: var(--text-muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
table {
width: 100%;
border-collapse: collapse;
max-width: 32rem;
}
th,
td {
padding: var(--space-2);
text-align: left;
border-bottom: 1px solid var(--border);
font-size: var(--font-sm);
}
.num {
text-align: right;
font-variant-numeric: tabular-nums;
}
.muted {
color: var(--text-muted);
}
.error {
color: var(--danger);
}
</style>

293
frontend/src/lib/components/crawler/CrawlerHistoryTable.svelte Normal file
View File

@@ -0,0 +1,293 @@
<script lang="ts">
import { onMount } from 'svelte';
import Pager from '$lib/components/Pager.svelte';
import { fmtDuration } from '$lib/format';
import SearchBar from './SearchBar.svelte';
import {
listCrawlerJobHistory,
type CrawlerHistoryRow,
type CrawlerJobState,
type CrawlerJobKind,
type RequeueScope
} from '$lib/api/admin';
// Requeue is owned by the parent (shared with the dead-jobs table); we
// call it then reload so the row's new state shows. `busy` disables the
// inline button during a parent-driven action.
let {
onRequeue,
busy = false
}: {
onRequeue: (scope: RequeueScope) => Promise<void>;
busy?: boolean;
} = $props();
const LIMIT = 25;
let rows = $state<CrawlerHistoryRow[]>([]);
let total = $state(0);
let page = $state(1);
let stateFilter = $state<'' | CrawlerJobState>('');
let kindFilter = $state<'' | CrawlerJobKind>('');
let search = $state('');
let loading = $state(true);
let error = $state<string | null>(null);
let expanded = $state<string | null>(null);
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
async function load() {
loading = true;
error = null;
try {
const resp = await listCrawlerJobHistory({
state: stateFilter || undefined,
kind: kindFilter || undefined,
search: search.trim() || undefined,
limit: LIMIT,
offset: (page - 1) * LIMIT
});
rows = resp.items;
total = resp.page.total ?? resp.items.length;
} catch (e) {
error = e instanceof Error ? e.message : 'Failed to load history.';
} finally {
loading = false;
}
}
onMount(load);
function applyFilters() {
page = 1;
load();
}
function onPageChange(p: number) {
page = p;
load();
}
async function requeue(scope: RequeueScope) {
await onRequeue(scope);
await load();
}
/** A human "Manga · Ch N · pP" target, or the source key / em-dash. */
function target(r: CrawlerHistoryRow): string {
if (r.manga_title) {
let s = r.manga_title;
if (r.chapter_number != null) s += ` · Ch ${r.chapter_number}`;
if (r.page_number != null) s += ` · p${r.page_number}`;
return s;
}
return r.source_key ?? '—';
}
function fmtAgo(iso: string): string {
const then = new Date(iso).getTime();
const secs = Math.round((Date.now() - then) / 1000);
if (secs < 45) return 'just now';
if (secs < 90) return '1m ago';
const mins = Math.round(secs / 60);
if (mins < 60) return `${mins}m ago`;
const hrs = Math.round(mins / 60);
if (hrs < 24) return `${hrs}h ago`;
return new Date(iso).toLocaleDateString();
}
const KINDS: { value: '' | CrawlerJobKind; label: string }[] = [
{ value: '', label: 'All types' },
{ value: 'sync_manga', label: 'sync_manga' },
{ value: 'sync_chapter_list', label: 'sync_chapter_list' },
{ value: 'sync_chapter_content', label: 'sync_chapter' },
{ value: 'analyze_page', label: 'analyze_page' }
];
const STATES: { value: '' | CrawlerJobState; label: string }[] = [
{ value: '', label: 'All states' },
{ value: 'done', label: 'done' },
{ value: 'dead', label: 'dead' },
{ value: 'running', label: 'running' },
{ value: 'pending', label: 'pending' }
];
</script>
<section class="history" data-testid="crawler-history">
<div class="toolbar">
<select
bind:value={stateFilter}
onchange={applyFilters}
aria-label="Filter by state"
data-testid="crawler-history-state"
>
{#each STATES as s (s.value)}<option value={s.value}>{s.label}</option>{/each}
</select>
<select
bind:value={kindFilter}
onchange={applyFilters}
aria-label="Filter by type"
data-testid="crawler-history-kind"
>
{#each KINDS as k (k.value)}<option value={k.value}>{k.label}</option>{/each}
</select>
<SearchBar
bind:value={search}
placeholder="Search manga / chapter…"
onSearch={applyFilters}
/>
</div>
<p class="muted note">
Showing recent jobs — completed jobs are pruned after the retention
window; dead jobs persist until requeued.
</p>
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
{#if loading}
<p class="muted" data-testid="crawler-history-loading">Loading…</p>
{:else if rows.length === 0}
<p class="muted" data-testid="crawler-history-empty">No jobs match.</p>
{:else}
<table>
<thead>
<tr>
<th>State</th>
<th>Type</th>
<th>Target</th>
<th class="num">Dur</th>
<th>Att.</th>
<th>Updated</th>
<th class="actions"></th>
</tr>
</thead>
<tbody>
{#each rows as r (r.id)}
<tr
class:clickable={!!r.last_error}
onclick={() =>
r.last_error && (expanded = expanded === r.id ? null : r.id)}
data-testid={`crawler-history-row-${r.id}`}
>
<td>
<span class="badge state-{r.state}">{r.state}</span>
</td>
<td class="kind">{r.kind ?? '—'}</td>
<td>{target(r)}</td>
<td class="num">{fmtDuration(r.duration_ms)}</td>
<td>{r.attempts}/{r.max_attempts}</td>
<td title={new Date(r.updated_at).toLocaleString()}
>{fmtAgo(r.updated_at)}</td
>
<td class="actions">
{#if r.state === 'dead'}
<button
type="button"
disabled={busy}
onclick={(e) => {
e.stopPropagation();
requeue({ scope: 'job', job_id: r.id });
}}
data-testid={`crawler-history-requeue-${r.id}`}
>
⤴ Requeue
</button>
{/if}
</td>
</tr>
{#if expanded === r.id && r.last_error}
<tr class="errrow">
<td colspan="7"><code>{r.last_error}</code></td>
</tr>
{/if}
{/each}
</tbody>
</table>
<Pager {page} {totalPages} onChange={onPageChange} testid="crawler-history-pager" />
{/if}
</section>
<style>
.toolbar {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: var(--space-2);
margin-bottom: var(--space-2);
}
select {
height: 36px;
padding: 0 var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
.note {
font-size: var(--font-xs);
}
.muted {
color: var(--text-muted);
}
table {
width: 100%;
border-collapse: collapse;
}
th,
td {
padding: var(--space-2);
text-align: left;
border-bottom: 1px solid var(--border);
font-size: var(--font-sm);
}
.kind {
color: var(--text-muted);
font-family: var(--font-mono, monospace);
font-size: var(--font-xs);
}
.actions {
text-align: right;
}
.num {
text-align: right;
font-variant-numeric: tabular-nums;
}
tr.clickable {
cursor: pointer;
}
tr.clickable:hover {
background: var(--surface);
}
.errrow td {
background: var(--surface);
color: var(--text-muted);
}
.errrow code {
white-space: pre-wrap;
word-break: break-word;
font-size: var(--font-xs);
}
.error {
color: var(--danger, #dc2626);
}
/* State dots reuse the page's shared badge palette. */
.badge {
text-transform: uppercase;
}
.state-done {
background: #dcfce7;
color: #166534;
border-color: #86efac;
}
.state-dead {
background: color-mix(in srgb, var(--danger, #dc2626) 16%, transparent);
color: var(--danger, #dc2626);
border-color: color-mix(in srgb, var(--danger, #dc2626) 45%, transparent);
}
.state-running,
.state-pending {
background: #fef3c7;
color: #92400e;
border-color: #fcd34d;
}
</style>

330
frontend/src/lib/components/crawler/CrawlerMetricsPanel.svelte Normal file
View File

@@ -0,0 +1,330 @@
<script lang="ts">
import { onMount } from 'svelte';
import Pager from '$lib/components/Pager.svelte';
import { fmtDuration } from '$lib/format';
import {
getCrawlerMetrics,
listCrawlerOps,
type OpSummary,
type OpRow,
type CrawlOp
} from '$lib/api/admin';
const LIMIT = 25;
let days = $state(7);
let summary = $state<OpSummary[]>([]);
let summaryLoading = $state(true);
let rows = $state<OpRow[]>([]);
let total = $state(0);
let page = $state(1);
let opFilter = $state<'' | CrawlOp>('');
let outcomeFilter = $state<'' | 'ok' | 'failed'>('');
let opsLoading = $state(true);
let error = $state<string | null>(null);
let expanded = $state<string | null>(null);
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
// Human labels + canonical order for the summary table.
const OP_LABELS: Record<CrawlOp, string> = {
manga_list: 'manga list walk',
manga_detail: 'manga detail',
manga_cover: 'manga cover',
chapter: 'whole chapter'
};
const OP_ORDER: CrawlOp[] = ['manga_list', 'manga_detail', 'manga_cover', 'chapter'];
const orderedSummary = $derived(
OP_ORDER.map((op) => summary.find((s) => s.op === op)).filter(
(s): s is OpSummary => s != null
)
);
// Derived per-page crawl time from the chapter roll-up (Σms ÷ Σpages ≈
// avg_ms ÷ avg_items). Shown as an indented sub-row under "whole chapter".
const chapter = $derived(summary.find((s) => s.op === 'chapter'));
const perPageMs = $derived(
chapter && chapter.avg_ms != null && chapter.avg_items && chapter.avg_items > 0
? chapter.avg_ms / chapter.avg_items
: null
);
function successPct(s: OpSummary): string {
if (s.n === 0) return '—';
return `${Math.round((s.ok / s.n) * 100)}%`;
}
async function loadSummary() {
summaryLoading = true;
try {
summary = (await getCrawlerMetrics(days)).summary;
} catch (e) {
error = e instanceof Error ? e.message : 'Failed to load metrics.';
} finally {
summaryLoading = false;
}
}
async function loadOps() {
opsLoading = true;
try {
const resp = await listCrawlerOps({
op: opFilter || undefined,
outcome: outcomeFilter || undefined,
days,
limit: LIMIT,
offset: (page - 1) * LIMIT
});
rows = resp.items;
total = resp.page.total ?? resp.items.length;
} catch (e) {
error = e instanceof Error ? e.message : 'Failed to load operations.';
} finally {
opsLoading = false;
}
}
onMount(() => {
loadSummary();
loadOps();
});
function onWindowChange() {
page = 1;
loadSummary();
loadOps();
}
function onOpsFilter() {
page = 1;
loadOps();
}
function onPageChange(p: number) {
page = p;
loadOps();
}
function opLabel(op: CrawlOp): string {
return OP_LABELS[op] ?? op;
}
function fmtAgo(iso: string): string {
const secs = Math.round((Date.now() - new Date(iso).getTime()) / 1000);
if (secs < 45) return 'just now';
const mins = Math.round(secs / 60);
if (mins < 60) return `${mins}m ago`;
const hrs = Math.round(mins / 60);
if (hrs < 24) return `${hrs}h ago`;
return new Date(iso).toLocaleDateString();
}
</script>
<section data-testid="crawler-metrics">
<div class="winrow">
<label>
Window
<select
bind:value={days}
onchange={onWindowChange}
data-testid="crawler-metrics-window"
>
<option value={1}>24 hours</option>
<option value={7}>7 days</option>
<option value={30}>30 days</option>
<option value={0}>All time</option>
</select>
</label>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
<h2>Average durations by type</h2>
{#if summaryLoading}
<p class="muted">Loading…</p>
{:else if orderedSummary.length === 0}
<p class="muted" data-testid="crawler-metrics-empty">No operations recorded yet.</p>
{:else}
<table>
<thead>
<tr>
<th>Type</th>
<th class="num">Avg</th>
<th class="num">N</th>
<th class="num">OK / Fail</th>
<th class="num">Success</th>
</tr>
</thead>
<tbody>
{#each orderedSummary as s (s.op)}
<tr data-testid={`crawler-metrics-row-${s.op}`}>
<td>{opLabel(s.op)}</td>
<td class="num">{fmtDuration(s.avg_ms)}</td>
<td class="num">{s.n}</td>
<td class="num">{s.ok} / {s.failed}</td>
<td class="num">{successPct(s)}</td>
</tr>
{#if s.op === 'chapter' && perPageMs != null}
<tr class="derived" data-testid="crawler-metrics-perpage">
<td>└ per page (≈)</td>
<td class="num">{fmtDuration(perPageMs)}</td>
<td class="num" colspan="3"
>≈{Math.round(chapter?.avg_items ?? 0)} pages/chapter</td
>
</tr>
{/if}
{/each}
</tbody>
</table>
{/if}
<h2>Recent operations</h2>
<div class="toolbar">
<select bind:value={opFilter} onchange={onOpsFilter} aria-label="Filter by type">
<option value="">All types</option>
{#each OP_ORDER as op (op)}<option value={op}>{opLabel(op)}</option>{/each}
</select>
<select
bind:value={outcomeFilter}
onchange={onOpsFilter}
aria-label="Filter by outcome"
>
<option value="">All outcomes</option>
<option value="ok">ok</option>
<option value="failed">failed</option>
</select>
</div>
{#if opsLoading}
<p class="muted">Loading…</p>
{:else if rows.length === 0}
<p class="muted" data-testid="crawler-metrics-ops-empty">No operations match.</p>
{:else}
<table>
<thead>
<tr>
<th>Type</th>
<th>Target</th>
<th class="num">Dur</th>
<th>Outcome</th>
<th>When</th>
</tr>
</thead>
<tbody>
{#each rows as r (r.id)}
<tr
class:clickable={!!r.error}
onclick={() => r.error && (expanded = expanded === r.id ? null : r.id)}
data-testid={`crawler-op-${r.id}`}
>
<td>{opLabel(r.op)}</td>
<td>
{r.manga_title
? `${r.manga_title}${r.chapter_number != null ? ` · Ch ${r.chapter_number}` : ''}`
: '—'}
</td>
<td class="num">{fmtDuration(r.duration_ms)}</td>
<td>
<span class="dot {r.outcome}"></span>{r.outcome}
</td>
<td title={new Date(r.finished_at).toLocaleString()}
>{fmtAgo(r.finished_at)}</td
>
</tr>
{#if expanded === r.id && r.error}
<tr class="errrow"><td colspan="5"><code>{r.error}</code></td></tr>
{/if}
{/each}
</tbody>
</table>
<Pager {page} {totalPages} onChange={onPageChange} testid="crawler-metrics-pager" />
{/if}
</section>
<style>
.winrow {
display: flex;
justify-content: flex-end;
margin-bottom: var(--space-2);
}
label {
font-size: var(--font-sm);
color: var(--text-muted);
}
select {
height: 32px;
margin-left: var(--space-1);
padding: 0 var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
h2 {
margin: var(--space-4) 0 var(--space-2);
font-size: var(--font-sm);
color: var(--text-muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.toolbar {
display: flex;
gap: var(--space-2);
margin-bottom: var(--space-2);
}
table {
width: 100%;
border-collapse: collapse;
max-width: 48rem;
}
th,
td {
padding: var(--space-2);
text-align: left;
border-bottom: 1px solid var(--border);
font-size: var(--font-sm);
}
.num {
text-align: right;
font-variant-numeric: tabular-nums;
}
.derived td {
color: var(--text-muted);
font-size: var(--font-xs);
}
tr.clickable {
cursor: pointer;
}
tr.clickable:hover {
background: var(--surface);
}
.errrow td {
background: var(--surface);
}
.errrow code {
white-space: pre-wrap;
word-break: break-word;
font-size: var(--font-xs);
color: var(--text-muted);
}
.dot {
display: inline-block;
width: 8px;
height: 8px;
border-radius: 50%;
margin-right: 6px;
vertical-align: middle;
}
.dot.ok {
background: #2e7d32;
}
.dot.failed {
background: var(--danger);
}
.muted {
color: var(--text-muted);
}
.error {
color: var(--danger);
}
</style>

29
frontend/src/lib/format.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { describe, it, expect } from 'vitest';
import { fmtDuration } from './format';
describe('fmtDuration', () => {
it('renders sub-second as ms', () => {
expect(fmtDuration(0)).toBe('0ms');
expect(fmtDuration(420)).toBe('420ms');
expect(fmtDuration(999)).toBe('999ms');
});
it('renders seconds with one decimal under 10s, none above', () => {
expect(fmtDuration(6100)).toBe('6.1s');
expect(fmtDuration(42000)).toBe('42s');
});
it('renders minutes + zero-padded seconds', () => {
expect(fmtDuration(62000)).toBe('1m 02s');
expect(fmtDuration(125000)).toBe('2m 05s');
});
it('rounds the seconds carry into minutes (no "1m 60s")', () => {
expect(fmtDuration(119600)).toBe('2m 00s');
});
it('renders null/undefined as an em-dash', () => {
expect(fmtDuration(null)).toBe('—');
expect(fmtDuration(undefined)).toBe('—');
});
});

14
frontend/src/lib/format.ts Normal file
View File

@@ -0,0 +1,14 @@
/** Human-readable duration from milliseconds.
* `420 → "420ms"`, `6100 → "6.1s"`, `62000 → "1m 02s"`, `null → "—"`. */
export function fmtDuration(ms: number | null | undefined): string {
if (ms == null) return '—';
if (ms < 1000) return `${Math.round(ms)}ms`;
const secs = ms / 1000;
if (secs < 60) return `${secs.toFixed(secs < 10 ? 1 : 0)}s`;
// Round to whole seconds *before* splitting so e.g. 119_600ms renders
// "2m 00s", not "1m 60s".
const whole = Math.round(secs);
const mins = Math.floor(whole / 60);
const rem = whole % 60;
return `${mins}m ${String(rem).padStart(2, '0')}s`;
}

175
frontend/src/routes/admin/analysis/+page.svelte
View File

@@ -1,5 +1,5 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
import { onMount, onDestroy, tick } from 'svelte';
import { ApiError } from '$lib/api/client';
import {
reenqueueAnalysis,
@@ -19,9 +19,16 @@
import { chapterLabel } from '$lib/api/chapters';
import CoverageBadge from '$lib/components/CoverageBadge.svelte';
import Modal from '$lib/components/Modal.svelte';
import AnalysisHistoryTable from '$lib/components/analysis/AnalysisHistoryTable.svelte';
import AnalysisMetricsPanel from '$lib/components/analysis/AnalysisMetricsPanel.svelte';
import SegmentedControl from '$lib/components/SegmentedControl.svelte';
const LIMIT = 25;
// Segmented Live/History/Metrics view. Live = the SSE coverage
// dashboard; History = a searchable log; Metrics = durations/averages.
let view = $state<'live' | 'history' | 'metrics'>('live');
// Global enqueue toggle.
let includeAnalyzed = $state(false);
@@ -65,6 +72,44 @@
let source: EventSource | null = null;
let sseErrors = 0;
// Global "now analyzing" pointer, driven by the `started` SSE event so
// the operator sees what the worker is on even when nothing is
// expanded. `phase` flips to done/failed for a moment on the matching
// completed/failed event, then the banner clears.
type NowAnalyzing = {
manga_id: string;
manga_title: string;
chapter_id: string;
chapter_number: number;
page_id: string;
page_number: number;
phase: 'analyzing' | 'done' | 'failed';
};
let nowAnalyzing = $state<NowAnalyzing | null>(null);
let clearTimer: ReturnType<typeof setTimeout> | null = null;
function scheduleClear(pageId: string) {
if (clearTimer) clearTimeout(clearTimer);
clearTimer = setTimeout(() => {
// Don't clear if a newer page took over the banner.
if (nowAnalyzing?.page_id === pageId) nowAnalyzing = null;
}, 4000);
}
/** Switch to Live, expand the now-analyzing chapter, scroll its chip. */
async function jumpToNowAnalyzing() {
const t = nowAnalyzing;
if (!t) return;
view = 'live';
await tick();
if (expandedManga !== t.manga_id) await toggleManga(t.manga_id);
if (expandedChapter !== t.chapter_id) await toggleChapter(t.chapter_id);
await tick();
document
.querySelector(`[data-testid="admin-analysis-page-${t.page_id}"]`)
?.scrollIntoView({ behavior: 'smooth', block: 'center' });
}
onMount(() => {
void loadCoverage(true);
openStream();
@@ -73,6 +118,7 @@
onDestroy(() => {
source?.close();
source = null;
if (clearTimer) clearTimeout(clearTimer);
});
function pushTicker(text: string) {
@@ -155,16 +201,34 @@
}
if (ev.kind === 'started') {
liveStatus = { ...liveStatus, [ev.page_id]: 'analyzing' };
pushTicker(`Analyzing page ${ev.page_number}`);
if (clearTimer) clearTimeout(clearTimer);
nowAnalyzing = {
manga_id: ev.manga_id,
manga_title: ev.manga_title,
chapter_id: ev.chapter_id,
chapter_number: ev.chapter_number,
page_id: ev.page_id,
page_number: ev.page_number,
phase: 'analyzing'
};
pushTicker(`Analyzing ${ev.manga_title} · p${ev.page_number}`);
} else if (ev.kind === 'completed') {
liveStatus = { ...liveStatus, [ev.page_id]: 'done' };
if (!counted.has(ev.page_id)) {
counted.add(ev.page_id);
bumpCoverage(ev.manga_id, ev.chapter_id);
}
if (nowAnalyzing?.page_id === ev.page_id) {
nowAnalyzing = { ...nowAnalyzing, phase: 'done' };
scheduleClear(ev.page_id);
}
pushTicker(`✓ Analyzed page ${ev.page_number}`);
} else if (ev.kind === 'failed') {
liveStatus = { ...liveStatus, [ev.page_id]: 'failed' };
if (nowAnalyzing?.page_id === ev.page_id) {
nowAnalyzing = { ...nowAnalyzing, phase: 'failed' };
scheduleClear(ev.page_id);
}
pushTicker(`✗ Failed page ${ev.page_number}`);
}
}
@@ -393,6 +457,60 @@
page's result. Updates stream live as pages are queued and processed.
</p>
<div class="viewtabs">
<SegmentedControl
options={[
{ label: 'Live', value: 'live' },
{ label: 'History', value: 'history' },
{ label: 'Metrics', value: 'metrics' }
]}
value={view}
onchange={(v) => (view = v)}
ariaLabel="Analysis view"
testid="admin-analysis-tab"
/>
</div>
{#if nowAnalyzing}
<div
class="now-analyzing {nowAnalyzing.phase}"
data-testid="admin-analysis-now"
aria-live="polite"
>
<span class="now-icon">
{nowAnalyzing.phase === 'done' ? '✓' : nowAnalyzing.phase === 'failed' ? '✗' : '⟳'}
</span>
<span class="now-label">
<strong
>{nowAnalyzing.phase === 'done'
? 'Analyzed'
: nowAnalyzing.phase === 'failed'
? 'Failed'
: 'Now analyzing'}</strong
>
{nowAnalyzing.manga_title} · Ch {nowAnalyzing.chapter_number} · Page
{nowAnalyzing.page_number}
</span>
<button
type="button"
class="now-jump"
onclick={jumpToNowAnalyzing}
data-testid="admin-analysis-now-jump"
>
Jump →
</button>
</div>
{/if}
{#if view === 'history'}
<AnalysisHistoryTable onOpenDetail={openDetail} />
{/if}
{#if view === 'metrics'}
<AnalysisMetricsPanel />
{/if}
{#if view === 'live'}
{#if ticker.length > 0}
<ul class="ticker" data-testid="admin-analysis-ticker" aria-live="polite">
{#each ticker as t (t.id)}
@@ -586,6 +704,7 @@
{#if error}
<p class="error" role="alert" data-testid="admin-analysis-error">{error}</p>
{/if}
{/if}
<Modal
open={detailOpen}
@@ -708,6 +827,58 @@
.live-pill.on .live-dot {
animation: pulse 1.6s ease-in-out infinite;
}
.viewtabs {
margin-bottom: var(--space-3);
}
.now-analyzing {
position: sticky;
top: var(--space-2);
z-index: 5;
display: flex;
align-items: center;
gap: var(--space-3);
max-width: 44rem;
margin-bottom: var(--space-3);
padding: var(--space-2) var(--space-3);
border: 1px solid color-mix(in srgb, #d4762a 45%, transparent);
border-radius: var(--radius-md);
background: color-mix(in srgb, #d4762a 12%, transparent);
font-size: var(--font-sm);
}
.now-analyzing.done {
border-color: color-mix(in srgb, #2e7d32 45%, transparent);
background: color-mix(in srgb, #2e7d32 12%, transparent);
}
.now-analyzing.failed {
border-color: color-mix(in srgb, var(--danger) 45%, transparent);
background: color-mix(in srgb, var(--danger) 12%, transparent);
}
.now-icon {
font-size: var(--font-lg);
color: #b85e1a;
}
.now-analyzing.analyzing .now-icon {
animation: spin 1.4s linear infinite;
display: inline-block;
}
.now-analyzing.done .now-icon {
color: #2e7d32;
}
.now-analyzing.failed .now-icon {
color: var(--danger);
}
.now-label {
flex: 1;
min-width: 0;
}
.now-jump {
flex-shrink: 0;
}
@keyframes spin {
to {
transform: rotate(360deg);
}
}
.ticker {
list-style: none;
margin: 0 0 var(--space-3);

30
frontend/src/routes/admin/crawler/+page.svelte
View File

@@ -6,6 +6,9 @@
import ActiveJobsTable from '$lib/components/crawler/ActiveJobsTable.svelte';
import MissingCoversTable from '$lib/components/crawler/MissingCoversTable.svelte';
import DeadJobsTable from '$lib/components/crawler/DeadJobsTable.svelte';
import CrawlerHistoryTable from '$lib/components/crawler/CrawlerHistoryTable.svelte';
import CrawlerMetricsPanel from '$lib/components/crawler/CrawlerMetricsPanel.svelte';
import SegmentedControl from '$lib/components/SegmentedControl.svelte';
import SessionModal from '$lib/components/crawler/SessionModal.svelte';
import RestartConfirmModal from '$lib/components/crawler/RestartConfirmModal.svelte';
import RequeueAllConfirmModal from '$lib/components/crawler/RequeueAllConfirmModal.svelte';
@@ -27,6 +30,10 @@
type RequeueScope
} from '$lib/api/admin';
// Segmented Live/History/Metrics view. Live keeps the SSE-driven
// dashboard; History is a searchable job log; Metrics shows durations.
let view = $state<'live' | 'history' | 'metrics'>('live');
let status: CrawlerStatus | null = $state(null);
let error: string | null = $state(null);
let notice: string | null = $state(null);
@@ -387,6 +394,20 @@
</span>
</div>
<div class="viewtabs">
<SegmentedControl
options={[
{ label: 'Live', value: 'live' },
{ label: 'History', value: 'history' },
{ label: 'Metrics', value: 'metrics' }
]}
value={view}
onchange={(v) => (view = v)}
ariaLabel="Crawler view"
testid="crawler-tab"
/>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
@@ -394,6 +415,11 @@
<p class="notice" role="status">{notice}</p>
{/if}
{#if view === 'history'}
<CrawlerHistoryTable onRequeue={requeue} {busy} />
{:else if view === 'metrics'}
<CrawlerMetricsPanel />
{:else}
{#if status}
<CrawlerHero {status} />
@@ -446,6 +472,7 @@
onRequeue={requeue}
onRequeueAll={() => (requeueAllModalOpen = true)}
/>
{/if}
<RestartConfirmModal
open={restartModalOpen}
@@ -488,6 +515,9 @@
.livedot.on {
color: var(--success, #0a7d2c);
}
.viewtabs {
margin-bottom: var(--space-4);
}
.notice {
color: var(--success, #0a7d2c);
padding: var(--space-2) var(--space-3);