diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3044d5b..ae91b23 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.85.0" +version = "0.85.1" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2aa00bc..10cf0b7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.85.0" +version = "0.85.1" edition = "2021" default-run = "mangalord" diff --git a/backend/migrations/0029_crawler_jobs_sync_chapter_list_index.sql b/backend/migrations/0029_crawler_jobs_sync_chapter_list_index.sql new file mode 100644 index 0000000..e83c6e2 --- /dev/null +++ b/backend/migrations/0029_crawler_jobs_sync_chapter_list_index.sql @@ -0,0 +1,24 @@ +-- Index the `sync_chapter_list` in-flight path used by the admin sync-state +-- derivation (overview `manga_stats` + the Mangas tab listing). +-- +-- A manga is "in_progress" if a pending/running job targets it. For the +-- `sync_manga` kind that join is already covered by +-- crawler_jobs_sync_manga_key_idx (0020). The OTHER kind, `sync_chapter_list`, +-- carries the target `manga_id` directly in its payload and had NO index — so +-- the EXISTS fell back to a full seqscan of crawler_jobs. Per manga that is +-- cheap; across the whole library (overview scans every manga) it is O(mangas +-- x jobs), and the disabled-analysis backlog (thousands of pending +-- `analyze_page` rows that can never match this filter) inflates every scan. +-- +-- Partial on the same `state IN ('pending','running') AND kind = ...` +-- predicate as the sibling indexes so it stays tiny (only in-flight list +-- jobs) and Postgres can probe it instead of scanning. Mirrors 0020. +-- +-- Not CONCURRENTLY: sqlx::migrate! wraps each migration in a transaction; +-- CREATE INDEX CONCURRENTLY can't run inside one. The table is small at our +-- scale so a brief build lock on deploy is safe. IF NOT EXISTS keeps it +-- idempotent with any operator who pre-created it on the live DB. +CREATE INDEX IF NOT EXISTS crawler_jobs_sync_chapter_list_manga_idx + ON crawler_jobs ((payload->>'manga_id')) + WHERE state IN ('pending', 'running') + AND payload->>'kind' = 'sync_chapter_list'; diff --git a/backend/src/api/admin/overview.rs b/backend/src/api/admin/overview.rs index f206495..3af942c 100644 --- a/backend/src/api/admin/overview.rs +++ b/backend/src/api/admin/overview.rs @@ -9,11 +9,15 @@ //! //! Admin-only (`RequireAdmin`, cookie-only). +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + use axum::extract::State; use axum::routing::get; use axum::{Json, Router}; use chrono::{DateTime, Utc}; use serde::Serialize; +use tokio::sync::Mutex; use crate::app::AppState; use crate::auth::extractor::RequireAdmin; @@ -24,14 +28,14 @@ pub fn routes() -> Router { Router::new().route("/admin/overview", get(overview)) } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct OverviewStats { pub users: UsersOverview, pub mangas: repo::admin_view::MangaStats, pub analysis: AnalysisOverview, } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct UsersOverview { pub total: i64, pub admins: i64, @@ -39,16 +43,42 @@ pub struct UsersOverview { pub newest_created_at: Option>, } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct AnalysisOverview { pub analyzed_pages: i64, pub total_pages: i64, } +/// Process-wide single-flight cache for the overview aggregate. +/// +/// The admin dashboard polls this endpoint on a timer; without coalescing, a +/// slow DB (e.g. a deep crawl/analysis backlog) lets identical requests stack +/// and pin Postgres. Holding the async mutex across the recompute gives BOTH a +/// short-TTL cache and single-flight: concurrent pollers block on the lock, +/// then read the freshly cached value instead of each launching their own +/// aggregate. Resets on process restart, which is fine for a dashboard stat. +/// +/// Unlike per-`AppState` shared state (e.g. `auth_limiter`), this is a single +/// production-instance cache, so a process global is adequate. It is +/// test-safe because the cache lives in the handler *body*: the two auth tests +/// are rejected at the `RequireAdmin` extractor before reaching it, and the +/// one aggregate test issues a single request. A future test that mutates data +/// and re-reads within the TTL would need to account for this. +static OVERVIEW_CACHE: OnceLock>> = OnceLock::new(); +const OVERVIEW_TTL: Duration = Duration::from_secs(10); + async fn overview( State(state): State, _admin: RequireAdmin, ) -> AppResult> { + let cache = OVERVIEW_CACHE.get_or_init(|| Mutex::new(None)); + let mut guard = cache.lock().await; + if let Some((at, cached)) = guard.as_ref() { + if at.elapsed() < OVERVIEW_TTL { + return Ok(Json(cached.clone())); + } + } + // Independent reads — run concurrently so latency is the slowest // query, not their sum (mirrors the storage handler). let (user_counts, user_newest, mangas, coverage) = tokio::try_join!( @@ -65,7 +95,7 @@ async fn overview( }; let (analyzed_pages, total_pages) = coverage; - Ok(Json(OverviewStats { + let stats = OverviewStats { users: UsersOverview { total, admins, @@ -77,5 +107,7 @@ async fn overview( analyzed_pages, total_pages, }, - })) + }; + *guard = Some((Instant::now(), stats.clone())); + Ok(Json(stats)) } diff --git a/backend/src/repo/admin_view.rs b/backend/src/repo/admin_view.rs index f7cf2d2..1064462 100644 --- a/backend/src/repo/admin_view.rs +++ b/backend/src/repo/admin_view.rs @@ -85,7 +85,7 @@ const MANGA_SYNC_STATE_CASE: &str = r#" /// Library shape for the admin overview: total mangas split by derived /// sync state, plus library-wide chapter/page totals and the newest manga. -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct MangaStats { pub total: i64, pub synced: i64, @@ -97,23 +97,71 @@ pub struct MangaStats { pub newest_seen_at: Option>, } -/// Aggregate the manga library for the overview dashboard. Sync-state -/// counts reuse `MANGA_SYNC_STATE_CASE` so they can't drift from the -/// per-row classification on the Mangas tab. +/// Aggregate the manga library for the overview dashboard. +/// +/// Unlike the paginated Mangas tab (`list_mangas_with_sync_state`, which +/// evaluates `MANGA_SYNC_STATE_CASE` for only the page slice), this scans the +/// WHOLE library. The per-row correlated form is therefore O(mangas x jobs): +/// each manga seqscans `crawler_jobs`, and the disabled-analysis backlog +/// inflates every scan. Instead we collect the set of "in-flight" manga ids in +/// ONE pass over the in-flight sync jobs (index-backed: 0020 + 0029), then +/// classify each manga with a hash semi-join — O(mangas + jobs). +/// +/// The `in_progress` rule here MUST stay in lockstep with the first arm of +/// `MANGA_SYNC_STATE_CASE`: pending/running `sync_chapter_list` (by manga_id) +/// OR `sync_manga` (resolved through `manga_sources`). The `dropped`/`synced` +/// arms are identical to the shared case. pub async fn manga_stats(pool: &PgPool) -> AppResult { - let counts_sql = format!( - r#" + let counts_sql = r#" + WITH in_progress_mangas AS ( + -- sync_chapter_list jobs carry the target manga_id directly + SELECT (cj.payload->>'manga_id')::uuid AS manga_id + FROM crawler_jobs cj + WHERE cj.state IN ('pending', 'running') + AND cj.payload->>'kind' = 'sync_chapter_list' + AND cj.payload->>'manga_id' IS NOT NULL + UNION + -- sync_manga jobs resolve to a manga via manga_sources + SELECT ms.manga_id + FROM crawler_jobs cj + JOIN manga_sources ms + ON ms.source_id = cj.payload->>'source_id' + AND ms.source_manga_key = cj.payload->>'source_manga_key' + WHERE cj.state IN ('pending', 'running') + AND cj.payload->>'kind' = 'sync_manga' + ) SELECT COUNT(*)::bigint AS total, COUNT(*) FILTER (WHERE s = 'synced')::bigint AS synced, COUNT(*) FILTER (WHERE s = 'in_progress')::bigint AS in_progress, COUNT(*) FILTER (WHERE s = 'dropped')::bigint AS dropped - FROM (SELECT {case} AS s FROM mangas m) q - "#, - case = MANGA_SYNC_STATE_CASE - ); + FROM ( + SELECT + CASE + WHEN m.id IN (SELECT manga_id FROM in_progress_mangas) + THEN 'in_progress' + WHEN EXISTS (SELECT 1 FROM manga_sources ms WHERE ms.manga_id = m.id) + AND NOT EXISTS ( + SELECT 1 FROM manga_sources ms + WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL + ) + THEN 'dropped' + ELSE 'synced' + END AS s + FROM mangas m + ) q + "#; + // Backstop: even with the single-pass rewrite + indexes this is cheap, but + // a `statement_timeout` guarantees a pathological plan can never pin a + // backend for minutes and let pollers stack again. SET LOCAL is scoped to + // the transaction. + let mut tx = pool.begin().await?; + sqlx::query("SET LOCAL statement_timeout = '5s'") + .execute(&mut *tx) + .await?; let (total, synced, in_progress, dropped): (i64, i64, i64, i64) = - sqlx::query_as(&counts_sql).fetch_one(pool).await?; + sqlx::query_as(counts_sql).fetch_one(&mut *tx).await?; + tx.commit().await?; let (total_chapters,): (i64,) = sqlx::query_as("SELECT COUNT(*)::bigint FROM chapters") diff --git a/frontend/package.json b/frontend/package.json index 028bd1f..548eb87 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.85.0", + "version": "0.85.1", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/routes/admin/+page.svelte b/frontend/src/routes/admin/+page.svelte index 707577d..bcc3b91 100644 --- a/frontend/src/routes/admin/+page.svelte +++ b/frontend/src/routes/admin/+page.svelte @@ -190,10 +190,13 @@ openCrawlerStream(); openAnalysisStream(); sysTimer = setInterval(refreshSys, 5000); + // 60s: the overview aggregate is server-cached (~10s TTL) and these + // are background freshness updates, so a slower poll halves idle DB + // load without the dashboard feeling stale. overviewTimer = setInterval(() => { refreshOverview(); refreshMetrics(); - }, 30000); + }, 60000); if (typeof document !== 'undefined') { document.addEventListener('visibilitychange', onVisibilityChange); window.addEventListener('pagehide', onPageHide);