feat(admin): crawler observability dashboard + reliability hardening (0.55.0)

Admin-only crawler dashboard backed by an SSE live-status stream, coordinated browser restart, runtime PHPSESSID refresh, dead-letter requeue, and a batch of reliability fixes. Closes everything from the two-pass audit (10 commits' worth) and bumps 0.52.0 -> 0.55.0. Backend: - New /admin/crawler/* surface (cookie-auth, RequireAdmin) split into status / control / dead_jobs / backlog modules. SSE stream composes in-memory status with DB-derived queue counts, memoizes the counts for 1s and debounces watch pokes for 250ms (~10x QPS reduction per subscriber). One-shot GET /admin/crawler shares the same compose path. - POST /admin/crawler/run gated by manual_pass_lock try_lock_owned (409 Conflict on overlapping click); browser restart goes through the coordinated_restart gate (drain + relaunch + auto-clear of the sticky session_expired flag on Ok). - Runtime PHPSESSID refresh via SessionController (allow-list validation, never logged, audit row carries SHA-256 fingerprint). Storage layer is repo::crawler::runtime_session_{load,persist}. - Dead-letter requeue with four scopes (all/manga/chapter/job); scope=all requires confirm:true; DISTINCT ON dedup keeps the partial unique index from rejecting requeues for chapters with multiple dead rows. SQL is four &'static str constants per scope. - StatusHandle + ChapterGuard / CoverGuard RAII model survives panics; last-writer-wins on cover so concurrent dispatches don't clobber each other's slot. Pure functions (should_stop / should_mark_clean_exit / should_abort_pass) with named regression tests. - Reliability bundle: per-lease heartbeat, jitter on retries, per-job timeout, circuit breaker on consecutive failures, BrowserManager coordinated restart gate, request fingerprint changes. - Streaming page download: Storage::put_stream trait method, LocalStorage impl atomic via temp + fsync + UUID-suffixed rename. Pages stream through with peak memory ~one HTTP chunk + 64-byte sniff prefix instead of one full image per dispatch. - New partial indexes (migration 0022): mangas_missing_cover_idx and crawler_jobs_dead_idx, both ordered by updated_at DESC to match the dashboard's LIMIT/OFFSET reads. - Security hardening: admin_csrf_guard (Origin/Referer allowlist on /admin/* mutations, opt-in via ADMIN_ALLOWED_ORIGINS), admin_no_store_guard (Cache-Control: no-store on admin responses), audit rows carry per-scope target_id. Frontend: - /admin/crawler page decomposed into lib/components/crawler/ (11 components: ProgressBar, SearchBar, CrawlerHero, CrawlerControls, ActiveChaptersCard, ActiveJobsTable, MissingCoversTable, DeadJobsTable, RestartConfirmModal, RequeueAllConfirmModal, SessionModal). Page is 532 LOC of orchestration; each component 22-148 LOC. - EventSource lifecycle wired to visibilitychange / pagehide / pageshow (BFCache); after 5 consecutive errors probes the status endpoint so a 401 routes through the global on401Hook instead of infinite silent reconnects. - Backlog $effect refetches debounced 500ms with per-loader AbortControllers; refresh after a control action only runs when the SSE stream is dead. - Inline requeue button on /admin/mangas patches the affected row's sync_state locally (no full chapter-list refetch); proper aria-label. Requeue-all gets its own confirm modal; both confirm modals autofocus Cancel. - SvelteKit reverse proxy bypasses its 5-minute AbortController for Accept: text/event-stream; pure shouldBypassProxyTimeout helper covered by unit tests. Config / docs: - New env vars (.env.example): ADMIN_ALLOWED_ORIGINS, CRAWLER_JOB_TIMEOUT_SECS, CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES, CRAWLER_BROWSER_RESTART_THRESHOLD. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-06 18:49:56 +02:00
parent 679abae736
commit d6ac648ac9
54 changed files with 6076 additions and 137 deletions
--- a/frontend/src/lib/api/admin.ts
+++ b/frontend/src/lib/api/admin.ts
@@ -3,7 +3,7 @@
 // won't reach these routes). 403s thrown here propagate up to the
 // /admin layout, which renders the framework error page.

-import { request, type Page } from './client';
+import { request, apiUrl, type Page } from './client';
 import type { User } from './auth';
 import type { MangaDetail } from './mangas';
 import type { Chapter } from './chapters';
@@ -214,3 +214,192 @@ export async function resyncChapter(id: string): Promise<ChapterResyncResponse>
        { method: 'POST' }
    );
 }
+
+// ---- crawler observability + control ---------------------------------------
+
+/** Current daemon activity. Discriminated on `state`. */
+export type CrawlerPhase =
+    | { state: 'idle'; next_fire: string | null }
+    | { state: 'walking_list' }
+    | { state: 'fetching_metadata'; index: number; total: number | null; title: string }
+    | { state: 'cover_backfill'; index: number; total: number };
+
+/** A chapter being crawled right now, with a live page count. */
+export type ActiveChapter = {
+    manga_id: string;
+    manga_title: string;
+    chapter_id: string;
+    chapter_number: number;
+    pages_done: number;
+    pages_total: number | null;
+};
+
+export type CrawlerLastPass = {
+    at: string | null;
+    discovered: number;
+    upserted: number;
+    covers_fetched: number;
+    mangas_failed: number;
+};
+
+export type CrawlerStatus = {
+    daemon: 'running' | 'disabled';
+    phase: CrawlerPhase | null;
+    worker_count: number;
+    active_chapters: ActiveChapter[];
+    current_cover: { manga_id: string; manga_title: string } | null;
+    covers_queued: number;
+    last_pass: CrawlerLastPass;
+    session: { expired: boolean; configured: boolean };
+    browser: 'healthy' | 'draining' | 'restarting' | 'down';
+    queue: { pending: number; running: number; dead: number };
+};
+
+export async function getCrawlerStatus(): Promise<CrawlerStatus> {
+    return request<CrawlerStatus>('/v1/admin/crawler');
+}
+
+/** URL of the Server-Sent Events live-status stream. Open with
+ *  `new EventSource(...)` while the crawler page is mounted and close it on
+ *  navigate-away so the subscription is scoped to the active page. Each
+ *  message is a named `status` event whose `data` is a {@link CrawlerStatus}. */
+export function crawlerStatusStreamUrl(): string {
+    return apiUrl('/v1/admin/crawler/stream');
+}
+
+/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */
+export async function runCrawlerPass(): Promise<{ started: boolean }> {
+    return request('/v1/admin/crawler/run', { method: 'POST' });
+}
+
+/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
+export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
+    return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
+}
+
+/** POST /v1/admin/crawler/session — refresh PHPSESSID and re-probe. */
+export async function updateCrawlerSession(
+    phpsessid: string
+): Promise<{ valid: boolean; error: string | null }> {
+    return request('/v1/admin/crawler/session', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ phpsessid })
+    });
+}
+
+/** POST /v1/admin/crawler/session/clear-expired — resume idled workers. */
+export async function clearCrawlerSessionExpired(): Promise<{ cleared: boolean }> {
+    return request('/v1/admin/crawler/session/clear-expired', { method: 'POST' });
+}
+
+export type DeadJob = {
+    id: string;
+    kind: string;
+    chapter_id: string | null;
+    manga_id: string | null;
+    manga_title: string | null;
+    chapter_number: number | null;
+    attempts: number;
+    max_attempts: number;
+    last_error: string | null;
+    updated_at: string;
+};
+
+export type DeadJobsPage = { items: DeadJob[]; page: Page };
+
+export async function listDeadJobs(
+    opts?: {
+        search?: string;
+        limit?: number;
+        offset?: number;
+    },
+    init?: RequestInit
+): Promise<DeadJobsPage> {
+    const params = new URLSearchParams();
+    if (opts?.search) params.set('search', opts.search);
+    if (opts?.limit != null) params.set('limit', String(opts.limit));
+    if (opts?.offset != null) params.set('offset', String(opts.offset));
+    const qs = params.toString();
+    return request<DeadJobsPage>(
+        `/v1/admin/crawler/dead-jobs${qs ? `?${qs}` : ''}`,
+        init
+    );
+}
+
+/** Requeue scope: all dead jobs, one manga's, one chapter's, or a single job.
+ *  `scope: 'all'` requires an explicit `confirm: true` so a careless
+ *  click (or CSRF bait) can't flip the entire dead pile. */
+export type RequeueScope =
+    | { scope: 'all'; confirm: true }
+    | { scope: 'manga'; manga_id: string }
+    | { scope: 'chapter'; chapter_id: string }
+    | { scope: 'job'; job_id: string };
+
+export async function requeueDeadJobs(scope: RequeueScope): Promise<{ requeued: number }> {
+    return request('/v1/admin/crawler/dead-jobs/requeue', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify(scope)
+    });
+}
+
+/** A queued/running chapter-content job (which chapters are queued). */
+export type ActiveJob = {
+    id: string;
+    chapter_id: string | null;
+    manga_id: string | null;
+    manga_title: string | null;
+    chapter_number: number | null;
+    state: 'pending' | 'running';
+    attempts: number;
+    max_attempts: number;
+    updated_at: string;
+};
+
+export type ActiveJobsPage = { items: ActiveJob[]; page: Page };
+
+/** GET /v1/admin/crawler/active-jobs — which chapters of which mangas are
+ *  queued or running now. */
+export async function listActiveJobs(
+    opts?: {
+        search?: string;
+        limit?: number;
+        offset?: number;
+    },
+    init?: RequestInit
+): Promise<ActiveJobsPage> {
+    const params = new URLSearchParams();
+    if (opts?.search) params.set('search', opts.search);
+    if (opts?.limit != null) params.set('limit', String(opts.limit));
+    if (opts?.offset != null) params.set('offset', String(opts.offset));
+    const qs = params.toString();
+    return request<ActiveJobsPage>(
+        `/v1/admin/crawler/active-jobs${qs ? `?${qs}` : ''}`,
+        init
+    );
+}
+
+/** A manga queued for a cover fetch (no cover yet + a live source). */
+export type MissingCover = { manga_id: string; manga_title: string };
+export type MissingCoversPage = { items: MissingCover[]; page: Page };
+
+/** GET /v1/admin/crawler/covers — which manga covers are queued. */
+export async function listMissingCovers(
+    opts?: {
+        search?: string;
+        limit?: number;
+        offset?: number;
+    },
+    init?: RequestInit
+): Promise<MissingCoversPage> {
+    const params = new URLSearchParams();
+    if (opts?.search) params.set('search', opts.search);
+    if (opts?.limit != null) params.set('limit', String(opts.limit));
+    if (opts?.offset != null) params.set('offset', String(opts.offset));
+    const qs = params.toString();
+    return request<MissingCoversPage>(
+        `/v1/admin/crawler/covers${qs ? `?${qs}` : ''}`,
+        init
+    );
+}