feat(admin): crawler observability dashboard + reliability hardening (0.55.0)

Admin-only crawler dashboard backed by an SSE live-status stream,
coordinated browser restart, runtime PHPSESSID refresh, dead-letter
requeue, and a batch of reliability fixes. Closes everything from
the two-pass audit (10 commits' worth) and bumps 0.52.0 -> 0.55.0.

Backend:

- New /admin/crawler/* surface (cookie-auth, RequireAdmin) split
  into status / control / dead_jobs / backlog modules. SSE stream
  composes in-memory status with DB-derived queue counts, memoizes
  the counts for 1s and debounces watch pokes for 250ms (~10x QPS
  reduction per subscriber). One-shot GET /admin/crawler shares the
  same compose path.
- POST /admin/crawler/run gated by manual_pass_lock try_lock_owned
  (409 Conflict on overlapping click); browser restart goes through
  the coordinated_restart gate (drain + relaunch + auto-clear of the
  sticky session_expired flag on Ok).
- Runtime PHPSESSID refresh via SessionController (allow-list
  validation, never logged, audit row carries SHA-256 fingerprint).
  Storage layer is repo::crawler::runtime_session_{load,persist}.
- Dead-letter requeue with four scopes (all/manga/chapter/job);
  scope=all requires confirm:true; DISTINCT ON dedup keeps the
  partial unique index from rejecting requeues for chapters with
  multiple dead rows. SQL is four &'static str constants per scope.
- StatusHandle + ChapterGuard / CoverGuard RAII model survives
  panics; last-writer-wins on cover so concurrent dispatches don't
  clobber each other's slot. Pure functions (should_stop /
  should_mark_clean_exit / should_abort_pass) with named regression
  tests.
- Reliability bundle: per-lease heartbeat, jitter on retries,
  per-job timeout, circuit breaker on consecutive failures, BrowserManager
  coordinated restart gate, request fingerprint changes.
- Streaming page download: Storage::put_stream trait method,
  LocalStorage impl atomic via temp + fsync + UUID-suffixed rename.
  Pages stream through with peak memory ~one HTTP chunk + 64-byte
  sniff prefix instead of one full image per dispatch.
- New partial indexes (migration 0022): mangas_missing_cover_idx
  and crawler_jobs_dead_idx, both ordered by updated_at DESC to
  match the dashboard's LIMIT/OFFSET reads.
- Security hardening: admin_csrf_guard (Origin/Referer allowlist
  on /admin/* mutations, opt-in via ADMIN_ALLOWED_ORIGINS),
  admin_no_store_guard (Cache-Control: no-store on admin
  responses), audit rows carry per-scope target_id.

Frontend:

- /admin/crawler page decomposed into lib/components/crawler/
  (11 components: ProgressBar, SearchBar, CrawlerHero,
  CrawlerControls, ActiveChaptersCard, ActiveJobsTable,
  MissingCoversTable, DeadJobsTable, RestartConfirmModal,
  RequeueAllConfirmModal, SessionModal). Page is 532 LOC of
  orchestration; each component 22-148 LOC.
- EventSource lifecycle wired to visibilitychange / pagehide /
  pageshow (BFCache); after 5 consecutive errors probes the status
  endpoint so a 401 routes through the global on401Hook instead of
  infinite silent reconnects.
- Backlog $effect refetches debounced 500ms with per-loader
  AbortControllers; refresh after a control action only runs when
  the SSE stream is dead.
- Inline requeue button on /admin/mangas patches the affected row's
  sync_state locally (no full chapter-list refetch); proper
  aria-label. Requeue-all gets its own confirm modal; both confirm
  modals autofocus Cancel.
- SvelteKit reverse proxy bypasses its 5-minute AbortController
  for Accept: text/event-stream; pure shouldBypassProxyTimeout
  helper covered by unit tests.

Config / docs:

- New env vars (.env.example): ADMIN_ALLOWED_ORIGINS,
  CRAWLER_JOB_TIMEOUT_SECS, CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES,
  CRAWLER_BROWSER_RESTART_THRESHOLD.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

frontend/src/lib/api/admin.test.ts

@@ -16,7 +16,17 @@ import {
     listAdminChapters,
     getSystemStats,
     resyncManga,
-    resyncChapter
+    resyncChapter,
+    getCrawlerStatus,
+    crawlerStatusStreamUrl,
+    runCrawlerPass,
+    restartCrawlerBrowser,
+    updateCrawlerSession,
+    clearCrawlerSessionExpired,
+    listDeadJobs,
+    requeueDeadJobs,
+    listActiveJobs,
+    listMissingCovers
 } from './admin';
 
 function ok(body: unknown, status = 200): Response {
@@ -329,3 +339,159 @@ describe('admin api client', () => {
         expect(got.pages).toBeNull();
     });
 });
+
+describe('admin crawler api client', () => {
+    let fetchSpy: MockInstance<typeof globalThis.fetch>;
+    beforeEach(() => {
+        fetchSpy = vi.spyOn(globalThis, 'fetch');
+    });
+    afterEach(() => {
+        vi.restoreAllMocks();
+    });
+
+    const statusFixture = {
+        daemon: 'running',
+        phase: { state: 'fetching_metadata', index: 3, total: 10, title: 'One Piece' },
+        worker_count: 2,
+        active_chapters: [
+            {
+                manga_id: 'm-1',
+                manga_title: 'Bleach',
+                chapter_id: 'c-1',
+                chapter_number: 12,
+                pages_done: 4,
+                pages_total: 20
+            }
+        ],
+        current_cover: { manga_id: 'm-2', manga_title: 'Naruto' },
+        covers_queued: 7,
+        last_pass: { at: null, discovered: 0, upserted: 0, covers_fetched: 0, mangas_failed: 0 },
+        session: { expired: false, configured: true },
+        browser: 'healthy',
+        queue: { pending: 2, running: 1, dead: 4 }
+    };
+
+    it('crawlerStatusStreamUrl points at the SSE endpoint under the API base', () => {
+        expect(crawlerStatusStreamUrl()).toMatch(/\/v1\/admin\/crawler\/stream$/);
+    });
+
+    it('getCrawlerStatus GETs /v1/admin/crawler with live chapter/cover fields', async () => {
+        fetchSpy.mockResolvedValueOnce(ok(statusFixture));
+        const s = await getCrawlerStatus();
+        expect(s.queue.dead).toBe(4);
+        expect(s.phase?.state).toBe('fetching_metadata');
+        expect(s.active_chapters[0].pages_done).toBe(4);
+        expect(s.active_chapters[0].pages_total).toBe(20);
+        expect(s.current_cover?.manga_title).toBe('Naruto');
+        expect(s.covers_queued).toBe(7);
+        const url = fetchSpy.mock.calls[0][0] as string;
+        expect(url).toMatch(/\/v1\/admin\/crawler$/);
+    });
+
+    it('listActiveJobs GETs /v1/admin/crawler/active-jobs with search', async () => {
+        fetchSpy.mockResolvedValueOnce(
+            ok({ items: [], page: { limit: 20, offset: 0, total: 0 } })
+        );
+        await listActiveJobs({ search: 'bleach' });
+        const url = fetchSpy.mock.calls[0][0] as string;
+        expect(url).toMatch(/\/v1\/admin\/crawler\/active-jobs\?/);
+        expect(url).toContain('search=bleach');
+    });
+
+    it('listMissingCovers GETs /v1/admin/crawler/covers', async () => {
+        fetchSpy.mockResolvedValueOnce(
+            ok({ items: [{ manga_id: 'm-1', manga_title: 'X' }], page: { limit: 20, offset: 0, total: 1 } })
+        );
+        const r = await listMissingCovers();
+        expect(r.items[0].manga_title).toBe('X');
+        expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
+    });
+
+    it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
+        fetchSpy.mockResolvedValueOnce(ok({ started: true }));
+        const r = await runCrawlerPass();
+        expect(r.started).toBe(true);
+        const init = fetchSpy.mock.calls[0][1] as RequestInit;
+        expect(init.method).toBe('POST');
+        expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
+    });
+
+    it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
+        fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
+        const r = await restartCrawlerBrowser();
+        expect(r.ok).toBe(true);
+        expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/browser\/restart$/);
+    });
+
+    it('updateCrawlerSession POSTs the phpsessid body', async () => {
+        fetchSpy.mockResolvedValueOnce(ok({ valid: true, error: null }));
+        const r = await updateCrawlerSession('abc123');
+        expect(r.valid).toBe(true);
+        const init = fetchSpy.mock.calls[0][1] as RequestInit;
+        expect(init.method).toBe('POST');
+        expect(JSON.parse(init.body as string)).toEqual({ phpsessid: 'abc123' });
+    });
+
+    it('clearCrawlerSessionExpired POSTs clear-expired', async () => {
+        fetchSpy.mockResolvedValueOnce(ok({ cleared: true }));
+        const r = await clearCrawlerSessionExpired();
+        expect(r.cleared).toBe(true);
+        expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/session\/clear-expired$/);
+    });
+
+    it('listDeadJobs forwards search + pagination', async () => {
+        fetchSpy.mockResolvedValueOnce(
+            ok({ items: [], page: { limit: 20, offset: 20, total: 0 } })
+        );
+        await listDeadJobs({ search: 'naruto', limit: 20, offset: 20 });
+        const url = fetchSpy.mock.calls[0][0] as string;
+        expect(url).toContain('search=naruto');
+        expect(url).toContain('offset=20');
+    });
+
+    it('requeueDeadJobs POSTs the scope body', async () => {
+        fetchSpy.mockResolvedValueOnce(ok({ requeued: 3 }));
+        const r = await requeueDeadJobs({ scope: 'manga', manga_id: 'm-9' });
+        expect(r.requeued).toBe(3);
+        const init = fetchSpy.mock.calls[0][1] as RequestInit;
+        expect(JSON.parse(init.body as string)).toEqual({ scope: 'manga', manga_id: 'm-9' });
+    });
+
+    it('requeueDeadJobs serialises chapter scope verbatim', async () => {
+        // F7 — the per-chapter requeue button in /admin/mangas sends this
+        // exact shape; pin it so a future rename of `chapter_id` doesn't
+        // silently break the inline button.
+        fetchSpy.mockResolvedValueOnce(ok({ requeued: 1 }));
+        const r = await requeueDeadJobs({ scope: 'chapter', chapter_id: 'c-7' });
+        expect(r.requeued).toBe(1);
+        const init = fetchSpy.mock.calls[0][1] as RequestInit;
+        expect(JSON.parse(init.body as string)).toEqual({
+            scope: 'chapter',
+            chapter_id: 'c-7'
+        });
+    });
+
+    it('requeueDeadJobs serialises job + all (with confirm) scopes', async () => {
+        // Round out the variant coverage: { scope: 'job', job_id } and
+        // { scope: 'all', confirm: true }. The audit (S1) requires the
+        // confirm flag on the wire for scope=all; this pins it so a
+        // future refactor can't drop it.
+        fetchSpy.mockResolvedValueOnce(ok({ requeued: 1 }));
+        await requeueDeadJobs({ scope: 'job', job_id: 'j-1' });
+        expect(JSON.parse(fetchSpy.mock.calls[0][1]!.body as string)).toEqual({
+            scope: 'job',
+            job_id: 'j-1'
+        });
+        fetchSpy.mockResolvedValueOnce(ok({ requeued: 99 }));
+        await requeueDeadJobs({ scope: 'all', confirm: true });
+        expect(JSON.parse(fetchSpy.mock.calls[1][1]!.body as string)).toEqual({
+            scope: 'all',
+            confirm: true
+        });
+    });
+
+    it('surfaces a 503 as ApiError', async () => {
+        fetchSpy.mockResolvedValueOnce(envelope(503, 'service_unavailable', 'disabled'));
+        await expect(runCrawlerPass()).rejects.toMatchObject({ status: 503 });
+    });
+});