feat(admin): crawler observability dashboard + reliability hardening (0.55.0)

Admin-only crawler dashboard backed by an SSE live-status stream,
coordinated browser restart, runtime PHPSESSID refresh, dead-letter
requeue, and a batch of reliability fixes. Closes everything from
the two-pass audit (10 commits' worth) and bumps 0.52.0 -> 0.55.0.

Backend:

- New /admin/crawler/* surface (cookie-auth, RequireAdmin) split
  into status / control / dead_jobs / backlog modules. SSE stream
  composes in-memory status with DB-derived queue counts, memoizes
  the counts for 1s and debounces watch pokes for 250ms (~10x QPS
  reduction per subscriber). One-shot GET /admin/crawler shares the
  same compose path.
- POST /admin/crawler/run gated by manual_pass_lock try_lock_owned
  (409 Conflict on overlapping click); browser restart goes through
  the coordinated_restart gate (drain + relaunch + auto-clear of the
  sticky session_expired flag on Ok).
- Runtime PHPSESSID refresh via SessionController (allow-list
  validation, never logged, audit row carries SHA-256 fingerprint).
  Storage layer is repo::crawler::runtime_session_{load,persist}.
- Dead-letter requeue with four scopes (all/manga/chapter/job);
  scope=all requires confirm:true; DISTINCT ON dedup keeps the
  partial unique index from rejecting requeues for chapters with
  multiple dead rows. SQL is four &'static str constants per scope.
- StatusHandle + ChapterGuard / CoverGuard RAII model survives
  panics; last-writer-wins on cover so concurrent dispatches don't
  clobber each other's slot. Pure functions (should_stop /
  should_mark_clean_exit / should_abort_pass) with named regression
  tests.
- Reliability bundle: per-lease heartbeat, jitter on retries,
  per-job timeout, circuit breaker on consecutive failures, BrowserManager
  coordinated restart gate, request fingerprint changes.
- Streaming page download: Storage::put_stream trait method,
  LocalStorage impl atomic via temp + fsync + UUID-suffixed rename.
  Pages stream through with peak memory ~one HTTP chunk + 64-byte
  sniff prefix instead of one full image per dispatch.
- New partial indexes (migration 0022): mangas_missing_cover_idx
  and crawler_jobs_dead_idx, both ordered by updated_at DESC to
  match the dashboard's LIMIT/OFFSET reads.
- Security hardening: admin_csrf_guard (Origin/Referer allowlist
  on /admin/* mutations, opt-in via ADMIN_ALLOWED_ORIGINS),
  admin_no_store_guard (Cache-Control: no-store on admin
  responses), audit rows carry per-scope target_id.

Frontend:

- /admin/crawler page decomposed into lib/components/crawler/
  (11 components: ProgressBar, SearchBar, CrawlerHero,
  CrawlerControls, ActiveChaptersCard, ActiveJobsTable,
  MissingCoversTable, DeadJobsTable, RestartConfirmModal,
  RequeueAllConfirmModal, SessionModal). Page is 532 LOC of
  orchestration; each component 22-148 LOC.
- EventSource lifecycle wired to visibilitychange / pagehide /
  pageshow (BFCache); after 5 consecutive errors probes the status
  endpoint so a 401 routes through the global on401Hook instead of
  infinite silent reconnects.
- Backlog $effect refetches debounced 500ms with per-loader
  AbortControllers; refresh after a control action only runs when
  the SSE stream is dead.
- Inline requeue button on /admin/mangas patches the affected row's
  sync_state locally (no full chapter-list refetch); proper
  aria-label. Requeue-all gets its own confirm modal; both confirm
  modals autofocus Cancel.
- SvelteKit reverse proxy bypasses its 5-minute AbortController
  for Accept: text/event-stream; pure shouldBypassProxyTimeout
  helper covered by unit tests.

Config / docs:

- New env vars (.env.example): ADMIN_ALLOWED_ORIGINS,
  CRAWLER_JOB_TIMEOUT_SECS, CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES,
  CRAWLER_BROWSER_RESTART_THRESHOLD.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-06 18:49:56 +02:00
parent 679abae736
commit d6ac648ac9
54 changed files with 6076 additions and 137 deletions

View File

@@ -6,6 +6,7 @@
{ href: '/admin', label: 'Overview' },
{ href: '/admin/users', label: 'Users' },
{ href: '/admin/mangas', label: 'Mangas' },
{ href: '/admin/crawler', label: 'Crawler' },
{ href: '/admin/system', label: 'System' }
];
</script>

View File

@@ -0,0 +1,532 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
import CrawlerHero from '$lib/components/crawler/CrawlerHero.svelte';
import CrawlerControls from '$lib/components/crawler/CrawlerControls.svelte';
import ActiveChaptersCard from '$lib/components/crawler/ActiveChaptersCard.svelte';
import ActiveJobsTable from '$lib/components/crawler/ActiveJobsTable.svelte';
import MissingCoversTable from '$lib/components/crawler/MissingCoversTable.svelte';
import DeadJobsTable from '$lib/components/crawler/DeadJobsTable.svelte';
import SessionModal from '$lib/components/crawler/SessionModal.svelte';
import RestartConfirmModal from '$lib/components/crawler/RestartConfirmModal.svelte';
import RequeueAllConfirmModal from '$lib/components/crawler/RequeueAllConfirmModal.svelte';
import {
getCrawlerStatus,
crawlerStatusStreamUrl,
runCrawlerPass,
restartCrawlerBrowser,
updateCrawlerSession,
clearCrawlerSessionExpired,
listDeadJobs,
requeueDeadJobs,
listActiveJobs,
listMissingCovers,
type CrawlerStatus,
type DeadJob,
type ActiveJob,
type MissingCover,
type RequeueScope
} from '$lib/api/admin';
let status: CrawlerStatus | null = $state(null);
let error: string | null = $state(null);
let notice: string | null = $state(null);
let live = $state(false);
let source: EventSource | null = null;
let busy = $state(false);
// SSE error tracking (Q3): EventSource swallows status codes, so an
// auth-loss looks identical to a network blip and would reconnect
// forever. After this many consecutive errors we probe the status
// endpoint, which routes through `on401Hook` and surfaces the
// logout if appropriate.
let consecutiveSseErrors = 0;
const SSE_ERROR_PROBE_THRESHOLD = 5;
// Backlog refetch AbortControllers (Q4): a burst of SSE frames or
// search-key changes would otherwise fire concurrent fetches whose
// resolution order is last-wins-flicker. Each loader cancels its
// predecessor.
let activeAbort: AbortController | null = null;
let coversAbort: AbortController | null = null;
let deadAbort: AbortController | null = null;
// P4: debounce timers for backlog refetches. The SSE frame
// (chapter page tick, etc.) fires ~once/sec; refetching the queue
// on every frame produced a thundering herd against the same DB
// queries the SSE handler is already running.
let activeDebounce: ReturnType<typeof setTimeout> | null = null;
let coversDebounce: ReturnType<typeof setTimeout> | null = null;
const BACKLOG_DEBOUNCE_MS = 500;
// Dead jobs
let deadJobs: DeadJob[] = $state([]);
let deadTotal = $state(0);
let deadSearch = $state('');
let deadPage = $state(1);
const DEAD_LIMIT = 20;
// Queued chapters (pending/running)
let activeJobs: ActiveJob[] = $state([]);
let activeTotal = $state(0);
let activeSearch = $state('');
let activePage = $state(1);
const ACTIVE_LIMIT = 20;
// Queued covers (mangas missing a cover)
let covers: MissingCover[] = $state([]);
let coversTotal = $state(0);
let coversSearch = $state('');
let coversPage = $state(1);
const COVERS_LIMIT = 20;
// Modals
let sessionModalOpen = $state(false);
let restartModalOpen = $state(false);
let requeueAllModalOpen = $state(false);
let phpsessid = $state('');
let sessionResult: string | null = $state(null);
async function refresh() {
try {
status = await getCrawlerStatus();
error = null;
} catch (e) {
error = e instanceof Error ? e.message : 'refresh failed';
}
}
async function loadDeadJobs() {
deadAbort?.abort();
const ctrl = new AbortController();
deadAbort = ctrl;
try {
const resp = await listDeadJobs(
{
search: deadSearch.trim() || undefined,
limit: DEAD_LIMIT,
offset: (deadPage - 1) * DEAD_LIMIT
},
{ signal: ctrl.signal }
);
deadJobs = resp.items;
deadTotal = resp.page.total ?? resp.items.length;
} catch (e) {
if (ctrl.signal.aborted) return; // superseded by a newer call
error = e instanceof Error ? e.message : 'failed to load dead jobs';
}
}
async function loadActiveJobs() {
activeAbort?.abort();
const ctrl = new AbortController();
activeAbort = ctrl;
try {
const resp = await listActiveJobs(
{
search: activeSearch.trim() || undefined,
limit: ACTIVE_LIMIT,
offset: (activePage - 1) * ACTIVE_LIMIT
},
{ signal: ctrl.signal }
);
activeJobs = resp.items;
activeTotal = resp.page.total ?? resp.items.length;
} catch (e) {
if (ctrl.signal.aborted) return;
error = e instanceof Error ? e.message : 'failed to load queued chapters';
}
}
async function loadCovers() {
coversAbort?.abort();
const ctrl = new AbortController();
coversAbort = ctrl;
try {
const resp = await listMissingCovers(
{
search: coversSearch.trim() || undefined,
limit: COVERS_LIMIT,
offset: (coversPage - 1) * COVERS_LIMIT
},
{ signal: ctrl.signal }
);
covers = resp.items;
coversTotal = resp.page.total ?? resp.items.length;
} catch (e) {
if (ctrl.signal.aborted) return;
error = e instanceof Error ? e.message : 'failed to load queued covers';
}
}
// Auto-refresh the (fetched, not streamed) backlog lists when the live
// status shows the relevant counts moved — keeps the lists feeling
// live without pushing big payloads over SSE. `$effect` re-runs when
// these tracked values change; the timers debounce a burst of
// adjacent counts (typical during a chapter download that emits one
// poke per stored page) into a single refetch.
let lastQueueKey = $state('');
let lastCoversKey = $state(-1);
$effect(() => {
const k = `${status?.queue.pending ?? 0}:${status?.queue.running ?? 0}`;
if (k !== lastQueueKey) {
lastQueueKey = k;
if (activeDebounce) clearTimeout(activeDebounce);
activeDebounce = setTimeout(() => loadActiveJobs(), BACKLOG_DEBOUNCE_MS);
}
});
$effect(() => {
const c = status?.covers_queued ?? -1;
if (c !== lastCoversKey) {
lastCoversKey = c;
if (coversDebounce) clearTimeout(coversDebounce);
coversDebounce = setTimeout(() => loadCovers(), BACKLOG_DEBOUNCE_MS);
}
});
// Live updates via Server-Sent Events instead of polling. The
// EventSource is opened on mount and closed on destroy, so the
// subscription exists only while this page is showing live data.
function openStream() {
if (source) return;
const es = new EventSource(crawlerStatusStreamUrl(), { withCredentials: true });
es.addEventListener('status', (e) => {
try {
status = JSON.parse((e as MessageEvent).data) as CrawlerStatus;
error = null;
live = true;
consecutiveSseErrors = 0;
} catch {
// ignore a malformed frame; the next one will replace it
}
});
es.onopen = () => {
live = true;
consecutiveSseErrors = 0;
};
es.onerror = () => {
// The browser auto-reconnects; reflect the gap in the UI.
live = false;
consecutiveSseErrors += 1;
// Q3: EventSource can't read the HTTP status, so an auth
// loss (401) looks identical to a network blip. After
// several consecutive failures, probe the status endpoint
// via the normal API client so on401Hook can fire if the
// session is gone.
if (consecutiveSseErrors >= SSE_ERROR_PROBE_THRESHOLD) {
consecutiveSseErrors = 0; // reset so we don't probe-storm
getCrawlerStatus().catch(() => {
// The client's on401 hook (set globally) handles
// the logout; any other error we ignore — the
// EventSource will keep retrying and the UI flag
// already reflects `live = false`.
});
}
};
source = es;
}
function closeStream() {
source?.close();
source = null;
live = false;
}
// Q2: close the stream when the tab is hidden so the browser
// doesn't accumulate a stale connection (mobile Safari throttles SSE
// aggressively in background tabs), and reopen on visible. Same for
// pagehide / pageshow to handle the BFCache.
function onVisibilityChange() {
if (typeof document === 'undefined') return;
if (document.visibilityState === 'hidden') {
closeStream();
} else if (document.visibilityState === 'visible') {
openStream();
}
}
function onPageHide() {
closeStream();
}
function onPageShow() {
openStream();
}
onMount(() => {
// One-shot fetch for instant initial paint + resilience if SSE is
// blocked; the stream then drives subsequent updates.
refresh();
loadDeadJobs();
openStream();
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', onVisibilityChange);
window.addEventListener('pagehide', onPageHide);
window.addEventListener('pageshow', onPageShow);
}
});
onDestroy(() => {
closeStream();
if (typeof document !== 'undefined') {
document.removeEventListener('visibilitychange', onVisibilityChange);
window.removeEventListener('pagehide', onPageHide);
window.removeEventListener('pageshow', onPageShow);
}
if (activeDebounce) clearTimeout(activeDebounce);
if (coversDebounce) clearTimeout(coversDebounce);
activeAbort?.abort();
coversAbort?.abort();
deadAbort?.abort();
});
async function withBusy(label: string, fn: () => Promise<void>) {
busy = true;
notice = null;
error = null;
try {
await fn();
} catch (e) {
error = e instanceof Error ? e.message : `${label} failed`;
} finally {
busy = false;
// Q5: only fall back to a fetch refresh when the live SSE
// stream isn't pushing updates. With `live === true` the
// backend's `status.poke()` after each action already
// pushes a fresh status frame; the extra fetch would race
// it and produce flicker.
if (!live) {
await refresh();
}
}
}
async function onRunPass() {
await withBusy('run pass', async () => {
await runCrawlerPass();
notice = 'Metadata pass started.';
});
}
async function onConfirmRestart() {
restartModalOpen = false;
await withBusy('restart browser', async () => {
const r = await restartCrawlerBrowser();
notice = r.ok ? 'Browser restarted.' : `Restart failed: ${r.error ?? 'unknown'}`;
});
}
async function onSaveSession() {
sessionResult = null;
busy = true;
try {
const r = await updateCrawlerSession(phpsessid);
sessionResult = r.valid
? '✓ Session valid — workers resumed.'
: `✕ Probe failed: ${r.error ?? 'unauthenticated'}`;
if (r.valid) {
sessionModalOpen = false;
phpsessid = '';
notice = 'Session updated.';
}
} catch (e) {
sessionResult = e instanceof Error ? e.message : 'update failed';
} finally {
busy = false;
if (!live) await refresh();
}
}
async function onClearExpired() {
await withBusy('clear expired', async () => {
await clearCrawlerSessionExpired();
notice = 'Session-expired flag cleared.';
});
}
async function requeue(scope: RequeueScope) {
await withBusy('requeue', async () => {
const r = await requeueDeadJobs(scope);
notice = `Requeued ${r.requeued} job(s).`;
await loadDeadJobs();
});
}
async function onConfirmRequeueAll() {
requeueAllModalOpen = false;
await requeue({ scope: 'all', confirm: true });
}
function onSearchDead() {
deadPage = 1;
loadDeadJobs();
}
function onDeadPageChange(p: number) {
deadPage = p;
loadDeadJobs();
}
function onSearchActive() {
activePage = 1;
loadActiveJobs();
}
function onActivePageChange(p: number) {
activePage = p;
loadActiveJobs();
}
function onSearchCovers() {
coversPage = 1;
loadCovers();
}
function onCoversPageChange(p: number) {
coversPage = p;
loadCovers();
}
const deadTotalPages = $derived(Math.max(1, Math.ceil(deadTotal / DEAD_LIMIT)));
const activeTotalPages = $derived(Math.max(1, Math.ceil(activeTotal / ACTIVE_LIMIT)));
const coversTotalPages = $derived(Math.max(1, Math.ceil(coversTotal / COVERS_LIMIT)));
</script>
<div class="titlebar">
<h1>Crawler</h1>
<span class="livedot" class:on={live} title={live ? 'Live (SSE)' : 'Reconnecting…'}>
{live ? '● live' : '○ reconnecting…'}
</span>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
{#if notice}
<p class="notice" role="status">{notice}</p>
{/if}
{#if status}
<CrawlerHero {status} />
<CrawlerControls
{status}
{busy}
{onRunPass}
onOpenRestart={() => (restartModalOpen = true)}
onOpenSession={() => {
sessionModalOpen = true;
sessionResult = null;
}}
{onClearExpired}
/>
<ActiveChaptersCard {status} />
{:else}
<p>Loading…</p>
{/if}
<ActiveJobsTable
jobs={activeJobs}
total={activeTotal}
page={activePage}
totalPages={activeTotalPages}
bind:search={activeSearch}
onSearch={onSearchActive}
onPageChange={onActivePageChange}
/>
<MissingCoversTable
{covers}
total={coversTotal}
page={coversPage}
totalPages={coversTotalPages}
bind:search={coversSearch}
onSearch={onSearchCovers}
onPageChange={onCoversPageChange}
/>
<DeadJobsTable
jobs={deadJobs}
total={deadTotal}
page={deadPage}
totalPages={deadTotalPages}
bind:search={deadSearch}
{busy}
onSearch={onSearchDead}
onPageChange={onDeadPageChange}
onRequeue={requeue}
onRequeueAll={() => (requeueAllModalOpen = true)}
/>
<RestartConfirmModal
open={restartModalOpen}
{busy}
onCancel={() => (restartModalOpen = false)}
onConfirm={onConfirmRestart}
/>
<RequeueAllConfirmModal
open={requeueAllModalOpen}
total={deadTotal}
{busy}
onCancel={() => (requeueAllModalOpen = false)}
onConfirm={onConfirmRequeueAll}
/>
<SessionModal
open={sessionModalOpen}
{busy}
bind:phpsessid
result={sessionResult}
onCancel={() => (sessionModalOpen = false)}
onSave={onSaveSession}
/>
<style>
h1 {
margin: 0;
}
.titlebar {
display: flex;
align-items: baseline;
gap: var(--space-3);
margin-bottom: var(--space-4);
}
.livedot {
font-size: var(--font-sm);
color: var(--text-muted);
}
.livedot.on {
color: var(--success, #0a7d2c);
}
.notice {
color: var(--success, #0a7d2c);
padding: var(--space-2) var(--space-3);
border: 1px solid var(--success, #0a7d2c);
border-radius: var(--radius-md);
margin-bottom: var(--space-3);
}
.error {
color: var(--danger, #dc2626);
padding: var(--space-2) var(--space-3);
border: 1px solid var(--danger, #dc2626);
border-radius: var(--radius-md);
margin-bottom: var(--space-3);
}
/* badges (shared convention with admin/mangas) */
:global(.badge) {
display: inline-block;
padding: 0 var(--space-2);
border-radius: var(--radius-sm, 4px);
font-size: var(--font-xs);
font-weight: var(--weight-semibold);
text-transform: uppercase;
letter-spacing: 0.04em;
border: 1px solid var(--border);
background: var(--surface);
}
:global(.badge-synced) {
background: #dcfce7;
color: #166534;
border-color: #86efac;
}
:global(.badge-in_progress),
:global(.badge-downloading) {
background: #fef3c7;
color: #92400e;
border-color: #fcd34d;
}
:global(.badge-not_downloaded) {
background: var(--surface-elevated);
color: var(--text-muted);
}
</style>

View File

@@ -3,6 +3,7 @@
import {
listAdminMangas,
listAdminChapters,
requeueDeadJobs,
type AdminMangasPage,
type AdminChapterRow,
type MangaSyncState
@@ -59,6 +60,39 @@
function badgeClass(state: string): string {
return `badge badge-${state}`;
}
let requeuingChapter: string | null = $state(null);
/** Requeue the dead job(s) for a single failed chapter, then patch
* the local state instead of refetching the whole list. Refetching
* 500 chapters every time an operator clicks "Requeue" was wasteful;
* the only field that changes is the per-chapter sync state of the
* affected row. F4 in the audit. */
async function requeueChapter(mangaId: string, chapterId: string) {
requeuingChapter = chapterId;
error = null;
try {
await requeueDeadJobs({ scope: 'chapter', chapter_id: chapterId });
const existing = chaptersByManga[mangaId];
if (existing && existing !== 'loading') {
// After requeue the dead job is back to `pending`; the
// chapter itself stays "not_downloaded" until a worker
// picks it up and stores pages.
chaptersByManga[mangaId] = {
items: existing.items.map((c) =>
c.id === chapterId
? { ...c, sync_state: 'not_downloaded' as const }
: c
),
total: existing.total
};
}
} catch (e) {
error = e instanceof ApiError ? e.message : 'requeue failed';
} finally {
requeuingChapter = null;
}
}
</script>
<h1>Mangas</h1>
@@ -153,6 +187,17 @@
<span class={badgeClass(c.sync_state)}>
{c.sync_state}
</span>
{#if c.sync_state === 'failed'}
<button
class="requeue"
onclick={() => requeueChapter(m.id, c.id)}
disabled={requeuingChapter === c.id}
title="Requeue this chapter"
aria-label={`Requeue chapter ${c.number}`}
>
↻ requeue
</button>
{/if}
</td>
</tr>
{/each}
@@ -272,6 +317,11 @@
color: #991b1b;
border-color: #fca5a5;
}
.requeue {
margin-left: var(--space-2);
font-size: var(--font-xs);
padding: 0 var(--space-2);
}
.badge-not_downloaded {
background: var(--surface-elevated);
color: var(--text-muted);