Admin-only crawler dashboard backed by an SSE live-status stream,
coordinated browser restart, runtime PHPSESSID refresh, dead-letter
requeue, and a batch of reliability fixes. Closes everything from
the two-pass audit (10 commits' worth) and bumps 0.52.0 -> 0.55.0.
Backend:
- New /admin/crawler/* surface (cookie-auth, RequireAdmin) split
into status / control / dead_jobs / backlog modules. SSE stream
composes in-memory status with DB-derived queue counts, memoizes
the counts for 1s and debounces watch pokes for 250ms (~10x QPS
reduction per subscriber). One-shot GET /admin/crawler shares the
same compose path.
- POST /admin/crawler/run gated by manual_pass_lock try_lock_owned
(409 Conflict on overlapping click); browser restart goes through
the coordinated_restart gate (drain + relaunch + auto-clear of the
sticky session_expired flag on Ok).
- Runtime PHPSESSID refresh via SessionController (allow-list
validation, never logged, audit row carries SHA-256 fingerprint).
Storage layer is repo::crawler::runtime_session_{load,persist}.
- Dead-letter requeue with four scopes (all/manga/chapter/job);
scope=all requires confirm:true; DISTINCT ON dedup keeps the
partial unique index from rejecting requeues for chapters with
multiple dead rows. SQL is four &'static str constants per scope.
- StatusHandle + ChapterGuard / CoverGuard RAII model survives
panics; last-writer-wins on cover so concurrent dispatches don't
clobber each other's slot. Pure functions (should_stop /
should_mark_clean_exit / should_abort_pass) with named regression
tests.
- Reliability bundle: per-lease heartbeat, jitter on retries,
per-job timeout, circuit breaker on consecutive failures, BrowserManager
coordinated restart gate, request fingerprint changes.
- Streaming page download: Storage::put_stream trait method,
LocalStorage impl atomic via temp + fsync + UUID-suffixed rename.
Pages stream through with peak memory ~one HTTP chunk + 64-byte
sniff prefix instead of one full image per dispatch.
- New partial indexes (migration 0022): mangas_missing_cover_idx
and crawler_jobs_dead_idx, both ordered by updated_at DESC to
match the dashboard's LIMIT/OFFSET reads.
- Security hardening: admin_csrf_guard (Origin/Referer allowlist
on /admin/* mutations, opt-in via ADMIN_ALLOWED_ORIGINS),
admin_no_store_guard (Cache-Control: no-store on admin
responses), audit rows carry per-scope target_id.
Frontend:
- /admin/crawler page decomposed into lib/components/crawler/
(11 components: ProgressBar, SearchBar, CrawlerHero,
CrawlerControls, ActiveChaptersCard, ActiveJobsTable,
MissingCoversTable, DeadJobsTable, RestartConfirmModal,
RequeueAllConfirmModal, SessionModal). Page is 532 LOC of
orchestration; each component 22-148 LOC.
- EventSource lifecycle wired to visibilitychange / pagehide /
pageshow (BFCache); after 5 consecutive errors probes the status
endpoint so a 401 routes through the global on401Hook instead of
infinite silent reconnects.
- Backlog $effect refetches debounced 500ms with per-loader
AbortControllers; refresh after a control action only runs when
the SSE stream is dead.
- Inline requeue button on /admin/mangas patches the affected row's
sync_state locally (no full chapter-list refetch); proper
aria-label. Requeue-all gets its own confirm modal; both confirm
modals autofocus Cancel.
- SvelteKit reverse proxy bypasses its 5-minute AbortController
for Accept: text/event-stream; pure shouldBypassProxyTimeout
helper covered by unit tests.
Config / docs:
- New env vars (.env.example): ADMIN_ALLOWED_ORIGINS,
CRAWLER_JOB_TIMEOUT_SECS, CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES,
CRAWLER_BROWSER_RESTART_THRESHOLD.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
345 lines
12 KiB
Svelte
345 lines
12 KiB
Svelte
<script lang="ts">
|
|
import { onMount } from 'svelte';
|
|
import {
|
|
listAdminMangas,
|
|
listAdminChapters,
|
|
requeueDeadJobs,
|
|
type AdminMangasPage,
|
|
type AdminChapterRow,
|
|
type MangaSyncState
|
|
} from '$lib/api/admin';
|
|
import { ApiError } from '$lib/api/client';
|
|
|
|
let mangasPage: AdminMangasPage | null = $state(null);
|
|
let search = $state('');
|
|
let syncFilter: MangaSyncState | '' = $state('');
|
|
let error: string | null = $state(null);
|
|
let expandedId: string | null = $state(null);
|
|
type ChaptersView = {
|
|
items: AdminChapterRow[];
|
|
total: number;
|
|
};
|
|
let chaptersByManga: Record<string, ChaptersView | 'loading'> = $state({});
|
|
|
|
async function load() {
|
|
error = null;
|
|
try {
|
|
mangasPage = await listAdminMangas({
|
|
search: search.trim() || undefined,
|
|
syncState: syncFilter || undefined,
|
|
limit: 100
|
|
});
|
|
} catch (e) {
|
|
error = e instanceof ApiError ? e.message : 'load failed';
|
|
}
|
|
}
|
|
|
|
onMount(load);
|
|
|
|
async function toggleChapters(id: string) {
|
|
if (expandedId === id) {
|
|
expandedId = null;
|
|
return;
|
|
}
|
|
expandedId = id;
|
|
if (!chaptersByManga[id]) {
|
|
chaptersByManga[id] = 'loading';
|
|
try {
|
|
const resp = await listAdminChapters(id, { limit: 500 });
|
|
chaptersByManga[id] = {
|
|
items: resp.items,
|
|
total: resp.page.total ?? resp.items.length
|
|
};
|
|
} catch {
|
|
delete chaptersByManga[id];
|
|
error = 'failed to load chapters';
|
|
}
|
|
}
|
|
}
|
|
|
|
function badgeClass(state: string): string {
|
|
return `badge badge-${state}`;
|
|
}
|
|
|
|
let requeuingChapter: string | null = $state(null);
|
|
|
|
/** Requeue the dead job(s) for a single failed chapter, then patch
|
|
* the local state instead of refetching the whole list. Refetching
|
|
* 500 chapters every time an operator clicks "Requeue" was wasteful;
|
|
* the only field that changes is the per-chapter sync state of the
|
|
* affected row. F4 in the audit. */
|
|
async function requeueChapter(mangaId: string, chapterId: string) {
|
|
requeuingChapter = chapterId;
|
|
error = null;
|
|
try {
|
|
await requeueDeadJobs({ scope: 'chapter', chapter_id: chapterId });
|
|
const existing = chaptersByManga[mangaId];
|
|
if (existing && existing !== 'loading') {
|
|
// After requeue the dead job is back to `pending`; the
|
|
// chapter itself stays "not_downloaded" until a worker
|
|
// picks it up and stores pages.
|
|
chaptersByManga[mangaId] = {
|
|
items: existing.items.map((c) =>
|
|
c.id === chapterId
|
|
? { ...c, sync_state: 'not_downloaded' as const }
|
|
: c
|
|
),
|
|
total: existing.total
|
|
};
|
|
}
|
|
} catch (e) {
|
|
error = e instanceof ApiError ? e.message : 'requeue failed';
|
|
} finally {
|
|
requeuingChapter = null;
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<h1>Mangas</h1>
|
|
|
|
<form
|
|
onsubmit={(e) => {
|
|
e.preventDefault();
|
|
load();
|
|
}}
|
|
>
|
|
<input
|
|
type="search"
|
|
placeholder="Search by title"
|
|
bind:value={search}
|
|
data-testid="admin-mangas-search"
|
|
/>
|
|
<label class="sync-label">
|
|
<span>Sync state</span>
|
|
<select bind:value={syncFilter} aria-label="sync state">
|
|
<option value="">All</option>
|
|
<option value="in_progress">In progress</option>
|
|
<option value="dropped">Dropped</option>
|
|
<option value="synced">Synced</option>
|
|
</select>
|
|
</label>
|
|
<button type="submit">Search</button>
|
|
</form>
|
|
|
|
{#if error}
|
|
<p class="error" role="alert">{error}</p>
|
|
{/if}
|
|
|
|
{#if mangasPage}
|
|
<p class="total">{mangasPage.page.total ?? mangasPage.items.length} mangas</p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Title</th>
|
|
<th>Sync</th>
|
|
<th>Chapters</th>
|
|
<th>Last seen</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{#each mangasPage.items as m (m.id)}
|
|
<tr>
|
|
<td>
|
|
<button class="link" onclick={() => toggleChapters(m.id)}>
|
|
{expandedId === m.id ? '▼' : '▶'} {m.title}
|
|
</button>
|
|
</td>
|
|
<td><span class={badgeClass(m.sync_state)}>{m.sync_state}</span></td>
|
|
<td>{m.chapter_count}</td>
|
|
<td>
|
|
{m.latest_seen_at
|
|
? new Date(m.latest_seen_at).toLocaleDateString()
|
|
: '—'}
|
|
</td>
|
|
</tr>
|
|
{#if expandedId === m.id}
|
|
<tr class="chapter-row">
|
|
<td colspan="4">
|
|
{#if chaptersByManga[m.id] === 'loading'}
|
|
<p>Loading chapters…</p>
|
|
{:else if chaptersByManga[m.id]}
|
|
{@const view = chaptersByManga[m.id] as ChaptersView}
|
|
{#if view.items.length === 0}
|
|
<p class="muted">No chapters.</p>
|
|
{:else}
|
|
{#if view.total > view.items.length}
|
|
<p class="muted">
|
|
Showing first {view.items.length} of {view.total}
|
|
chapters (cap reached).
|
|
</p>
|
|
{/if}
|
|
<table class="inner">
|
|
<thead>
|
|
<tr>
|
|
<th>#</th>
|
|
<th>Title</th>
|
|
<th>Pages</th>
|
|
<th>Sync</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{#each view.items as c (c.id)}
|
|
<tr>
|
|
<td>{c.number}</td>
|
|
<td>{c.title ?? '—'}</td>
|
|
<td>{c.page_count}</td>
|
|
<td>
|
|
<span class={badgeClass(c.sync_state)}>
|
|
{c.sync_state}
|
|
</span>
|
|
{#if c.sync_state === 'failed'}
|
|
<button
|
|
class="requeue"
|
|
onclick={() => requeueChapter(m.id, c.id)}
|
|
disabled={requeuingChapter === c.id}
|
|
title="Requeue this chapter"
|
|
aria-label={`Requeue chapter ${c.number}`}
|
|
>
|
|
↻ requeue
|
|
</button>
|
|
{/if}
|
|
</td>
|
|
</tr>
|
|
{/each}
|
|
</tbody>
|
|
</table>
|
|
{/if}
|
|
{/if}
|
|
</td>
|
|
</tr>
|
|
{/if}
|
|
{/each}
|
|
</tbody>
|
|
</table>
|
|
{:else}
|
|
<p>Loading…</p>
|
|
{/if}
|
|
|
|
<style>
|
|
h1 {
|
|
margin: 0 0 var(--space-4) 0;
|
|
}
|
|
form {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
margin-bottom: var(--space-3);
|
|
}
|
|
input[type='search'] {
|
|
flex: 1;
|
|
min-width: 0;
|
|
max-width: 24rem;
|
|
padding: var(--space-2) var(--space-3);
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
background: var(--surface);
|
|
color: var(--text);
|
|
}
|
|
.sync-label {
|
|
display: inline-flex;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
color: var(--text-muted);
|
|
font-size: var(--font-sm);
|
|
}
|
|
select {
|
|
padding: var(--space-2) var(--space-3);
|
|
border-radius: var(--radius-md);
|
|
border: 1px solid var(--border);
|
|
background: var(--surface);
|
|
color: var(--text);
|
|
}
|
|
button {
|
|
padding: var(--space-2) var(--space-3);
|
|
border-radius: var(--radius-md);
|
|
border: 1px solid var(--border);
|
|
background: var(--surface);
|
|
color: var(--text);
|
|
cursor: pointer;
|
|
}
|
|
button.link {
|
|
background: none;
|
|
border: none;
|
|
padding: 0;
|
|
color: var(--text);
|
|
cursor: pointer;
|
|
font-weight: inherit;
|
|
}
|
|
button.link:hover {
|
|
color: var(--primary);
|
|
}
|
|
.total {
|
|
color: var(--text-muted);
|
|
font-size: var(--font-sm);
|
|
margin: 0 0 var(--space-2) 0;
|
|
}
|
|
table {
|
|
width: 100%;
|
|
border-collapse: collapse;
|
|
}
|
|
th,
|
|
td {
|
|
padding: var(--space-2);
|
|
text-align: left;
|
|
border-bottom: 1px solid var(--border);
|
|
}
|
|
.chapter-row td {
|
|
background: var(--surface-elevated);
|
|
}
|
|
table.inner {
|
|
margin: var(--space-2) 0;
|
|
}
|
|
.badge {
|
|
display: inline-block;
|
|
padding: 0 var(--space-2);
|
|
border-radius: var(--radius-sm, 4px);
|
|
font-size: var(--font-xs, 0.75rem);
|
|
font-weight: var(--weight-semibold);
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.04em;
|
|
border: 1px solid var(--border);
|
|
background: var(--surface);
|
|
}
|
|
.badge-in_progress,
|
|
.badge-downloading {
|
|
background: #fef3c7;
|
|
color: #92400e;
|
|
border-color: #fcd34d;
|
|
}
|
|
.badge-dropped {
|
|
background: #fee2e2;
|
|
color: #991b1b;
|
|
border-color: #fca5a5;
|
|
}
|
|
.badge-failed {
|
|
background: #fee2e2;
|
|
color: #991b1b;
|
|
border-color: #fca5a5;
|
|
}
|
|
.requeue {
|
|
margin-left: var(--space-2);
|
|
font-size: var(--font-xs);
|
|
padding: 0 var(--space-2);
|
|
}
|
|
.badge-not_downloaded {
|
|
background: var(--surface-elevated);
|
|
color: var(--text-muted);
|
|
}
|
|
.badge-synced {
|
|
background: #dcfce7;
|
|
color: #166534;
|
|
border-color: #86efac;
|
|
}
|
|
.muted {
|
|
color: var(--text-muted);
|
|
}
|
|
.error {
|
|
color: var(--danger, #dc2626);
|
|
padding: var(--space-2) var(--space-3);
|
|
border: 1px solid var(--danger, #dc2626);
|
|
border-radius: var(--radius-md);
|
|
margin-bottom: var(--space-3);
|
|
}
|
|
</style>
|