feat(crawler): live status via SSE instead of polling

Replace the dashboard's 5s polling with a Server-Sent Events stream:

- StatusHandle gains a tokio `watch` version bumped on every mutation;
  GET /admin/crawler/stream subscribes and pushes a composed snapshot
  immediately on connect, then on every status change (instant, no
  lost-wakeup) plus a 5s backstop for DB queue counts / browser phase.
- Non-status signals poke the notifier so they push immediately too:
  session-expired (worker), session update / clear-expired / browser
  restart (endpoints).
- compose_status is shared by the one-shot GET and the stream; the stream
  tolerates transient DB errors with a keep-alive comment instead of
  tearing down.

Frontend: the crawler page opens an EventSource on mount and closes it on
destroy, so the subscription is scoped to the active page (no global
subscription). A one-shot fetch still paints initial state / serves as a
fallback if SSE is blocked; a live/reconnecting indicator reflects the
connection. The existing reverse proxy already streams SSE (its abort
timer is cleared once response headers arrive), so no proxy change needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-04 19:13:50 +02:00
parent 832042d2b7
commit da6e320836
8 changed files with 291 additions and 31 deletions

View File

@@ -18,6 +18,7 @@ import {
resyncManga,
resyncChapter,
getCrawlerStatus,
crawlerStatusStreamUrl,
runCrawlerPass,
restartCrawlerBrowser,
updateCrawlerSession,
@@ -356,6 +357,10 @@ describe('admin crawler api client', () => {
queue: { pending: 2, running: 1, dead: 4 }
};
it('crawlerStatusStreamUrl points at the SSE endpoint under the API base', () => {
expect(crawlerStatusStreamUrl()).toMatch(/\/v1\/admin\/crawler\/stream$/);
});
it('getCrawlerStatus GETs /v1/admin/crawler', async () => {
fetchSpy.mockResolvedValueOnce(ok(statusFixture));
const s = await getCrawlerStatus();

View File

@@ -3,7 +3,7 @@
// won't reach these routes). 403s thrown here propagate up to the
// /admin layout, which renders the framework error page.
import { request, type Page } from './client';
import { request, apiUrl, type Page } from './client';
import type { User } from './auth';
import type { MangaDetail } from './mangas';
import type { Chapter } from './chapters';
@@ -248,6 +248,14 @@ export async function getCrawlerStatus(): Promise<CrawlerStatus> {
return request<CrawlerStatus>('/v1/admin/crawler');
}
/** URL of the Server-Sent Events live-status stream. Open with
* `new EventSource(...)` while the crawler page is mounted and close it on
* navigate-away so the subscription is scoped to the active page. Each
* message is a named `status` event whose `data` is a {@link CrawlerStatus}. */
export function crawlerStatusStreamUrl(): string {
return apiUrl('/v1/admin/crawler/stream');
}
/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */
export async function runCrawlerPass(): Promise<{ started: boolean }> {
return request('/v1/admin/crawler/run', { method: 'POST' });

View File

@@ -12,6 +12,15 @@ export function fileUrl(key: string): string {
return `${BASE}/v1/files/${key}`;
}
/**
* Builds an API URL for non-`fetch` consumers (e.g. `EventSource` for SSE),
* applying the same `VITE_API_BASE` prefix as `request()`. `path` is the
* route after the base, e.g. `/v1/admin/crawler/stream`.
*/
export function apiUrl(path: string): string {
return `${BASE}${path}`;
}
export class ApiError extends Error {
constructor(
public readonly status: number,

View File

@@ -4,6 +4,7 @@
import Pager from '$lib/components/Pager.svelte';
import {
getCrawlerStatus,
crawlerStatusStreamUrl,
runCrawlerPass,
restartCrawlerBrowser,
updateCrawlerSession,
@@ -19,7 +20,8 @@
let status: CrawlerStatus | null = $state(null);
let error: string | null = $state(null);
let notice: string | null = $state(null);
let timer: ReturnType<typeof setInterval> | null = null;
let live = $state(false);
let source: EventSource | null = null;
let busy = $state(false);
// Dead jobs
@@ -58,13 +60,40 @@
}
}
// Live updates via Server-Sent Events instead of polling. The
// EventSource is opened on mount and closed on destroy, so the
// subscription exists only while this page is showing live data.
function openStream() {
const es = new EventSource(crawlerStatusStreamUrl(), { withCredentials: true });
es.addEventListener('status', (e) => {
try {
status = JSON.parse((e as MessageEvent).data) as CrawlerStatus;
error = null;
live = true;
} catch {
// ignore a malformed frame; the next one will replace it
}
});
es.onopen = () => {
live = true;
};
es.onerror = () => {
// The browser auto-reconnects; reflect the gap in the UI.
live = false;
};
source = es;
}
onMount(() => {
// One-shot fetch for instant initial paint + resilience if SSE is
// blocked; the stream then drives subsequent updates.
refresh();
loadDeadJobs();
timer = setInterval(refresh, 5000);
openStream();
});
onDestroy(() => {
if (timer) clearInterval(timer);
source?.close();
source = null;
});
async function withBusy(label: string, fn: () => Promise<void>) {
@@ -188,7 +217,12 @@
const deadTotalPages = $derived(Math.max(1, Math.ceil(deadTotal / DEAD_LIMIT)));
</script>
<h1>Crawler</h1>
<div class="titlebar">
<h1>Crawler</h1>
<span class="livedot" class:on={live} title={live ? 'Live (SSE)' : 'Reconnecting…'}>
{live ? '● live' : '○ reconnecting…'}
</span>
</div>
{#if error}
<p class="error" role="alert">{error}</p>
@@ -403,7 +437,20 @@
<style>
h1 {
margin: 0 0 var(--space-4) 0;
margin: 0;
}
.titlebar {
display: flex;
align-items: baseline;
gap: var(--space-3);
margin-bottom: var(--space-4);
}
.livedot {
font-size: var(--font-sm);
color: var(--text-muted);
}
.livedot.on {
color: var(--success, #0a7d2c);
}
h2 {
margin: 0 0 var(--space-3) 0;