feat(crawler): live status via SSE instead of polling
Replace the dashboard's 5s polling with a Server-Sent Events stream: - StatusHandle gains a tokio `watch` version bumped on every mutation; GET /admin/crawler/stream subscribes and pushes a composed snapshot immediately on connect, then on every status change (instant, no lost-wakeup) plus a 5s backstop for DB queue counts / browser phase. - Non-status signals poke the notifier so they push immediately too: session-expired (worker), session update / clear-expired / browser restart (endpoints). - compose_status is shared by the one-shot GET and the stream; the stream tolerates transient DB errors with a keep-alive comment instead of tearing down. Frontend: the crawler page opens an EventSource on mount and closes it on destroy, so the subscription is scoped to the active page (no global subscription). A one-shot fetch still paints initial state / serves as a fallback if SSE is blocked; a live/reconnecting indicator reflects the connection. The existing reverse proxy already streams SSE (its abort timer is cleared once response headers arrive), so no proxy change needed. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ import {
|
||||
resyncManga,
|
||||
resyncChapter,
|
||||
getCrawlerStatus,
|
||||
crawlerStatusStreamUrl,
|
||||
runCrawlerPass,
|
||||
restartCrawlerBrowser,
|
||||
updateCrawlerSession,
|
||||
@@ -356,6 +357,10 @@ describe('admin crawler api client', () => {
|
||||
queue: { pending: 2, running: 1, dead: 4 }
|
||||
};
|
||||
|
||||
it('crawlerStatusStreamUrl points at the SSE endpoint under the API base', () => {
|
||||
expect(crawlerStatusStreamUrl()).toMatch(/\/v1\/admin\/crawler\/stream$/);
|
||||
});
|
||||
|
||||
it('getCrawlerStatus GETs /v1/admin/crawler', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(statusFixture));
|
||||
const s = await getCrawlerStatus();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// won't reach these routes). 403s thrown here propagate up to the
|
||||
// /admin layout, which renders the framework error page.
|
||||
|
||||
import { request, type Page } from './client';
|
||||
import { request, apiUrl, type Page } from './client';
|
||||
import type { User } from './auth';
|
||||
import type { MangaDetail } from './mangas';
|
||||
import type { Chapter } from './chapters';
|
||||
@@ -248,6 +248,14 @@ export async function getCrawlerStatus(): Promise<CrawlerStatus> {
|
||||
return request<CrawlerStatus>('/v1/admin/crawler');
|
||||
}
|
||||
|
||||
/** URL of the Server-Sent Events live-status stream. Open with
|
||||
* `new EventSource(...)` while the crawler page is mounted and close it on
|
||||
* navigate-away so the subscription is scoped to the active page. Each
|
||||
* message is a named `status` event whose `data` is a {@link CrawlerStatus}. */
|
||||
export function crawlerStatusStreamUrl(): string {
|
||||
return apiUrl('/v1/admin/crawler/stream');
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */
|
||||
export async function runCrawlerPass(): Promise<{ started: boolean }> {
|
||||
return request('/v1/admin/crawler/run', { method: 'POST' });
|
||||
|
||||
@@ -12,6 +12,15 @@ export function fileUrl(key: string): string {
|
||||
return `${BASE}/v1/files/${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an API URL for non-`fetch` consumers (e.g. `EventSource` for SSE),
|
||||
* applying the same `VITE_API_BASE` prefix as `request()`. `path` is the
|
||||
* route after the base, e.g. `/v1/admin/crawler/stream`.
|
||||
*/
|
||||
export function apiUrl(path: string): string {
|
||||
return `${BASE}${path}`;
|
||||
}
|
||||
|
||||
export class ApiError extends Error {
|
||||
constructor(
|
||||
public readonly status: number,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import {
|
||||
getCrawlerStatus,
|
||||
crawlerStatusStreamUrl,
|
||||
runCrawlerPass,
|
||||
restartCrawlerBrowser,
|
||||
updateCrawlerSession,
|
||||
@@ -19,7 +20,8 @@
|
||||
let status: CrawlerStatus | null = $state(null);
|
||||
let error: string | null = $state(null);
|
||||
let notice: string | null = $state(null);
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let live = $state(false);
|
||||
let source: EventSource | null = null;
|
||||
let busy = $state(false);
|
||||
|
||||
// Dead jobs
|
||||
@@ -58,13 +60,40 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Live updates via Server-Sent Events instead of polling. The
|
||||
// EventSource is opened on mount and closed on destroy, so the
|
||||
// subscription exists only while this page is showing live data.
|
||||
function openStream() {
|
||||
const es = new EventSource(crawlerStatusStreamUrl(), { withCredentials: true });
|
||||
es.addEventListener('status', (e) => {
|
||||
try {
|
||||
status = JSON.parse((e as MessageEvent).data) as CrawlerStatus;
|
||||
error = null;
|
||||
live = true;
|
||||
} catch {
|
||||
// ignore a malformed frame; the next one will replace it
|
||||
}
|
||||
});
|
||||
es.onopen = () => {
|
||||
live = true;
|
||||
};
|
||||
es.onerror = () => {
|
||||
// The browser auto-reconnects; reflect the gap in the UI.
|
||||
live = false;
|
||||
};
|
||||
source = es;
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
// One-shot fetch for instant initial paint + resilience if SSE is
|
||||
// blocked; the stream then drives subsequent updates.
|
||||
refresh();
|
||||
loadDeadJobs();
|
||||
timer = setInterval(refresh, 5000);
|
||||
openStream();
|
||||
});
|
||||
onDestroy(() => {
|
||||
if (timer) clearInterval(timer);
|
||||
source?.close();
|
||||
source = null;
|
||||
});
|
||||
|
||||
async function withBusy(label: string, fn: () => Promise<void>) {
|
||||
@@ -188,7 +217,12 @@
|
||||
const deadTotalPages = $derived(Math.max(1, Math.ceil(deadTotal / DEAD_LIMIT)));
|
||||
</script>
|
||||
|
||||
<h1>Crawler</h1>
|
||||
<div class="titlebar">
|
||||
<h1>Crawler</h1>
|
||||
<span class="livedot" class:on={live} title={live ? 'Live (SSE)' : 'Reconnecting…'}>
|
||||
{live ? '● live' : '○ reconnecting…'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<p class="error" role="alert">{error}</p>
|
||||
@@ -403,7 +437,20 @@
|
||||
|
||||
<style>
|
||||
h1 {
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
margin: 0;
|
||||
}
|
||||
.titlebar {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: var(--space-3);
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
.livedot {
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.livedot.on {
|
||||
color: var(--success, #0a7d2c);
|
||||
}
|
||||
h2 {
|
||||
margin: 0 0 var(--space-3) 0;
|
||||
|
||||
Reference in New Issue
Block a user