Mangalord/frontend/src/routes/admin/crawler/+page.svelte

<script lang="ts">
    import { onMount, onDestroy } from 'svelte';
    import CrawlerHero from '$lib/components/crawler/CrawlerHero.svelte';
    import CrawlerControls from '$lib/components/crawler/CrawlerControls.svelte';
    import ActiveChaptersCard from '$lib/components/crawler/ActiveChaptersCard.svelte';
    import ActiveJobsTable from '$lib/components/crawler/ActiveJobsTable.svelte';
    import MissingCoversTable from '$lib/components/crawler/MissingCoversTable.svelte';
    import DeadJobsTable from '$lib/components/crawler/DeadJobsTable.svelte';
    import CrawlerHistoryTable from '$lib/components/crawler/CrawlerHistoryTable.svelte';
    import CrawlerMetricsPanel from '$lib/components/crawler/CrawlerMetricsPanel.svelte';
    import SegmentedControl from '$lib/components/SegmentedControl.svelte';
    import SessionModal from '$lib/components/crawler/SessionModal.svelte';
    import RestartConfirmModal from '$lib/components/crawler/RestartConfirmModal.svelte';
    import RequeueAllConfirmModal from '$lib/components/crawler/RequeueAllConfirmModal.svelte';
    import {
        getCrawlerStatus,
        crawlerStatusStreamUrl,
        runCrawlerPass,
        reconcileCrawler,
        restartCrawlerBrowser,
        updateCrawlerSession,
        clearCrawlerSessionExpired,
        listDeadJobs,
        requeueDeadJobs,
        listActiveJobs,
        listMissingCovers,
        type CrawlerStatus,
        type DeadJob,
        type ActiveJob,
        type MissingCover,
        type RequeueScope
    } from '$lib/api/admin';

    // Segmented Live/History/Metrics view. Live keeps the SSE-driven
    // dashboard; History is a searchable job log; Metrics shows durations.
    let view = $state<'live' | 'history' | 'metrics'>('live');

    let status: CrawlerStatus | null = $state(null);
    let error: string | null = $state(null);
    let notice: string | null = $state(null);
    let live = $state(false);
    let source: EventSource | null = null;
    let busy = $state(false);
    // SSE error tracking (Q3): EventSource swallows status codes, so an
    // auth-loss looks identical to a network blip and would reconnect
    // forever. After this many consecutive errors we probe the status
    // endpoint, which routes through `on401Hook` and surfaces the
    // logout if appropriate.
    let consecutiveSseErrors = 0;
    const SSE_ERROR_PROBE_THRESHOLD = 5;
    // Backlog refetch AbortControllers (Q4): a burst of SSE frames or
    // search-key changes would otherwise fire concurrent fetches whose
    // resolution order is last-wins-flicker. Each loader cancels its
    // predecessor.
    let activeAbort: AbortController | null = null;
    let coversAbort: AbortController | null = null;
    let deadAbort: AbortController | null = null;
    // P4: debounce timers for backlog refetches. The SSE frame
    // (chapter page tick, etc.) fires ~once/sec; refetching the queue
    // on every frame produced a thundering herd against the same DB
    // queries the SSE handler is already running.
    let activeDebounce: ReturnType<typeof setTimeout> | null = null;
    let coversDebounce: ReturnType<typeof setTimeout> | null = null;
    const BACKLOG_DEBOUNCE_MS = 500;

    // Dead jobs
    let deadJobs: DeadJob[] = $state([]);
    let deadTotal = $state(0);
    let deadSearch = $state('');
    let deadPage = $state(1);
    const DEAD_LIMIT = 20;

    // Queued chapters (pending/running)
    let activeJobs: ActiveJob[] = $state([]);
    let activeTotal = $state(0);
    let activeSearch = $state('');
    let activePage = $state(1);
    const ACTIVE_LIMIT = 20;

    // Queued covers (mangas missing a cover)
    let covers: MissingCover[] = $state([]);
    let coversTotal = $state(0);
    let coversSearch = $state('');
    let coversPage = $state(1);
    const COVERS_LIMIT = 20;

    // Modals
    let sessionModalOpen = $state(false);
    let restartModalOpen = $state(false);
    let requeueAllModalOpen = $state(false);
    let phpsessid = $state('');
    let sessionResult: string | null = $state(null);

    async function refresh() {
        try {
            status = await getCrawlerStatus();
            error = null;
        } catch (e) {
            error = e instanceof Error ? e.message : 'refresh failed';
        }
    }

    async function loadDeadJobs() {
        deadAbort?.abort();
        const ctrl = new AbortController();
        deadAbort = ctrl;
        try {
            const resp = await listDeadJobs(
                {
                    search: deadSearch.trim() || undefined,
                    limit: DEAD_LIMIT,
                    offset: (deadPage - 1) * DEAD_LIMIT
                },
                { signal: ctrl.signal }
            );
            deadJobs = resp.items;
            deadTotal = resp.page.total ?? resp.items.length;
        } catch (e) {
            if (ctrl.signal.aborted) return; // superseded by a newer call
            error = e instanceof Error ? e.message : 'failed to load dead jobs';
        }
    }

    async function loadActiveJobs() {
        activeAbort?.abort();
        const ctrl = new AbortController();
        activeAbort = ctrl;
        try {
            const resp = await listActiveJobs(
                {
                    search: activeSearch.trim() || undefined,
                    limit: ACTIVE_LIMIT,
                    offset: (activePage - 1) * ACTIVE_LIMIT
                },
                { signal: ctrl.signal }
            );
            activeJobs = resp.items;
            activeTotal = resp.page.total ?? resp.items.length;
        } catch (e) {
            if (ctrl.signal.aborted) return;
            error = e instanceof Error ? e.message : 'failed to load queued chapters';
        }
    }

    async function loadCovers() {
        coversAbort?.abort();
        const ctrl = new AbortController();
        coversAbort = ctrl;
        try {
            const resp = await listMissingCovers(
                {
                    search: coversSearch.trim() || undefined,
                    limit: COVERS_LIMIT,
                    offset: (coversPage - 1) * COVERS_LIMIT
                },
                { signal: ctrl.signal }
            );
            covers = resp.items;
            coversTotal = resp.page.total ?? resp.items.length;
        } catch (e) {
            if (ctrl.signal.aborted) return;
            error = e instanceof Error ? e.message : 'failed to load queued covers';
        }
    }

    // Auto-refresh the (fetched, not streamed) backlog lists when the live
    // status shows the relevant counts moved — keeps the lists feeling
    // live without pushing big payloads over SSE. `$effect` re-runs when
    // these tracked values change; the timers debounce a burst of
    // adjacent counts (typical during a chapter download that emits one
    // poke per stored page) into a single refetch.
    let lastQueueKey = $state('');
    let lastCoversKey = $state(-1);
    $effect(() => {
        const k = `${status?.queue.pending ?? 0}:${status?.queue.running ?? 0}`;
        if (k !== lastQueueKey) {
            lastQueueKey = k;
            if (activeDebounce) clearTimeout(activeDebounce);
            activeDebounce = setTimeout(() => loadActiveJobs(), BACKLOG_DEBOUNCE_MS);
        }
    });
    $effect(() => {
        const c = status?.covers_queued ?? -1;
        if (c !== lastCoversKey) {
            lastCoversKey = c;
            if (coversDebounce) clearTimeout(coversDebounce);
            coversDebounce = setTimeout(() => loadCovers(), BACKLOG_DEBOUNCE_MS);
        }
    });

    // Live updates via Server-Sent Events instead of polling. The
    // EventSource is opened on mount and closed on destroy, so the
    // subscription exists only while this page is showing live data.
    function openStream() {
        if (source) return;
        const es = new EventSource(crawlerStatusStreamUrl(), { withCredentials: true });
        es.addEventListener('status', (e) => {
            try {
                status = JSON.parse((e as MessageEvent).data) as CrawlerStatus;
                error = null;
                live = true;
                consecutiveSseErrors = 0;
            } catch {
                // ignore a malformed frame; the next one will replace it
            }
        });
        es.onopen = () => {
            live = true;
            consecutiveSseErrors = 0;
        };
        es.onerror = () => {
            // The browser auto-reconnects; reflect the gap in the UI.
            live = false;
            consecutiveSseErrors += 1;
            // Q3: EventSource can't read the HTTP status, so an auth
            // loss (401) looks identical to a network blip. After
            // several consecutive failures, probe the status endpoint
            // via the normal API client so on401Hook can fire if the
            // session is gone.
            if (consecutiveSseErrors >= SSE_ERROR_PROBE_THRESHOLD) {
                consecutiveSseErrors = 0; // reset so we don't probe-storm
                getCrawlerStatus().catch(() => {
                    // The client's on401 hook (set globally) handles
                    // the logout; any other error we ignore — the
                    // EventSource will keep retrying and the UI flag
                    // already reflects `live = false`.
                });
            }
        };
        source = es;
    }

    function closeStream() {
        source?.close();
        source = null;
        live = false;
    }

    // Q2: close the stream when the tab is hidden so the browser
    // doesn't accumulate a stale connection (mobile Safari throttles SSE
    // aggressively in background tabs), and reopen on visible. Same for
    // pagehide / pageshow to handle the BFCache.
    function onVisibilityChange() {
        if (typeof document === 'undefined') return;
        if (document.visibilityState === 'hidden') {
            closeStream();
        } else if (document.visibilityState === 'visible') {
            openStream();
        }
    }
    function onPageHide() {
        closeStream();
    }
    function onPageShow() {
        openStream();
    }

    onMount(() => {
        // One-shot fetch for instant initial paint + resilience if SSE is
        // blocked; the stream then drives subsequent updates.
        refresh();
        loadDeadJobs();
        openStream();
        if (typeof document !== 'undefined') {
            document.addEventListener('visibilitychange', onVisibilityChange);
            window.addEventListener('pagehide', onPageHide);
            window.addEventListener('pageshow', onPageShow);
        }
    });
    onDestroy(() => {
        closeStream();
        if (typeof document !== 'undefined') {
            document.removeEventListener('visibilitychange', onVisibilityChange);
            window.removeEventListener('pagehide', onPageHide);
            window.removeEventListener('pageshow', onPageShow);
        }
        if (activeDebounce) clearTimeout(activeDebounce);
        if (coversDebounce) clearTimeout(coversDebounce);
        activeAbort?.abort();
        coversAbort?.abort();
        deadAbort?.abort();
    });

    async function withBusy(label: string, fn: () => Promise<void>) {
        busy = true;
        notice = null;
        error = null;
        try {
            await fn();
        } catch (e) {
            error = e instanceof Error ? e.message : `${label} failed`;
        } finally {
            busy = false;
            // Q5: only fall back to a fetch refresh when the live SSE
            // stream isn't pushing updates. With `live === true` the
            // backend's `status.poke()` after each action already
            // pushes a fresh status frame; the extra fetch would race
            // it and produce flicker.
            if (!live) {
                await refresh();
            }
        }
    }

    async function onRunPass() {
        await withBusy('run pass', async () => {
            await runCrawlerPass();
            notice = 'Metadata pass started.';
        });
    }

    async function onReconcile() {
        await withBusy('reconcile', async () => {
            await reconcileCrawler();
            notice = 'Reconcile started — walking the full list for missing mangas.';
        });
    }

    async function onConfirmRestart() {
        restartModalOpen = false;
        await withBusy('restart browser', async () => {
            const r = await restartCrawlerBrowser();
            notice = r.ok ? 'Browser restarted.' : `Restart failed: ${r.error ?? 'unknown'}`;
        });
    }

    async function onSaveSession() {
        sessionResult = null;
        busy = true;
        try {
            const r = await updateCrawlerSession(phpsessid);
            sessionResult = r.valid
                ? '✓ Session valid — workers resumed.'
                : `✕ Probe failed: ${r.error ?? 'unauthenticated'}`;
            if (r.valid) {
                sessionModalOpen = false;
                phpsessid = '';
                notice = 'Session updated.';
            }
        } catch (e) {
            sessionResult = e instanceof Error ? e.message : 'update failed';
        } finally {
            busy = false;
            if (!live) await refresh();
        }
    }

    async function onClearExpired() {
        await withBusy('clear expired', async () => {
            await clearCrawlerSessionExpired();
            notice = 'Session-expired flag cleared.';
        });
    }

    async function requeue(scope: RequeueScope) {
        await withBusy('requeue', async () => {
            const r = await requeueDeadJobs(scope);
            notice = `Requeued ${r.requeued} job(s).`;
            await loadDeadJobs();
        });
    }

    async function onConfirmRequeueAll() {
        requeueAllModalOpen = false;
        await requeue({ scope: 'all', confirm: true });
    }

    function onSearchDead() {
        deadPage = 1;
        loadDeadJobs();
    }
    function onDeadPageChange(p: number) {
        deadPage = p;
        loadDeadJobs();
    }
    function onSearchActive() {
        activePage = 1;
        loadActiveJobs();
    }
    function onActivePageChange(p: number) {
        activePage = p;
        loadActiveJobs();
    }
    function onSearchCovers() {
        coversPage = 1;
        loadCovers();
    }
    function onCoversPageChange(p: number) {
        coversPage = p;
        loadCovers();
    }

    const deadTotalPages = $derived(Math.max(1, Math.ceil(deadTotal / DEAD_LIMIT)));
    const activeTotalPages = $derived(Math.max(1, Math.ceil(activeTotal / ACTIVE_LIMIT)));
    const coversTotalPages = $derived(Math.max(1, Math.ceil(coversTotal / COVERS_LIMIT)));
</script>

<div class="titlebar">
    <h1>Crawler</h1>
    <span class="livedot" class:on={live} title={live ? 'Live (SSE)' : 'Reconnecting…'}>
        {live ? '● live' : '○ reconnecting…'}
    </span>
</div>

<div class="viewtabs">
    <SegmentedControl
        options={[
            { label: 'Live', value: 'live' },
            { label: 'History', value: 'history' },
            { label: 'Metrics', value: 'metrics' }
        ]}
        value={view}
        onchange={(v) => (view = v)}
        ariaLabel="Crawler view"
        testid="crawler-tab"
    />
</div>

{#if error}
    <p class="error" role="alert">{error}</p>
{/if}
{#if notice}
    <p class="notice" role="status">{notice}</p>
{/if}

{#if view === 'history'}
    <CrawlerHistoryTable onRequeue={requeue} {busy} />
{:else if view === 'metrics'}
    <CrawlerMetricsPanel />
{:else}
{#if status}
    <CrawlerHero {status} />

    <CrawlerControls
        {status}
        {busy}
        {onRunPass}
        {onReconcile}
        onOpenRestart={() => (restartModalOpen = true)}
        onOpenSession={() => {
            sessionModalOpen = true;
            sessionResult = null;
        }}
        {onClearExpired}
    />

    <ActiveChaptersCard {status} />
{:else}
    <p>Loading…</p>
{/if}

<ActiveJobsTable
    jobs={activeJobs}
    total={activeTotal}
    page={activePage}
    totalPages={activeTotalPages}
    bind:search={activeSearch}
    onSearch={onSearchActive}
    onPageChange={onActivePageChange}
/>

<MissingCoversTable
    {covers}
    total={coversTotal}
    page={coversPage}
    totalPages={coversTotalPages}
    bind:search={coversSearch}
    onSearch={onSearchCovers}
    onPageChange={onCoversPageChange}
/>

<DeadJobsTable
    jobs={deadJobs}
    total={deadTotal}
    page={deadPage}
    totalPages={deadTotalPages}
    bind:search={deadSearch}
    {busy}
    onSearch={onSearchDead}
    onPageChange={onDeadPageChange}
    onRequeue={requeue}
    onRequeueAll={() => (requeueAllModalOpen = true)}
/>
{/if}

<RestartConfirmModal
    open={restartModalOpen}
    {busy}
    onCancel={() => (restartModalOpen = false)}
    onConfirm={onConfirmRestart}
/>

<RequeueAllConfirmModal
    open={requeueAllModalOpen}
    total={deadTotal}
    {busy}
    onCancel={() => (requeueAllModalOpen = false)}
    onConfirm={onConfirmRequeueAll}
/>

<SessionModal
    open={sessionModalOpen}
    {busy}
    bind:phpsessid
    result={sessionResult}
    onCancel={() => (sessionModalOpen = false)}
    onSave={onSaveSession}
/>

<style>
    h1 {
        margin: 0;
    }
    .titlebar {
        display: flex;
        align-items: baseline;
        gap: var(--space-3);
        margin-bottom: var(--space-4);
    }
    .livedot {
        font-size: var(--font-sm);
        color: var(--text-muted);
    }
    .livedot.on {
        color: var(--success, #0a7d2c);
    }
    .viewtabs {
        margin-bottom: var(--space-4);
    }
    .notice {
        color: var(--success, #0a7d2c);
        padding: var(--space-2) var(--space-3);
        border: 1px solid var(--success, #0a7d2c);
        border-radius: var(--radius-md);
        margin-bottom: var(--space-3);
    }
    .error {
        color: var(--danger, #dc2626);
        padding: var(--space-2) var(--space-3);
        border: 1px solid var(--danger, #dc2626);
        border-radius: var(--radius-md);
        margin-bottom: var(--space-3);
    }
    /* badges (shared convention with admin/mangas) */
    :global(.badge) {
        display: inline-block;
        padding: 0 var(--space-2);
        border-radius: var(--radius-sm, 4px);
        font-size: var(--font-xs);
        font-weight: var(--weight-semibold);
        text-transform: uppercase;
        letter-spacing: 0.04em;
        border: 1px solid var(--border);
        background: var(--surface);
    }
    :global(.badge-synced) {
        background: #dcfce7;
        color: #166534;
        border-color: #86efac;
    }
    :global(.badge-in_progress),
    :global(.badge-downloading) {
        background: #fef3c7;
        color: #92400e;
        border-color: #fcd34d;
    }
    :global(.badge-not_downloaded) {
        background: var(--surface-elevated);
        color: var(--text-muted);
    }
</style>