The interleaved metadata pass misses mangas to list drift (a title slips a pagination slot during the slow detail walk). Add a reconcile pass: a cheap, full, list-only walk (refs only, no detail visit, no early stop) that set-diffs the walked keys against manga_sources and enqueues the strictly- missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only has to appear somewhere in the list. - Build the previously-dead SyncManga worker by extending RealChapterDispatcher to run the shared pipeline::process_manga_ref (fetch → upsert → cover → chapters), refactored out of run_metadata_pass so both paths stay in lockstep. The crawl worker now leases both sync_chapter_content and sync_manga (jobs::lease_kinds); both serialize on the single exclusive browser. - SyncManga payload carries url + title so the worker can rebuild the ref and the dead-jobs/history UI can label a missing manga that has no manga row yet. - "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is skipped when a pending/running/dead SyncManga job already exists, so a gone manga (detail 404 → retries → dead) is left dead and not retried. - New POST /v1/admin/crawler/reconcile (fire-and-forget, shares manual_pass_lock) + "Reconcile missing" admin button + Reconciling status phase streamed over SSE. - dead-jobs/history queries surface payload title/url/key; tables fall back to them for sync_manga rows; both searches match the payload title. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
151 lines
5.0 KiB
Svelte
151 lines
5.0 KiB
Svelte
<script lang="ts">
|
|
import type { CrawlerStatus, CrawlerPhase } from '$lib/api/admin';
|
|
import ProgressBar from './ProgressBar.svelte';
|
|
|
|
let { status }: { status: CrawlerStatus } = $props();
|
|
|
|
function phaseLabel(p: CrawlerPhase | null): string {
|
|
if (!p) return 'Daemon disabled';
|
|
switch (p.state) {
|
|
case 'idle':
|
|
return p.next_fire
|
|
? `Idle — next pass ${new Date(p.next_fire).toLocaleString()}`
|
|
: 'Idle';
|
|
case 'walking_list':
|
|
return 'Walking source list';
|
|
case 'fetching_metadata':
|
|
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
|
|
case 'cover_backfill':
|
|
return `Backfilling covers · ${p.index + 1}/${p.total}`;
|
|
case 'reconciling':
|
|
return `Reconciling · walked ${p.walked} · enqueued ${p.enqueued}`;
|
|
default: {
|
|
// Exhaustive default: if a new phase state ships
|
|
// without a label, TypeScript flags the missing case
|
|
// at compile time. Keeps the UI from showing
|
|
// `undefined` if the server gets ahead of the client.
|
|
const _exhaustive: never = p;
|
|
void _exhaustive;
|
|
return 'Unknown phase';
|
|
}
|
|
}
|
|
}
|
|
|
|
function phasePercent(p: CrawlerPhase | null): number | null {
|
|
if (p && p.state === 'fetching_metadata' && p.total && p.total > 0) {
|
|
return Math.min(100, (p.index / p.total) * 100);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sessionPill(s: CrawlerStatus): { cls: string; text: string } {
|
|
if (s.daemon === 'disabled') return { cls: 'badge-not_downloaded', text: 'n/a' };
|
|
if (s.session.expired) return { cls: 'badge-in_progress', text: 'Expired' };
|
|
if (!s.session.configured) return { cls: 'badge-not_downloaded', text: 'Not set' };
|
|
return { cls: 'badge-synced', text: 'OK' };
|
|
}
|
|
|
|
function browserPill(s: CrawlerStatus): { cls: string; text: string } {
|
|
switch (s.browser) {
|
|
case 'healthy':
|
|
return { cls: 'badge-synced', text: 'Up' };
|
|
case 'draining':
|
|
case 'restarting':
|
|
return { cls: 'badge-in_progress', text: s.browser };
|
|
default:
|
|
return { cls: 'badge-not_downloaded', text: 'Down' };
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<section class="hero" data-testid="crawler-hero">
|
|
<div class="pills">
|
|
<span class="pill"
|
|
>Daemon
|
|
<span
|
|
class={`badge ${status.daemon === 'running' ? 'badge-synced' : 'badge-not_downloaded'}`}
|
|
>{status.daemon}</span
|
|
></span
|
|
>
|
|
<span class="pill"
|
|
>Session
|
|
<span class={`badge ${sessionPill(status).cls}`}>{sessionPill(status).text}</span></span
|
|
>
|
|
<span class="pill"
|
|
>Browser
|
|
<span class={`badge ${browserPill(status).cls}`}>{browserPill(status).text}</span></span
|
|
>
|
|
</div>
|
|
|
|
<p class="phase" data-testid="crawler-phase">{phaseLabel(status.phase)}</p>
|
|
{#if phasePercent(status.phase) !== null}
|
|
<ProgressBar percent={phasePercent(status.phase) ?? 0} />
|
|
{/if}
|
|
|
|
{#if status.session.expired}
|
|
<p class="warn">
|
|
⚠ Chapter downloads paused — session expired. Metadata + list crawl continue.
|
|
</p>
|
|
{/if}
|
|
|
|
{#if status.current_cover}
|
|
<p class="cover" data-testid="current-cover">
|
|
🖼 Fetching cover: <strong>{status.current_cover.manga_title}</strong>
|
|
</p>
|
|
{/if}
|
|
|
|
<p class="lastpass">
|
|
Last pass:
|
|
{#if status.last_pass.at}
|
|
{new Date(status.last_pass.at).toLocaleString()} ·
|
|
{status.last_pass.discovered} seen · {status.last_pass.upserted} upserted ·
|
|
{status.last_pass.mangas_failed} failed
|
|
{:else}
|
|
— none yet this session
|
|
{/if}
|
|
</p>
|
|
</section>
|
|
|
|
<style>
|
|
.hero {
|
|
padding: var(--space-4);
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
background: var(--surface);
|
|
margin-bottom: var(--space-4);
|
|
}
|
|
.pills {
|
|
display: flex;
|
|
gap: var(--space-4);
|
|
flex-wrap: wrap;
|
|
margin-bottom: var(--space-3);
|
|
}
|
|
.pill {
|
|
font-size: var(--font-sm);
|
|
color: var(--text-muted);
|
|
display: inline-flex;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
}
|
|
.phase {
|
|
font-size: var(--font-lg);
|
|
font-weight: var(--weight-semibold);
|
|
margin: var(--space-2) 0;
|
|
}
|
|
.lastpass {
|
|
color: var(--text-muted);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.warn {
|
|
color: #92400e;
|
|
background: #fef3c7;
|
|
border: 1px solid #fcd34d;
|
|
padding: var(--space-2) var(--space-3);
|
|
border-radius: var(--radius-md);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.cover {
|
|
font-size: var(--font-sm);
|
|
}
|
|
</style>
|