Files
Mangalord/frontend/src/lib/components/crawler/CrawlerHero.svelte
MechaCat02 dd300a150c feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)
The interleaved metadata pass misses mangas to list drift (a title slips a
pagination slot during the slow detail walk). Add a reconcile pass: a cheap,
full, list-only walk (refs only, no detail visit, no early stop) that
set-diffs the walked keys against manga_sources and enqueues the strictly-
missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only
has to appear somewhere in the list.

- Build the previously-dead SyncManga worker by extending RealChapterDispatcher
  to run the shared pipeline::process_manga_ref (fetch → upsert → cover →
  chapters), refactored out of run_metadata_pass so both paths stay in lockstep.
  The crawl worker now leases both sync_chapter_content and sync_manga
  (jobs::lease_kinds); both serialize on the single exclusive browser.
- SyncManga payload carries url + title so the worker can rebuild the ref and
  the dead-jobs/history UI can label a missing manga that has no manga row yet.
- "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is
  skipped when a pending/running/dead SyncManga job already exists, so a gone
  manga (detail 404 → retries → dead) is left dead and not retried.
- New POST /v1/admin/crawler/reconcile (fire-and-forget, shares
  manual_pass_lock) + "Reconcile missing" admin button + Reconciling status
  phase streamed over SSE.
- dead-jobs/history queries surface payload title/url/key; tables fall back to
  them for sync_manga rows; both searches match the payload title.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-19 11:15:11 +02:00

151 lines
5.0 KiB
Svelte

<script lang="ts">
import type { CrawlerStatus, CrawlerPhase } from '$lib/api/admin';
import ProgressBar from './ProgressBar.svelte';
let { status }: { status: CrawlerStatus } = $props();
function phaseLabel(p: CrawlerPhase | null): string {
if (!p) return 'Daemon disabled';
switch (p.state) {
case 'idle':
return p.next_fire
? `Idle — next pass ${new Date(p.next_fire).toLocaleString()}`
: 'Idle';
case 'walking_list':
return 'Walking source list';
case 'fetching_metadata':
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
case 'cover_backfill':
return `Backfilling covers · ${p.index + 1}/${p.total}`;
case 'reconciling':
return `Reconciling · walked ${p.walked} · enqueued ${p.enqueued}`;
default: {
// Exhaustive default: if a new phase state ships
// without a label, TypeScript flags the missing case
// at compile time. Keeps the UI from showing
// `undefined` if the server gets ahead of the client.
const _exhaustive: never = p;
void _exhaustive;
return 'Unknown phase';
}
}
}
function phasePercent(p: CrawlerPhase | null): number | null {
if (p && p.state === 'fetching_metadata' && p.total && p.total > 0) {
return Math.min(100, (p.index / p.total) * 100);
}
return null;
}
function sessionPill(s: CrawlerStatus): { cls: string; text: string } {
if (s.daemon === 'disabled') return { cls: 'badge-not_downloaded', text: 'n/a' };
if (s.session.expired) return { cls: 'badge-in_progress', text: 'Expired' };
if (!s.session.configured) return { cls: 'badge-not_downloaded', text: 'Not set' };
return { cls: 'badge-synced', text: 'OK' };
}
function browserPill(s: CrawlerStatus): { cls: string; text: string } {
switch (s.browser) {
case 'healthy':
return { cls: 'badge-synced', text: 'Up' };
case 'draining':
case 'restarting':
return { cls: 'badge-in_progress', text: s.browser };
default:
return { cls: 'badge-not_downloaded', text: 'Down' };
}
}
</script>
<section class="hero" data-testid="crawler-hero">
<div class="pills">
<span class="pill"
>Daemon
<span
class={`badge ${status.daemon === 'running' ? 'badge-synced' : 'badge-not_downloaded'}`}
>{status.daemon}</span
></span
>
<span class="pill"
>Session
<span class={`badge ${sessionPill(status).cls}`}>{sessionPill(status).text}</span></span
>
<span class="pill"
>Browser
<span class={`badge ${browserPill(status).cls}`}>{browserPill(status).text}</span></span
>
</div>
<p class="phase" data-testid="crawler-phase">{phaseLabel(status.phase)}</p>
{#if phasePercent(status.phase) !== null}
<ProgressBar percent={phasePercent(status.phase) ?? 0} />
{/if}
{#if status.session.expired}
<p class="warn">
⚠ Chapter downloads paused — session expired. Metadata + list crawl continue.
</p>
{/if}
{#if status.current_cover}
<p class="cover" data-testid="current-cover">
🖼 Fetching cover: <strong>{status.current_cover.manga_title}</strong>
</p>
{/if}
<p class="lastpass">
Last pass:
{#if status.last_pass.at}
{new Date(status.last_pass.at).toLocaleString()} ·
{status.last_pass.discovered} seen · {status.last_pass.upserted} upserted ·
{status.last_pass.mangas_failed} failed
{:else}
— none yet this session
{/if}
</p>
</section>
<style>
.hero {
padding: var(--space-4);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
margin-bottom: var(--space-4);
}
.pills {
display: flex;
gap: var(--space-4);
flex-wrap: wrap;
margin-bottom: var(--space-3);
}
.pill {
font-size: var(--font-sm);
color: var(--text-muted);
display: inline-flex;
align-items: center;
gap: var(--space-2);
}
.phase {
font-size: var(--font-lg);
font-weight: var(--weight-semibold);
margin: var(--space-2) 0;
}
.lastpass {
color: var(--text-muted);
font-size: var(--font-sm);
}
.warn {
color: #92400e;
background: #fef3c7;
border: 1px solid #fcd34d;
padding: var(--space-2) var(--space-3);
border-radius: var(--radius-md);
font-size: var(--font-sm);
}
.cover {
font-size: var(--font-sm);
}
</style>