dd300a150c
The interleaved metadata pass misses mangas to list drift (a title slips a pagination slot during the slow detail walk). Add a reconcile pass: a cheap, full, list-only walk (refs only, no detail visit, no early stop) that set-diffs the walked keys against manga_sources and enqueues the strictly- missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only has to appear somewhere in the list. - Build the previously-dead SyncManga worker by extending RealChapterDispatcher to run the shared pipeline::process_manga_ref (fetch → upsert → cover → chapters), refactored out of run_metadata_pass so both paths stay in lockstep. The crawl worker now leases both sync_chapter_content and sync_manga (jobs::lease_kinds); both serialize on the single exclusive browser. - SyncManga payload carries url + title so the worker can rebuild the ref and the dead-jobs/history UI can label a missing manga that has no manga row yet. - "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is skipped when a pending/running/dead SyncManga job already exists, so a gone manga (detail 404 → retries → dead) is left dead and not retried. - New POST /v1/admin/crawler/reconcile (fire-and-forget, shares manual_pass_lock) + "Reconcile missing" admin button + Reconciling status phase streamed over SSE. - dead-jobs/history queries surface payload title/url/key; tables fall back to them for sync_manga rows; both searches match the payload title. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
303 lines
9.7 KiB
Svelte
303 lines
9.7 KiB
Svelte
<script lang="ts">
|
|
import { onMount } from 'svelte';
|
|
import Pager from '$lib/components/Pager.svelte';
|
|
import { fmtDuration } from '$lib/format';
|
|
import SearchBar from './SearchBar.svelte';
|
|
import {
|
|
listCrawlerJobHistory,
|
|
type CrawlerHistoryRow,
|
|
type CrawlerJobState,
|
|
type CrawlerJobKind,
|
|
type RequeueScope
|
|
} from '$lib/api/admin';
|
|
|
|
// Requeue is owned by the parent (shared with the dead-jobs table); we
|
|
// call it then reload so the row's new state shows. `busy` disables the
|
|
// inline button during a parent-driven action.
|
|
let {
|
|
onRequeue,
|
|
busy = false
|
|
}: {
|
|
onRequeue: (scope: RequeueScope) => Promise<void>;
|
|
busy?: boolean;
|
|
} = $props();
|
|
|
|
const LIMIT = 25;
|
|
|
|
let rows = $state<CrawlerHistoryRow[]>([]);
|
|
let total = $state(0);
|
|
let page = $state(1);
|
|
let stateFilter = $state<'' | CrawlerJobState>('');
|
|
let kindFilter = $state<'' | CrawlerJobKind>('');
|
|
let search = $state('');
|
|
let loading = $state(true);
|
|
let error = $state<string | null>(null);
|
|
let expanded = $state<string | null>(null);
|
|
|
|
const totalPages = $derived(Math.max(1, Math.ceil(total / LIMIT)));
|
|
|
|
async function load() {
|
|
loading = true;
|
|
error = null;
|
|
try {
|
|
const resp = await listCrawlerJobHistory({
|
|
state: stateFilter || undefined,
|
|
kind: kindFilter || undefined,
|
|
search: search.trim() || undefined,
|
|
limit: LIMIT,
|
|
offset: (page - 1) * LIMIT
|
|
});
|
|
rows = resp.items;
|
|
total = resp.page.total ?? resp.items.length;
|
|
} catch (e) {
|
|
error = e instanceof Error ? e.message : 'Failed to load history.';
|
|
} finally {
|
|
loading = false;
|
|
}
|
|
}
|
|
|
|
onMount(load);
|
|
|
|
function applyFilters() {
|
|
page = 1;
|
|
load();
|
|
}
|
|
function onPageChange(p: number) {
|
|
page = p;
|
|
load();
|
|
}
|
|
async function requeue(scope: RequeueScope) {
|
|
await onRequeue(scope);
|
|
await load();
|
|
}
|
|
|
|
/** A human "Manga · Ch N · pP" target. Falls back to the payload title
|
|
* (sync_manga jobs with no manga row yet), then the source key. */
|
|
function target(r: CrawlerHistoryRow): string {
|
|
if (r.manga_title) {
|
|
let s = r.manga_title;
|
|
if (r.chapter_number != null) s += ` · Ch ${r.chapter_number}`;
|
|
if (r.page_number != null) s += ` · p${r.page_number}`;
|
|
return s;
|
|
}
|
|
return r.payload_title ?? r.source_key ?? '—';
|
|
}
|
|
|
|
function fmtAgo(iso: string): string {
|
|
const then = new Date(iso).getTime();
|
|
const secs = Math.round((Date.now() - then) / 1000);
|
|
if (secs < 45) return 'just now';
|
|
if (secs < 90) return '1m ago';
|
|
const mins = Math.round(secs / 60);
|
|
if (mins < 60) return `${mins}m ago`;
|
|
const hrs = Math.round(mins / 60);
|
|
if (hrs < 24) return `${hrs}h ago`;
|
|
return new Date(iso).toLocaleDateString();
|
|
}
|
|
|
|
const KINDS: { value: '' | CrawlerJobKind; label: string }[] = [
|
|
{ value: '', label: 'All types' },
|
|
{ value: 'sync_manga', label: 'sync_manga' },
|
|
{ value: 'sync_chapter_list', label: 'sync_chapter_list' },
|
|
{ value: 'sync_chapter_content', label: 'sync_chapter' },
|
|
{ value: 'analyze_page', label: 'analyze_page' }
|
|
];
|
|
const STATES: { value: '' | CrawlerJobState; label: string }[] = [
|
|
{ value: '', label: 'All states' },
|
|
{ value: 'done', label: 'done' },
|
|
{ value: 'dead', label: 'dead' },
|
|
{ value: 'running', label: 'running' },
|
|
{ value: 'pending', label: 'pending' }
|
|
];
|
|
</script>
|
|
|
|
<section class="history" data-testid="crawler-history">
|
|
<div class="toolbar">
|
|
<select
|
|
bind:value={stateFilter}
|
|
onchange={applyFilters}
|
|
aria-label="Filter by state"
|
|
data-testid="crawler-history-state"
|
|
>
|
|
{#each STATES as s (s.value)}<option value={s.value}>{s.label}</option>{/each}
|
|
</select>
|
|
<select
|
|
bind:value={kindFilter}
|
|
onchange={applyFilters}
|
|
aria-label="Filter by type"
|
|
data-testid="crawler-history-kind"
|
|
>
|
|
{#each KINDS as k (k.value)}<option value={k.value}>{k.label}</option>{/each}
|
|
</select>
|
|
<SearchBar
|
|
bind:value={search}
|
|
placeholder="Search manga / chapter…"
|
|
onSearch={applyFilters}
|
|
/>
|
|
</div>
|
|
<p class="muted note">
|
|
Showing recent jobs — completed jobs are pruned after the retention
|
|
window; dead jobs persist until requeued.
|
|
</p>
|
|
|
|
{#if error}
|
|
<p class="error" role="alert">{error}</p>
|
|
{/if}
|
|
|
|
{#if loading}
|
|
<p class="muted" data-testid="crawler-history-loading">Loading…</p>
|
|
{:else if rows.length === 0}
|
|
<p class="muted" data-testid="crawler-history-empty">No jobs match.</p>
|
|
{:else}
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>State</th>
|
|
<th>Type</th>
|
|
<th>Target</th>
|
|
<th class="num">Dur</th>
|
|
<th>Att.</th>
|
|
<th>Updated</th>
|
|
<th class="actions"></th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{#each rows as r (r.id)}
|
|
<tr
|
|
class:clickable={!!r.last_error}
|
|
onclick={() =>
|
|
r.last_error && (expanded = expanded === r.id ? null : r.id)}
|
|
data-testid={`crawler-history-row-${r.id}`}
|
|
>
|
|
<td>
|
|
<span class="badge state-{r.state}">{r.state}</span>
|
|
</td>
|
|
<td class="kind">{r.kind ?? '—'}</td>
|
|
<td>
|
|
{#if !r.manga_title && r.payload_title && r.source_url}
|
|
<a href={r.source_url} target="_blank" rel="noreferrer noopener"
|
|
>{target(r)}</a
|
|
>
|
|
{:else}
|
|
{target(r)}
|
|
{/if}
|
|
</td>
|
|
<td class="num">{fmtDuration(r.duration_ms)}</td>
|
|
<td>{r.attempts}/{r.max_attempts}</td>
|
|
<td title={new Date(r.updated_at).toLocaleString()}
|
|
>{fmtAgo(r.updated_at)}</td
|
|
>
|
|
<td class="actions">
|
|
{#if r.state === 'dead'}
|
|
<button
|
|
type="button"
|
|
disabled={busy}
|
|
onclick={(e) => {
|
|
e.stopPropagation();
|
|
requeue({ scope: 'job', job_id: r.id });
|
|
}}
|
|
data-testid={`crawler-history-requeue-${r.id}`}
|
|
>
|
|
⤴ Requeue
|
|
</button>
|
|
{/if}
|
|
</td>
|
|
</tr>
|
|
{#if expanded === r.id && r.last_error}
|
|
<tr class="errrow">
|
|
<td colspan="7"><code>{r.last_error}</code></td>
|
|
</tr>
|
|
{/if}
|
|
{/each}
|
|
</tbody>
|
|
</table>
|
|
<Pager {page} {totalPages} onChange={onPageChange} testid="crawler-history-pager" />
|
|
{/if}
|
|
</section>
|
|
|
|
<style>
|
|
.toolbar {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
margin-bottom: var(--space-2);
|
|
}
|
|
select {
|
|
height: 36px;
|
|
padding: 0 var(--space-2);
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
background: var(--surface);
|
|
color: var(--text);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.note {
|
|
font-size: var(--font-xs);
|
|
}
|
|
.muted {
|
|
color: var(--text-muted);
|
|
}
|
|
table {
|
|
width: 100%;
|
|
border-collapse: collapse;
|
|
}
|
|
th,
|
|
td {
|
|
padding: var(--space-2);
|
|
text-align: left;
|
|
border-bottom: 1px solid var(--border);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.kind {
|
|
color: var(--text-muted);
|
|
font-family: var(--font-mono, monospace);
|
|
font-size: var(--font-xs);
|
|
}
|
|
.actions {
|
|
text-align: right;
|
|
}
|
|
.num {
|
|
text-align: right;
|
|
font-variant-numeric: tabular-nums;
|
|
}
|
|
tr.clickable {
|
|
cursor: pointer;
|
|
}
|
|
tr.clickable:hover {
|
|
background: var(--surface);
|
|
}
|
|
.errrow td {
|
|
background: var(--surface);
|
|
color: var(--text-muted);
|
|
}
|
|
.errrow code {
|
|
white-space: pre-wrap;
|
|
word-break: break-word;
|
|
font-size: var(--font-xs);
|
|
}
|
|
.error {
|
|
color: var(--danger, #dc2626);
|
|
}
|
|
/* State dots reuse the page's shared badge palette. */
|
|
.badge {
|
|
text-transform: uppercase;
|
|
}
|
|
.state-done {
|
|
background: #dcfce7;
|
|
color: #166534;
|
|
border-color: #86efac;
|
|
}
|
|
.state-dead {
|
|
background: color-mix(in srgb, var(--danger, #dc2626) 16%, transparent);
|
|
color: var(--danger, #dc2626);
|
|
border-color: color-mix(in srgb, var(--danger, #dc2626) 45%, transparent);
|
|
}
|
|
.state-running,
|
|
.state-pending {
|
|
background: #fef3c7;
|
|
color: #92400e;
|
|
border-color: #fcd34d;
|
|
}
|
|
</style>
|