fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): reconcile pass to enqueue mangas missing from the DB (0.85.0)

Browse Source 
The interleaved metadata pass misses mangas to list drift (a title slips a
pagination slot during the slow detail walk). Add a reconcile pass: a cheap,
full, list-only walk (refs only, no detail visit, no early stop) that
set-diffs the walked keys against manga_sources and enqueues the strictly-
missing ones as SyncManga jobs. A set-diff is immune to drift — a manga only
has to appear somewhere in the list.

- Build the previously-dead SyncManga worker by extending RealChapterDispatcher
  to run the shared pipeline::process_manga_ref (fetch → upsert → cover →
  chapters), refactored out of run_metadata_pass so both paths stay in lockstep.
  The crawl worker now leases both sync_chapter_content and sync_manga
  (jobs::lease_kinds); both serialize on the single exclusive browser.
- SyncManga payload carries url + title so the worker can rebuild the ref and
  the dead-jobs/history UI can label a missing manga that has no manga row yet.
- "Missing" = strict NOT EXISTS (dropped rows count as present). Re-enqueue is
  skipped when a pending/running/dead SyncManga job already exists, so a gone
  manga (detail 404 → retries → dead) is left dead and not retried.
- New POST /v1/admin/crawler/reconcile (fire-and-forget, shares
  manual_pass_lock) + "Reconcile missing" admin button + Reconciling status
  phase streamed over SSE.
- dead-jobs/history queries surface payload title/url/key; tables fall back to
  them for sync_manga rows; both searches match the payload title.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-16 21:50:54 +02:00
parent 35664bccc7
commit dd300a150c
26 changed files with 1387 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
frontend/src/lib/api/admin.test.ts
View File

@@ -23,6 +23,7 @@ import {
getCrawlerStatus,
crawlerStatusStreamUrl,
runCrawlerPass,
reconcileCrawler,
restartCrawlerBrowser,
updateCrawlerSession,
clearCrawlerSessionExpired,
@@ -479,6 +480,15 @@ describe('admin crawler api client', () => {
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
});
it('reconcileCrawler POSTs /v1/admin/crawler/reconcile', async () => {
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
const r = await reconcileCrawler();
expect(r.started).toBe(true);
const init = fetchSpy.mock.calls[0][1] as RequestInit;
expect(init.method).toBe('POST');
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/reconcile$/);
});
it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
const r = await restartCrawlerBrowser();

19
frontend/src/lib/api/admin.ts
View File

@@ -332,7 +332,8 @@ export type CrawlerPhase =
| { state: 'idle'; next_fire: string | null }
| { state: 'walking_list' }
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
| { state: 'cover_backfill'; index: number; total: number };
| { state: 'cover_backfill'; index: number; total: number }
| { state: 'reconciling'; walked: number; enqueued: number };
/** A chapter being crawled right now, with a live page count. */
export type ActiveChapter = {
@@ -382,6 +383,12 @@ export async function runCrawlerPass(): Promise<{ started: boolean }> {
return request('/v1/admin/crawler/run', { method: 'POST' });
}
/** POST /v1/admin/crawler/reconcile — full list-only walk that enqueues
* mangas missing from the DB as sync_manga jobs. */
export async function reconcileCrawler(): Promise<{ started: boolean }> {
return request('/v1/admin/crawler/reconcile', { method: 'POST' });
}
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
@@ -410,6 +417,12 @@ export type DeadJob = {
manga_id: string | null;
manga_title: string | null;
chapter_number: number | null;
/** Payload title (fallback label for sync_manga jobs with no manga row). */
payload_title: string | null;
/** Source detail URL from the payload (sync_manga). */
source_url: string | null;
/** Source-native key from the payload (sync_manga). */
source_key: string | null;
attempts: number;
max_attempts: number;
last_error: string | null;
@@ -513,6 +526,10 @@ export type CrawlerHistoryRow = {
chapter_number: number | null;
page_number: number | null;
source_key: string | null;
/** Payload title (fallback label for sync_manga jobs with no manga row). */
payload_title: string | null;
/** Source detail URL from the payload (sync_manga). */
source_url: string | null;
attempts: number;
max_attempts: number;
last_error: string | null;

8
frontend/src/lib/components/crawler/CrawlerControls.svelte
View File

@@ -5,6 +5,7 @@
status,
busy,
onRunPass,
onReconcile,
onOpenRestart,
onOpenSession,
onClearExpired
@@ -12,6 +13,7 @@
status: CrawlerStatus;
busy: boolean;
onRunPass: () => void;
onReconcile: () => void;
onOpenRestart: () => void;
onOpenSession: () => void;
onClearExpired: () => void;
@@ -22,6 +24,12 @@
<button onclick={onRunPass} disabled={busy || status.daemon !== 'running'}
>Run metadata pass now</button
>
<button
onclick={onReconcile}
disabled={busy || status.daemon !== 'running'}
title="Full list-only walk; enqueues mangas missing from the DB"
>Reconcile missing</button
>
<button onclick={onOpenRestart} disabled={busy || status.daemon !== 'running'}
>Restart browser</button
>

2
frontend/src/lib/components/crawler/CrawlerHero.svelte
View File

@@ -17,6 +17,8 @@
return `Fetching metadata · ${p.index}/${p.total ?? '?'} · ${p.title}`;
case 'cover_backfill':
return `Backfilling covers · ${p.index + 1}/${p.total}`;
case 'reconciling':
return `Reconciling · walked ${p.walked} · enqueued ${p.enqueued}`;
default: {
// Exhaustive default: if a new phase state ships
// without a label, TypeScript flags the missing case

15
frontend/src/lib/components/crawler/CrawlerHistoryTable.svelte
View File

@@ -71,7 +71,8 @@
await load();
}
/** A human "Manga · Ch N · pP" target, or the source key / em-dash. */
/** A human "Manga · Ch N · pP" target. Falls back to the payload title
* (sync_manga jobs with no manga row yet), then the source key. */
function target(r: CrawlerHistoryRow): string {
if (r.manga_title) {
let s = r.manga_title;
@@ -79,7 +80,7 @@
if (r.page_number != null) s += ` · p${r.page_number}`;
return s;
}
return r.source_key ?? '—';
return r.payload_title ?? r.source_key ?? '—';
}
function fmtAgo(iso: string): string {
@@ -172,7 +173,15 @@
<span class="badge state-{r.state}">{r.state}</span>
</td>
<td class="kind">{r.kind ?? '—'}</td>
<td>{target(r)}</td>
<td>
{#if !r.manga_title && r.payload_title && r.source_url}
<a href={r.source_url} target="_blank" rel="noreferrer noopener"
>{target(r)}</a
>
{:else}
{target(r)}
{/if}
</td>
<td class="num">{fmtDuration(r.duration_ms)}</td>
<td>{r.attempts}/{r.max_attempts}</td>
<td title={new Date(r.updated_at).toLocaleString()}

14
frontend/src/lib/components/crawler/DeadJobsTable.svelte
View File

@@ -56,7 +56,19 @@
{#each jobs as j (j.id)}
<tr>
<td>
{j.manga_title ?? '(unknown)'}
{#if j.manga_title}
{j.manga_title}
{:else if j.payload_title}
{#if j.source_url}
<a href={j.source_url} target="_blank" rel="noreferrer noopener"
>{j.payload_title}</a
>
{:else}
{j.payload_title}
{/if}
{:else}
{j.source_key ?? '(unknown)'}
{/if}
{#if j.chapter_number != null}· ch.{j.chapter_number}{/if}
</td>
<td>{j.attempts}/{j.max_attempts}</td>

69
frontend/src/lib/components/crawler/DeadJobsTable.svelte.test.ts Normal file
View File

@@ -0,0 +1,69 @@
import { describe, it, expect, afterEach } from 'vitest';
import { render, screen, cleanup } from '@testing-library/svelte';
import DeadJobsTable from './DeadJobsTable.svelte';
import type { DeadJob } from '$lib/api/admin';
afterEach(() => cleanup());
function deadJob(overrides: Partial<DeadJob>): DeadJob {
return {
id: 'job-1',
kind: 'sync_chapter_content',
chapter_id: null,
manga_id: null,
manga_title: null,
chapter_number: null,
payload_title: null,
source_url: null,
source_key: null,
attempts: 5,
max_attempts: 5,
last_error: 'boom',
updated_at: '2026-06-16T00:00:00Z',
...overrides
};
}
function baseProps(jobs: DeadJob[]) {
return {
jobs,
total: jobs.length,
page: 1,
totalPages: 1,
search: '',
busy: false,
onSearch: () => {},
onPageChange: () => {},
onRequeue: () => {},
onRequeueAll: () => {}
};
}
describe('DeadJobsTable', () => {
it('prefers manga_title when present', () => {
const jobs = [deadJob({ manga_title: 'Naruto', chapter_number: 700 })];
render(DeadJobsTable, { props: baseProps(jobs) });
expect(screen.getByText(/Naruto/)).toBeTruthy();
});
it('falls back to payload_title (linked to source_url) for a sync_manga job with no manga row', () => {
const jobs = [
deadJob({
kind: 'sync_manga',
manga_title: null,
payload_title: 'Ghost Manga',
source_url: 'http://x/manga/gone-1',
source_key: 'gone-1'
})
];
render(DeadJobsTable, { props: baseProps(jobs) });
const link = screen.getByRole('link', { name: 'Ghost Manga' }) as HTMLAnchorElement;
expect(link.getAttribute('href')).toBe('http://x/manga/gone-1');
});
it('falls back to source_key when there is no title at all', () => {
const jobs = [deadJob({ kind: 'sync_manga', source_key: 'only-key' })];
render(DeadJobsTable, { props: baseProps(jobs) });
expect(screen.getByText('only-key')).toBeTruthy();
});
});

9
frontend/src/routes/admin/crawler/+page.svelte
View File

@@ -16,6 +16,7 @@
getCrawlerStatus,
crawlerStatusStreamUrl,
runCrawlerPass,
reconcileCrawler,
restartCrawlerBrowser,
updateCrawlerSession,
clearCrawlerSessionExpired,
@@ -308,6 +309,13 @@
});
}
async function onReconcile() {
await withBusy('reconcile', async () => {
await reconcileCrawler();
notice = 'Reconcile started — walking the full list for missing mangas.';
});
}
async function onConfirmRestart() {
restartModalOpen = false;
await withBusy('restart browser', async () => {
@@ -427,6 +435,7 @@
{status}
{busy}
{onRunPass}
{onReconcile}
onOpenRestart={() => (restartModalOpen = true)}
onOpenSession={() => {
sessionModalOpen = true;