feat(crawler): live cover + chapter-content observability with realtime page counts
Extends the live dashboard so an operator can see exactly what's being fetched, in realtime: - Chapters being crawled now are tracked in the status as `active_chapters` (manga title · ch.N) with a live page counter that climbs per stored page (set_chapter_pages, pushed via the existing watch→SSE). The dispatcher registers each via an RAII ChapterGuard (sync Mutex) that removes the entry on completion, panic, or timeout-drop — replacing the old per-worker slot model. - Covers: status now carries the cover being fetched now (`current_cover`, set around download_and_store_cover in both the metadata pass and backfill) and a `covers_queued` backlog count; CoverBackfill phase gains index/total. - Two paginated backlog endpoints (fetched on demand, auto-refreshed when the live counts change): GET /admin/crawler/active-jobs (which chapters of which mangas are queued/running) and GET /admin/crawler/covers (mangas missing a cover). repo: list_active_jobs, list_missing_cover_mangas, count_missing_covers. - dispatch_target now also returns manga title + chapter number. Frontend: the crawler page replaces the Workers table with an Active-chapters table (live page bars), adds a current-cover line + covers-queued figure, and two backlog sections (Queued chapters / Queued covers) with search + Pager, auto-refetched via $effect on the live counts. Tests: status guard/page + cover unit tests; repo list/count tests; endpoint tests; frontend api tests. Version 0.53.1 -> 0.54.0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -24,7 +24,9 @@ import {
|
||||
updateCrawlerSession,
|
||||
clearCrawlerSessionExpired,
|
||||
listDeadJobs,
|
||||
requeueDeadJobs
|
||||
requeueDeadJobs,
|
||||
listActiveJobs,
|
||||
listMissingCovers
|
||||
} from './admin';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -350,7 +352,19 @@ describe('admin crawler api client', () => {
|
||||
const statusFixture = {
|
||||
daemon: 'running',
|
||||
phase: { state: 'fetching_metadata', index: 3, total: 10, title: 'One Piece' },
|
||||
workers: [{ state: 'idle' }],
|
||||
worker_count: 2,
|
||||
active_chapters: [
|
||||
{
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Bleach',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 12,
|
||||
pages_done: 4,
|
||||
pages_total: 20
|
||||
}
|
||||
],
|
||||
current_cover: { manga_id: 'm-2', manga_title: 'Naruto' },
|
||||
covers_queued: 7,
|
||||
last_pass: { at: null, discovered: 0, upserted: 0, covers_fetched: 0, mangas_failed: 0 },
|
||||
session: { expired: false, configured: true },
|
||||
browser: 'healthy',
|
||||
@@ -361,15 +375,38 @@ describe('admin crawler api client', () => {
|
||||
expect(crawlerStatusStreamUrl()).toMatch(/\/v1\/admin\/crawler\/stream$/);
|
||||
});
|
||||
|
||||
it('getCrawlerStatus GETs /v1/admin/crawler', async () => {
|
||||
it('getCrawlerStatus GETs /v1/admin/crawler with live chapter/cover fields', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(statusFixture));
|
||||
const s = await getCrawlerStatus();
|
||||
expect(s.queue.dead).toBe(4);
|
||||
expect(s.phase?.state).toBe('fetching_metadata');
|
||||
expect(s.active_chapters[0].pages_done).toBe(4);
|
||||
expect(s.active_chapters[0].pages_total).toBe(20);
|
||||
expect(s.current_cover?.manga_title).toBe('Naruto');
|
||||
expect(s.covers_queued).toBe(7);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler$/);
|
||||
});
|
||||
|
||||
it('listActiveJobs GETs /v1/admin/crawler/active-jobs with search', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 20, offset: 0, total: 0 } })
|
||||
);
|
||||
await listActiveJobs({ search: 'bleach' });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler\/active-jobs\?/);
|
||||
expect(url).toContain('search=bleach');
|
||||
});
|
||||
|
||||
it('listMissingCovers GETs /v1/admin/crawler/covers', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [{ manga_id: 'm-1', manga_title: 'X' }], page: { limit: 20, offset: 0, total: 1 } })
|
||||
);
|
||||
const r = await listMissingCovers();
|
||||
expect(r.items[0].manga_title).toBe('X');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
|
||||
});
|
||||
|
||||
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||
const r = await runCrawlerPass();
|
||||
|
||||
Reference in New Issue
Block a user