diff --git a/frontend/src/lib/api/admin.test.ts b/frontend/src/lib/api/admin.test.ts index 01db5ec..a2fc06c 100644 --- a/frontend/src/lib/api/admin.test.ts +++ b/frontend/src/lib/api/admin.test.ts @@ -16,7 +16,14 @@ import { listAdminChapters, getSystemStats, resyncManga, - resyncChapter + resyncChapter, + getCrawlerStatus, + runCrawlerPass, + restartCrawlerBrowser, + updateCrawlerSession, + clearCrawlerSessionExpired, + listDeadJobs, + requeueDeadJobs } from './admin'; function ok(body: unknown, status = 200): Response { @@ -329,3 +336,87 @@ describe('admin api client', () => { expect(got.pages).toBeNull(); }); }); + +describe('admin crawler api client', () => { + let fetchSpy: MockInstance; + beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, 'fetch'); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + const statusFixture = { + daemon: 'running', + phase: { state: 'fetching_metadata', index: 3, total: 10, title: 'One Piece' }, + workers: [{ state: 'idle' }], + last_pass: { at: null, discovered: 0, upserted: 0, covers_fetched: 0, mangas_failed: 0 }, + session: { expired: false, configured: true }, + browser: 'healthy', + queue: { pending: 2, running: 1, dead: 4 } + }; + + it('getCrawlerStatus GETs /v1/admin/crawler', async () => { + fetchSpy.mockResolvedValueOnce(ok(statusFixture)); + const s = await getCrawlerStatus(); + expect(s.queue.dead).toBe(4); + expect(s.phase?.state).toBe('fetching_metadata'); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toMatch(/\/v1\/admin\/crawler$/); + }); + + it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => { + fetchSpy.mockResolvedValueOnce(ok({ started: true })); + const r = await runCrawlerPass(); + expect(r.started).toBe(true); + const init = fetchSpy.mock.calls[0][1] as RequestInit; + expect(init.method).toBe('POST'); + expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/); + }); + + it('restartCrawlerBrowser POSTs the restart endpoint', async () => { + fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null })); + const r = await restartCrawlerBrowser(); + expect(r.ok).toBe(true); + expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/browser\/restart$/); + }); + + it('updateCrawlerSession POSTs the phpsessid body', async () => { + fetchSpy.mockResolvedValueOnce(ok({ valid: true, error: null })); + const r = await updateCrawlerSession('abc123'); + expect(r.valid).toBe(true); + const init = fetchSpy.mock.calls[0][1] as RequestInit; + expect(init.method).toBe('POST'); + expect(JSON.parse(init.body as string)).toEqual({ phpsessid: 'abc123' }); + }); + + it('clearCrawlerSessionExpired POSTs clear-expired', async () => { + fetchSpy.mockResolvedValueOnce(ok({ cleared: true })); + const r = await clearCrawlerSessionExpired(); + expect(r.cleared).toBe(true); + expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/session\/clear-expired$/); + }); + + it('listDeadJobs forwards search + pagination', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ items: [], page: { limit: 20, offset: 20, total: 0 } }) + ); + await listDeadJobs({ search: 'naruto', limit: 20, offset: 20 }); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toContain('search=naruto'); + expect(url).toContain('offset=20'); + }); + + it('requeueDeadJobs POSTs the scope body', async () => { + fetchSpy.mockResolvedValueOnce(ok({ requeued: 3 })); + const r = await requeueDeadJobs({ scope: 'manga', manga_id: 'm-9' }); + expect(r.requeued).toBe(3); + const init = fetchSpy.mock.calls[0][1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ scope: 'manga', manga_id: 'm-9' }); + }); + + it('surfaces a 503 as ApiError', async () => { + fetchSpy.mockResolvedValueOnce(envelope(503, 'service_unavailable', 'disabled')); + await expect(runCrawlerPass()).rejects.toMatchObject({ status: 503 }); + }); +}); diff --git a/frontend/src/lib/api/admin.ts b/frontend/src/lib/api/admin.ts index 930342d..09a021a 100644 --- a/frontend/src/lib/api/admin.ts +++ b/frontend/src/lib/api/admin.ts @@ -214,3 +214,105 @@ export async function resyncChapter(id: string): Promise { method: 'POST' } ); } + +// ---- crawler observability + control --------------------------------------- + +/** Current daemon activity. Discriminated on `state`. */ +export type CrawlerPhase = + | { state: 'idle'; next_fire: string | null } + | { state: 'walking_list' } + | { state: 'fetching_metadata'; index: number; total: number | null; title: string } + | { state: 'cover_backfill' }; + +export type CrawlerWorker = { state: 'idle' } | { state: 'working'; chapter_id: string }; + +export type CrawlerLastPass = { + at: string | null; + discovered: number; + upserted: number; + covers_fetched: number; + mangas_failed: number; +}; + +export type CrawlerStatus = { + daemon: 'running' | 'disabled'; + phase: CrawlerPhase | null; + workers: CrawlerWorker[]; + last_pass: CrawlerLastPass; + session: { expired: boolean; configured: boolean }; + browser: 'healthy' | 'draining' | 'restarting' | 'down'; + queue: { pending: number; running: number; dead: number }; +}; + +export async function getCrawlerStatus(): Promise { + return request('/v1/admin/crawler'); +} + +/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */ +export async function runCrawlerPass(): Promise<{ started: boolean }> { + return request('/v1/admin/crawler/run', { method: 'POST' }); +} + +/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */ +export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> { + return request('/v1/admin/crawler/browser/restart', { method: 'POST' }); +} + +/** POST /v1/admin/crawler/session — refresh PHPSESSID and re-probe. */ +export async function updateCrawlerSession( + phpsessid: string +): Promise<{ valid: boolean; error: string | null }> { + return request('/v1/admin/crawler/session', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ phpsessid }) + }); +} + +/** POST /v1/admin/crawler/session/clear-expired — resume idled workers. */ +export async function clearCrawlerSessionExpired(): Promise<{ cleared: boolean }> { + return request('/v1/admin/crawler/session/clear-expired', { method: 'POST' }); +} + +export type DeadJob = { + id: string; + kind: string; + chapter_id: string | null; + manga_id: string | null; + manga_title: string | null; + chapter_number: number | null; + attempts: number; + max_attempts: number; + last_error: string | null; + updated_at: string; +}; + +export type DeadJobsPage = { items: DeadJob[]; page: Page }; + +export async function listDeadJobs(opts?: { + search?: string; + limit?: number; + offset?: number; +}): Promise { + const params = new URLSearchParams(); + if (opts?.search) params.set('search', opts.search); + if (opts?.limit != null) params.set('limit', String(opts.limit)); + if (opts?.offset != null) params.set('offset', String(opts.offset)); + const qs = params.toString(); + return request(`/v1/admin/crawler/dead-jobs${qs ? `?${qs}` : ''}`); +} + +/** Requeue scope: all dead jobs, one manga's, one chapter's, or a single job. */ +export type RequeueScope = + | { scope: 'all' } + | { scope: 'manga'; manga_id: string } + | { scope: 'chapter'; chapter_id: string } + | { scope: 'job'; job_id: string }; + +export async function requeueDeadJobs(scope: RequeueScope): Promise<{ requeued: number }> { + return request('/v1/admin/crawler/dead-jobs/requeue', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(scope) + }); +} diff --git a/frontend/src/routes/admin/+layout.svelte b/frontend/src/routes/admin/+layout.svelte index 58575da..1da9af1 100644 --- a/frontend/src/routes/admin/+layout.svelte +++ b/frontend/src/routes/admin/+layout.svelte @@ -6,6 +6,7 @@ { href: '/admin', label: 'Overview' }, { href: '/admin/users', label: 'Users' }, { href: '/admin/mangas', label: 'Mangas' }, + { href: '/admin/crawler', label: 'Crawler' }, { href: '/admin/system', label: 'System' } ]; diff --git a/frontend/src/routes/admin/crawler/+page.svelte b/frontend/src/routes/admin/crawler/+page.svelte new file mode 100644 index 0000000..5fb7c7c --- /dev/null +++ b/frontend/src/routes/admin/crawler/+page.svelte @@ -0,0 +1,599 @@ + + +

Crawler

+ +{#if error} + +{/if} +{#if notice} +

{notice}

+{/if} + +{#if status} + +
+
+ Daemon + {status.daemon} + Session {sessionPill(status).text} + Browser {browserPill(status).text} +
+ +

{phaseLabel(status.phase)}

+ {#if phasePercent(status.phase) !== null} + {@render Bar({ percent: phasePercent(status.phase) ?? 0 })} + {/if} + + {#if status.session.expired} +

+ ⚠ Chapter downloads paused — session expired. Metadata + list crawl continue. +

+ {/if} + +

+ Last pass: + {#if status.last_pass.at} + {new Date(status.last_pass.at).toLocaleString()} · + {status.last_pass.discovered} seen · {status.last_pass.upserted} upserted · + {status.last_pass.mangas_failed} failed + {:else} + — none yet this session + {/if} +

+
+ + +
+ + + + {#if status.session.expired} + + {/if} +
+ + +
+
+

Queue

+
+
Pending
+
{status.queue.pending}
+
Running
+
{status.queue.running}
+
Dead
+
{status.queue.dead}
+
+
+
+

Workers

+ {#if status.workers.length === 0} +

none

+ {:else} + + + {#each status.workers as w, i (i)} + + + + + + {/each} + +
#{i} + {w.state} + {w.state === 'working' ? w.chapter_id : '—'}
+ {/if} +
+
+{:else} +

Loading…

+{/if} + + +
+
+

Dead jobs ({deadTotal})

+
+ e.key === 'Enter' && onSearchDead()} + /> + + +
+
+ + {#if deadJobs.length === 0} +

No dead jobs 🎉

+ {:else} + + + + + + + + + + + + {#each deadJobs as j (j.id)} + + + + + + + + {/each} + +
Manga / ChapterAtt.FailedLast errorAction
+ {j.manga_title ?? '(unknown)'} + {#if j.chapter_number != null}· ch.{j.chapter_number}{/if} + {j.attempts}/{j.max_attempts}{new Date(j.updated_at).toLocaleDateString()}{j.last_error ?? '—'} + + {#if j.manga_id} + + {/if} +
+ + {/if} +
+ + + (restartModalOpen = false)} size="sm"> + {#snippet children()} +

This relaunches Chromium and re-injects the session cookie.

+
    +
  • In-flight jobs are allowed to finish (bounded), then forced.
  • +
  • New jobs pause until the relaunch completes.
  • +
  • The metadata pass yields at its next checkpoint.
  • +
+ {/snippet} + {#snippet footer()} + + + {/snippet} +
+ + + (sessionModalOpen = false)} size="md"> + {#snippet children()} + + +

+ Saving rewrites the cookie everywhere, persists it, restarts the browser, and re-probes. +

+ {#if sessionResult} +

{sessionResult}

+ {/if} + {/snippet} + {#snippet footer()} + + + {/snippet} +
+ +{#snippet Bar({ percent }: { percent: number })} +
+
+ {percent.toFixed(0)}% +
+{/snippet} + + diff --git a/frontend/src/routes/admin/mangas/+page.svelte b/frontend/src/routes/admin/mangas/+page.svelte index 3824585..a56fae2 100644 --- a/frontend/src/routes/admin/mangas/+page.svelte +++ b/frontend/src/routes/admin/mangas/+page.svelte @@ -3,6 +3,7 @@ import { listAdminMangas, listAdminChapters, + requeueDeadJobs, type AdminMangasPage, type AdminChapterRow, type MangaSyncState @@ -59,6 +60,27 @@ function badgeClass(state: string): string { return `badge badge-${state}`; } + + let requeuingChapter: string | null = $state(null); + + /** Requeue the dead job(s) for a single failed chapter, then refresh + * that manga's chapter list so the pill updates. */ + async function requeueChapter(mangaId: string, chapterId: string) { + requeuingChapter = chapterId; + error = null; + try { + await requeueDeadJobs({ scope: 'chapter', chapter_id: chapterId }); + const resp = await listAdminChapters(mangaId, { limit: 500 }); + chaptersByManga[mangaId] = { + items: resp.items, + total: resp.page.total ?? resp.items.length + }; + } catch (e) { + error = e instanceof ApiError ? e.message : 'requeue failed'; + } finally { + requeuingChapter = null; + } + }

Mangas

@@ -153,6 +175,16 @@ {c.sync_state} + {#if c.sync_state === 'failed'} + + {/if} {/each} @@ -272,6 +304,11 @@ color: #991b1b; border-color: #fca5a5; } + .requeue { + margin-left: var(--space-2); + font-size: var(--font-xs); + padding: 0 var(--space-2); + } .badge-not_downloaded { background: var(--surface-elevated); color: var(--text-muted);