5fa4442904
Crawler + analysis admin dashboards gain a Live / History / Metrics segmented view. History: searchable, filterable, paginated job log per subsystem (crawler_jobs across all states/kinds; page_analysis terminal outcomes), with inline dead-job requeue and a Duration column. Live: enrich analysis SSE events with manga title + chapter number and add a sticky "Now analyzing" banner that jumps to and highlights the page. Metrics: new durable crawl_metrics table (migration 0028) timing every crawl op (manga list walk, manga detail, cover, whole chapter; per-page derived from chapter) plus page_analysis.duration_ms for analysis. New endpoints serve per-type average durations + success rates and a recent-ops log; a cron reaper (CRAWL_METRICS_RETENTION_DAYS) bounds growth. Tested: repo + API integration tests, vitest for the API client and fmtDuration, and Playwright for the History/Metrics tabs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
221 lines
8.2 KiB
TypeScript
221 lines
8.2 KiB
TypeScript
import { test, expect, type Page } from '@playwright/test';
|
|
|
|
// E2E for the admin Crawler "History" tab: the Live/History toggle, the
|
|
// searchable/filterable job log, and inline requeue of a dead job. The
|
|
// live status + SSE stream are left pending so the test focuses on history
|
|
// (the live dashboard is exercised by its own status mocks elsewhere).
|
|
|
|
const DESKTOP = { width: 1280, height: 720 } as const;
|
|
|
|
const deadJobId = 'd1111111-1111-1111-1111-111111111111';
|
|
const doneJobId = 'd2222222-2222-2222-2222-222222222222';
|
|
|
|
const adminUser = {
|
|
id: 'u11111111-1111-1111-1111-111111111111',
|
|
username: 'admin',
|
|
created_at: '2026-01-01T00:00:00Z',
|
|
is_admin: true
|
|
};
|
|
|
|
type Captured = { requeue: Record<string, unknown> | null };
|
|
|
|
async function mockAdmin(page: Page, cap: Captured) {
|
|
await page.route('**/api/v1/auth/config', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
|
})
|
|
);
|
|
await page.route('**/api/v1/auth/me', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ user: adminUser })
|
|
})
|
|
);
|
|
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
|
|
})
|
|
);
|
|
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
|
})
|
|
);
|
|
await page.route('**/api/v1/admin/system', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
disk: null,
|
|
memory: { total_bytes: 1, used_bytes: 0, percent_used: 0 },
|
|
cpu: { percent_used: 0 },
|
|
alerts: []
|
|
})
|
|
})
|
|
);
|
|
|
|
// Leave the live status fetch + SSE stream pending so the Live view
|
|
// stays in its "Loading…" state and doesn't churn; History is what we test.
|
|
await page.route('**/api/v1/admin/crawler', () => new Promise(() => {}));
|
|
await page.route('**/api/v1/admin/crawler/stream', () => new Promise(() => {}));
|
|
await page.route('**/api/v1/admin/crawler/dead-jobs**', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ items: [], page: { limit: 20, offset: 0, total: 0 } })
|
|
})
|
|
);
|
|
|
|
await page.route('**/api/v1/admin/crawler/history**', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
items: [
|
|
{
|
|
id: deadJobId,
|
|
state: 'dead',
|
|
kind: 'sync_chapter_content',
|
|
manga_id: 'm-1',
|
|
manga_title: 'Naruto',
|
|
chapter_id: 'c-1',
|
|
chapter_number: 700,
|
|
page_number: null,
|
|
source_key: null,
|
|
attempts: 5,
|
|
max_attempts: 5,
|
|
last_error: 'boom: upstream 500',
|
|
updated_at: '2026-06-15T00:00:00Z'
|
|
},
|
|
{
|
|
id: doneJobId,
|
|
state: 'done',
|
|
kind: 'analyze_page',
|
|
manga_id: 'm-2',
|
|
manga_title: 'Bleach',
|
|
chapter_id: 'c-2',
|
|
chapter_number: 3,
|
|
page_number: 7,
|
|
source_key: null,
|
|
attempts: 1,
|
|
max_attempts: 5,
|
|
last_error: null,
|
|
updated_at: '2026-06-15T00:00:00Z'
|
|
}
|
|
],
|
|
page: { limit: 25, offset: 0, total: 2 }
|
|
})
|
|
})
|
|
);
|
|
|
|
// Summary registered first (broad glob); the more-specific /ops route is
|
|
// registered after so it wins for the ops URL (Playwright: last match wins).
|
|
await page.route('**/api/v1/admin/crawler/metrics**', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
summary: [
|
|
{ op: 'manga_list', avg_ms: 42000, n: 8, ok: 8, failed: 0, avg_items: 50 },
|
|
{ op: 'manga_detail', avg_ms: 1300, n: 210, ok: 205, failed: 5, avg_items: null },
|
|
{ op: 'manga_cover', avg_ms: 480, n: 180, ok: 176, failed: 4, avg_items: null },
|
|
{ op: 'chapter', avg_ms: 6800, n: 430, ok: 421, failed: 9, avg_items: 32 }
|
|
]
|
|
})
|
|
})
|
|
);
|
|
await page.route('**/api/v1/admin/crawler/metrics/ops**', (r) =>
|
|
r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
items: [
|
|
{
|
|
id: 'op-1',
|
|
op: 'chapter',
|
|
manga_id: 'm-1',
|
|
manga_title: 'Berserk',
|
|
chapter_id: 'c-1',
|
|
chapter_number: 12,
|
|
outcome: 'ok',
|
|
duration_ms: 6100,
|
|
items: 20,
|
|
error: null,
|
|
finished_at: '2026-06-15T00:00:00Z'
|
|
}
|
|
],
|
|
page: { limit: 25, offset: 0, total: 1 }
|
|
})
|
|
})
|
|
);
|
|
|
|
await page.route('**/api/v1/admin/crawler/dead-jobs/requeue', (r) => {
|
|
cap.requeue = JSON.parse(r.request().postData() ?? '{}');
|
|
return r.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ requeued: 1 })
|
|
});
|
|
});
|
|
}
|
|
|
|
test.describe('/admin/crawler history', () => {
|
|
test('toggle to History lists jobs and requeues a dead job inline', async ({
|
|
page
|
|
}) => {
|
|
const cap: Captured = { requeue: null };
|
|
await mockAdmin(page, cap);
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.goto('/admin/crawler');
|
|
|
|
await page.getByTestId('crawler-tab-history').click();
|
|
|
|
const deadRow = page.getByTestId(`crawler-history-row-${deadJobId}`);
|
|
await expect(deadRow).toContainText('Naruto · Ch 700');
|
|
await expect(deadRow).toContainText('dead');
|
|
|
|
const doneRow = page.getByTestId(`crawler-history-row-${doneJobId}`);
|
|
await expect(doneRow).toContainText('Bleach · Ch 3 · p7');
|
|
|
|
// Click the dead row to expand its error detail.
|
|
await deadRow.click();
|
|
await expect(
|
|
page.getByTestId('crawler-history').locator('.errrow')
|
|
).toContainText('boom: upstream 500');
|
|
|
|
// Inline requeue posts scope=job.
|
|
await page.getByTestId(`crawler-history-requeue-${deadJobId}`).click();
|
|
await expect.poll(() => cap.requeue).toEqual({ scope: 'job', job_id: deadJobId });
|
|
});
|
|
|
|
test('Metrics tab shows averages by type, derived per-page, and the ops log', async ({
|
|
page
|
|
}) => {
|
|
const cap: Captured = { requeue: null };
|
|
await mockAdmin(page, cap);
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.goto('/admin/crawler');
|
|
|
|
await page.getByTestId('crawler-tab-metrics').click();
|
|
|
|
// Per-type averages.
|
|
await expect(page.getByTestId('crawler-metrics-row-chapter')).toContainText('6.8s');
|
|
await expect(page.getByTestId('crawler-metrics-row-manga_cover')).toContainText(
|
|
'480ms'
|
|
);
|
|
// Derived per-page row: 6800ms ÷ 32 ≈ 213ms.
|
|
await expect(page.getByTestId('crawler-metrics-perpage')).toContainText('213ms');
|
|
|
|
// Recent-ops log shows individual durations.
|
|
await expect(page.getByTestId('crawler-op-op-1')).toContainText('6.1s');
|
|
await expect(page.getByTestId('crawler-op-op-1')).toContainText('Berserk · Ch 12');
|
|
});
|
|
});
|