Admin-only crawler dashboard backed by an SSE live-status stream,
coordinated browser restart, runtime PHPSESSID refresh, dead-letter
requeue, and a batch of reliability fixes. Closes everything from
the two-pass audit (10 commits' worth) and bumps 0.52.0 -> 0.55.0.
Backend:
- New /admin/crawler/* surface (cookie-auth, RequireAdmin) split
into status / control / dead_jobs / backlog modules. SSE stream
composes in-memory status with DB-derived queue counts, memoizes
the counts for 1s and debounces watch pokes for 250ms (~10x QPS
reduction per subscriber). One-shot GET /admin/crawler shares the
same compose path.
- POST /admin/crawler/run gated by manual_pass_lock try_lock_owned
(409 Conflict on overlapping click); browser restart goes through
the coordinated_restart gate (drain + relaunch + auto-clear of the
sticky session_expired flag on Ok).
- Runtime PHPSESSID refresh via SessionController (allow-list
validation, never logged, audit row carries SHA-256 fingerprint).
Storage layer is repo::crawler::runtime_session_{load,persist}.
- Dead-letter requeue with four scopes (all/manga/chapter/job);
scope=all requires confirm:true; DISTINCT ON dedup keeps the
partial unique index from rejecting requeues for chapters with
multiple dead rows. SQL is four &'static str constants per scope.
- StatusHandle + ChapterGuard / CoverGuard RAII model survives
panics; last-writer-wins on cover so concurrent dispatches don't
clobber each other's slot. Pure functions (should_stop /
should_mark_clean_exit / should_abort_pass) with named regression
tests.
- Reliability bundle: per-lease heartbeat, jitter on retries,
per-job timeout, circuit breaker on consecutive failures, BrowserManager
coordinated restart gate, request fingerprint changes.
- Streaming page download: Storage::put_stream trait method,
LocalStorage impl atomic via temp + fsync + UUID-suffixed rename.
Pages stream through with peak memory ~one HTTP chunk + 64-byte
sniff prefix instead of one full image per dispatch.
- New partial indexes (migration 0022): mangas_missing_cover_idx
and crawler_jobs_dead_idx, both ordered by updated_at DESC to
match the dashboard's LIMIT/OFFSET reads.
- Security hardening: admin_csrf_guard (Origin/Referer allowlist
on /admin/* mutations, opt-in via ADMIN_ALLOWED_ORIGINS),
admin_no_store_guard (Cache-Control: no-store on admin
responses), audit rows carry per-scope target_id.
Frontend:
- /admin/crawler page decomposed into lib/components/crawler/
(11 components: ProgressBar, SearchBar, CrawlerHero,
CrawlerControls, ActiveChaptersCard, ActiveJobsTable,
MissingCoversTable, DeadJobsTable, RestartConfirmModal,
RequeueAllConfirmModal, SessionModal). Page is 532 LOC of
orchestration; each component 22-148 LOC.
- EventSource lifecycle wired to visibilitychange / pagehide /
pageshow (BFCache); after 5 consecutive errors probes the status
endpoint so a 401 routes through the global on401Hook instead of
infinite silent reconnects.
- Backlog $effect refetches debounced 500ms with per-loader
AbortControllers; refresh after a control action only runs when
the SSE stream is dead.
- Inline requeue button on /admin/mangas patches the affected row's
sync_state locally (no full chapter-list refetch); proper
aria-label. Requeue-all gets its own confirm modal; both confirm
modals autofocus Cancel.
- SvelteKit reverse proxy bypasses its 5-minute AbortController
for Accept: text/event-stream; pure shouldBypassProxyTimeout
helper covered by unit tests.
Config / docs:
- New env vars (.env.example): ADMIN_ALLOWED_ORIGINS,
CRAWLER_JOB_TIMEOUT_SECS, CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES,
CRAWLER_BROWSER_RESTART_THRESHOLD.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
406 lines
12 KiB
TypeScript
406 lines
12 KiB
TypeScript
// Admin-only API client. Every endpoint here is guarded by
|
|
// RequireAdmin on the backend (session cookie only — bearer tokens
|
|
// won't reach these routes). 403s thrown here propagate up to the
|
|
// /admin layout, which renders the framework error page.
|
|
|
|
import { request, apiUrl, type Page } from './client';
|
|
import type { User } from './auth';
|
|
import type { MangaDetail } from './mangas';
|
|
import type { Chapter } from './chapters';
|
|
|
|
// ---- users -----------------------------------------------------------------
|
|
|
|
export type AdminUsersPage = {
|
|
items: User[];
|
|
page: Page;
|
|
};
|
|
|
|
export type ListAdminUsersOptions = {
|
|
search?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
};
|
|
|
|
export async function listAdminUsers(
|
|
opts: ListAdminUsersOptions = {}
|
|
): Promise<AdminUsersPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts.search) params.set('search', opts.search);
|
|
if (opts.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<AdminUsersPage>(`/v1/admin/users${qs ? `?${qs}` : ''}`);
|
|
}
|
|
|
|
export async function deleteAdminUser(id: string): Promise<void> {
|
|
await request<void>(`/v1/admin/users/${encodeURIComponent(id)}`, {
|
|
method: 'DELETE'
|
|
});
|
|
}
|
|
|
|
export async function setUserAdmin(id: string, isAdmin: boolean): Promise<User> {
|
|
return request<User>(`/v1/admin/users/${encodeURIComponent(id)}`, {
|
|
method: 'PATCH',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify({ is_admin: isAdmin })
|
|
});
|
|
}
|
|
|
|
export type CreateAdminUserInput = {
|
|
username: string;
|
|
password: string;
|
|
is_admin?: boolean;
|
|
};
|
|
|
|
/** POST /v1/admin/users — admin-initiated account creation. Works
|
|
* regardless of the ALLOW_SELF_REGISTER toggle, since the entire
|
|
* point is for an admin to enroll someone when self-register is off. */
|
|
export async function createAdminUser(input: CreateAdminUserInput): Promise<User> {
|
|
return request<User>('/v1/admin/users', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify(input)
|
|
});
|
|
}
|
|
|
|
// ---- mangas / chapters with sync state -------------------------------------
|
|
|
|
export type MangaSyncState = 'in_progress' | 'dropped' | 'synced';
|
|
|
|
export type AdminMangaRow = {
|
|
id: string;
|
|
title: string;
|
|
status: string;
|
|
cover_image_path: string | null;
|
|
created_at: string;
|
|
updated_at: string;
|
|
sync_state: MangaSyncState;
|
|
chapter_count: number;
|
|
latest_seen_at: string | null;
|
|
};
|
|
|
|
export type AdminMangasPage = {
|
|
items: AdminMangaRow[];
|
|
page: Page;
|
|
};
|
|
|
|
export type ListAdminMangasOptions = {
|
|
search?: string;
|
|
syncState?: MangaSyncState;
|
|
limit?: number;
|
|
offset?: number;
|
|
};
|
|
|
|
export async function listAdminMangas(
|
|
opts: ListAdminMangasOptions = {}
|
|
): Promise<AdminMangasPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts.search) params.set('search', opts.search);
|
|
if (opts.syncState) params.set('sync_state', opts.syncState);
|
|
if (opts.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<AdminMangasPage>(`/v1/admin/mangas${qs ? `?${qs}` : ''}`);
|
|
}
|
|
|
|
export type ChapterSyncState =
|
|
| 'downloading'
|
|
| 'dropped'
|
|
| 'failed'
|
|
| 'not_downloaded'
|
|
| 'synced';
|
|
|
|
export type AdminChapterRow = {
|
|
id: string;
|
|
manga_id: string;
|
|
number: number;
|
|
title: string | null;
|
|
page_count: number;
|
|
created_at: string;
|
|
sync_state: ChapterSyncState;
|
|
latest_seen_at: string | null;
|
|
};
|
|
|
|
export type AdminChaptersPage = {
|
|
items: AdminChapterRow[];
|
|
page: Page;
|
|
};
|
|
|
|
export type ListAdminChaptersOptions = {
|
|
limit?: number;
|
|
offset?: number;
|
|
};
|
|
|
|
export async function listAdminChapters(
|
|
mangaId: string,
|
|
opts: ListAdminChaptersOptions = {}
|
|
): Promise<AdminChaptersPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<AdminChaptersPage>(
|
|
`/v1/admin/mangas/${encodeURIComponent(mangaId)}/chapters${qs ? `?${qs}` : ''}`
|
|
);
|
|
}
|
|
|
|
// ---- system ----------------------------------------------------------------
|
|
|
|
export type DiskStats = {
|
|
total_bytes: number;
|
|
used_bytes: number;
|
|
free_bytes: number;
|
|
percent_used: number;
|
|
};
|
|
|
|
export type MemoryStats = {
|
|
total_bytes: number;
|
|
used_bytes: number;
|
|
percent_used: number;
|
|
};
|
|
|
|
export type CpuStats = {
|
|
percent_used: number;
|
|
};
|
|
|
|
export type Alert = {
|
|
level: 'warning';
|
|
message: string;
|
|
};
|
|
|
|
export type SystemStats = {
|
|
disk: DiskStats | null;
|
|
memory: MemoryStats;
|
|
cpu: CpuStats;
|
|
alerts: Alert[];
|
|
};
|
|
|
|
export async function getSystemStats(): Promise<SystemStats> {
|
|
return request<SystemStats>('/v1/admin/system');
|
|
}
|
|
|
|
// ---- force resync ----------------------------------------------------------
|
|
|
|
export type MangaResyncResponse = {
|
|
manga: MangaDetail;
|
|
metadata_status: 'new' | 'updated' | 'unchanged';
|
|
cover_fetched: boolean;
|
|
};
|
|
|
|
export type ChapterResyncResponse = {
|
|
chapter: Chapter;
|
|
outcome: 'fetched' | 'skipped';
|
|
/** Page count when `outcome === 'fetched'`; null when skipped. */
|
|
pages: number | null;
|
|
};
|
|
|
|
/** POST /v1/admin/mangas/:id/resync — refetches metadata + cover from
|
|
* the manga's live crawler source. Long-running (one HTTP request per
|
|
* Chromium nav + image download), so the UI should disable the trigger
|
|
* and surface progress. */
|
|
export async function resyncManga(id: string): Promise<MangaResyncResponse> {
|
|
return request<MangaResyncResponse>(
|
|
`/v1/admin/mangas/${encodeURIComponent(id)}/resync`,
|
|
{ method: 'POST' }
|
|
);
|
|
}
|
|
|
|
/** POST /v1/admin/chapters/:id/resync — force-refetches a chapter's
|
|
* pages even if `page_count > 0`. Same long-running caveat as
|
|
* `resyncManga`. */
|
|
export async function resyncChapter(id: string): Promise<ChapterResyncResponse> {
|
|
return request<ChapterResyncResponse>(
|
|
`/v1/admin/chapters/${encodeURIComponent(id)}/resync`,
|
|
{ method: 'POST' }
|
|
);
|
|
}
|
|
|
|
// ---- crawler observability + control ---------------------------------------
|
|
|
|
/** Current daemon activity. Discriminated on `state`. */
|
|
export type CrawlerPhase =
|
|
| { state: 'idle'; next_fire: string | null }
|
|
| { state: 'walking_list' }
|
|
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
|
|
| { state: 'cover_backfill'; index: number; total: number };
|
|
|
|
/** A chapter being crawled right now, with a live page count. */
|
|
export type ActiveChapter = {
|
|
manga_id: string;
|
|
manga_title: string;
|
|
chapter_id: string;
|
|
chapter_number: number;
|
|
pages_done: number;
|
|
pages_total: number | null;
|
|
};
|
|
|
|
export type CrawlerLastPass = {
|
|
at: string | null;
|
|
discovered: number;
|
|
upserted: number;
|
|
covers_fetched: number;
|
|
mangas_failed: number;
|
|
};
|
|
|
|
export type CrawlerStatus = {
|
|
daemon: 'running' | 'disabled';
|
|
phase: CrawlerPhase | null;
|
|
worker_count: number;
|
|
active_chapters: ActiveChapter[];
|
|
current_cover: { manga_id: string; manga_title: string } | null;
|
|
covers_queued: number;
|
|
last_pass: CrawlerLastPass;
|
|
session: { expired: boolean; configured: boolean };
|
|
browser: 'healthy' | 'draining' | 'restarting' | 'down';
|
|
queue: { pending: number; running: number; dead: number };
|
|
};
|
|
|
|
export async function getCrawlerStatus(): Promise<CrawlerStatus> {
|
|
return request<CrawlerStatus>('/v1/admin/crawler');
|
|
}
|
|
|
|
/** URL of the Server-Sent Events live-status stream. Open with
|
|
* `new EventSource(...)` while the crawler page is mounted and close it on
|
|
* navigate-away so the subscription is scoped to the active page. Each
|
|
* message is a named `status` event whose `data` is a {@link CrawlerStatus}. */
|
|
export function crawlerStatusStreamUrl(): string {
|
|
return apiUrl('/v1/admin/crawler/stream');
|
|
}
|
|
|
|
/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */
|
|
export async function runCrawlerPass(): Promise<{ started: boolean }> {
|
|
return request('/v1/admin/crawler/run', { method: 'POST' });
|
|
}
|
|
|
|
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
|
|
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
|
|
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
|
|
}
|
|
|
|
/** POST /v1/admin/crawler/session — refresh PHPSESSID and re-probe. */
|
|
export async function updateCrawlerSession(
|
|
phpsessid: string
|
|
): Promise<{ valid: boolean; error: string | null }> {
|
|
return request('/v1/admin/crawler/session', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify({ phpsessid })
|
|
});
|
|
}
|
|
|
|
/** POST /v1/admin/crawler/session/clear-expired — resume idled workers. */
|
|
export async function clearCrawlerSessionExpired(): Promise<{ cleared: boolean }> {
|
|
return request('/v1/admin/crawler/session/clear-expired', { method: 'POST' });
|
|
}
|
|
|
|
export type DeadJob = {
|
|
id: string;
|
|
kind: string;
|
|
chapter_id: string | null;
|
|
manga_id: string | null;
|
|
manga_title: string | null;
|
|
chapter_number: number | null;
|
|
attempts: number;
|
|
max_attempts: number;
|
|
last_error: string | null;
|
|
updated_at: string;
|
|
};
|
|
|
|
export type DeadJobsPage = { items: DeadJob[]; page: Page };
|
|
|
|
export async function listDeadJobs(
|
|
opts?: {
|
|
search?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
},
|
|
init?: RequestInit
|
|
): Promise<DeadJobsPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts?.search) params.set('search', opts.search);
|
|
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<DeadJobsPage>(
|
|
`/v1/admin/crawler/dead-jobs${qs ? `?${qs}` : ''}`,
|
|
init
|
|
);
|
|
}
|
|
|
|
/** Requeue scope: all dead jobs, one manga's, one chapter's, or a single job.
|
|
* `scope: 'all'` requires an explicit `confirm: true` so a careless
|
|
* click (or CSRF bait) can't flip the entire dead pile. */
|
|
export type RequeueScope =
|
|
| { scope: 'all'; confirm: true }
|
|
| { scope: 'manga'; manga_id: string }
|
|
| { scope: 'chapter'; chapter_id: string }
|
|
| { scope: 'job'; job_id: string };
|
|
|
|
export async function requeueDeadJobs(scope: RequeueScope): Promise<{ requeued: number }> {
|
|
return request('/v1/admin/crawler/dead-jobs/requeue', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify(scope)
|
|
});
|
|
}
|
|
|
|
/** A queued/running chapter-content job (which chapters are queued). */
|
|
export type ActiveJob = {
|
|
id: string;
|
|
chapter_id: string | null;
|
|
manga_id: string | null;
|
|
manga_title: string | null;
|
|
chapter_number: number | null;
|
|
state: 'pending' | 'running';
|
|
attempts: number;
|
|
max_attempts: number;
|
|
updated_at: string;
|
|
};
|
|
|
|
export type ActiveJobsPage = { items: ActiveJob[]; page: Page };
|
|
|
|
/** GET /v1/admin/crawler/active-jobs — which chapters of which mangas are
|
|
* queued or running now. */
|
|
export async function listActiveJobs(
|
|
opts?: {
|
|
search?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
},
|
|
init?: RequestInit
|
|
): Promise<ActiveJobsPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts?.search) params.set('search', opts.search);
|
|
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<ActiveJobsPage>(
|
|
`/v1/admin/crawler/active-jobs${qs ? `?${qs}` : ''}`,
|
|
init
|
|
);
|
|
}
|
|
|
|
/** A manga queued for a cover fetch (no cover yet + a live source). */
|
|
export type MissingCover = { manga_id: string; manga_title: string };
|
|
export type MissingCoversPage = { items: MissingCover[]; page: Page };
|
|
|
|
/** GET /v1/admin/crawler/covers — which manga covers are queued. */
|
|
export async function listMissingCovers(
|
|
opts?: {
|
|
search?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
},
|
|
init?: RequestInit
|
|
): Promise<MissingCoversPage> {
|
|
const params = new URLSearchParams();
|
|
if (opts?.search) params.set('search', opts.search);
|
|
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
|
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
|
const qs = params.toString();
|
|
return request<MissingCoversPage>(
|
|
`/v1/admin/crawler/covers${qs ? `?${qs}` : ''}`,
|
|
init
|
|
);
|
|
}
|