feat(admin): operational health checks page with editable thresholds

New /admin/health rolls up signals already collected — disk/memory sensors,
dead-job + missing-cover backlogs, 24h crawl/analysis failure rates, metadata
cron freshness, and the live crawler session/browser flags — into a flat
checks list (ok/warn/critical) with an overall status and remediation links.
The overview gains a compact Health summary banner linking in.

GET /v1/admin/health evaluates the checks (DB sub-queries run concurrently via
try_join; all hit existing indexes). PUT /v1/admin/health/thresholds persists
the thresholds to app_settings (merged over compiled defaults, forward-compat
via serde(default)), validates ranges (400 on bad input), and audit-logs the
change in one transaction. New repo::crawler::last_metadata_tick_at getter and
a lightweight system::memory_percent_used (no CPU settle delay).

Pure status helpers (over_limit for gauges, exceeds for backlog counts so a
0-limit doesn't false-alarm at 0) are unit-tested; endpoint tests cover shape,
gating, persistence+audit, and validation.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-19 11:38:18 +02:00
parent 31013cc893
commit 314fc8738b
12 changed files with 1265 additions and 2 deletions

View File

@@ -43,7 +43,9 @@ import {
getCrawlerMetrics,
listCrawlerOps,
getAnalysisMetrics,
listAuditLog
listAuditLog,
getHealth,
updateHealthThresholds
} from './admin';
function ok(body: unknown, status = 200): Response {
@@ -506,6 +508,48 @@ describe('admin crawler api client', () => {
expect(url).toContain('days=7');
});
const healthFixture = {
status: 'ok' as const,
checks: [
{
id: 'memory',
label: 'Memory usage',
status: 'ok' as const,
value: '48%',
threshold: '90%',
hint: null,
action_href: null
}
],
thresholds: {
disk_pct: 90,
mem_pct: 90,
dead_jobs_max: 100,
crawl_fail_pct: 20,
analysis_fail_pct: 20,
cron_freshness_hours: 26,
missing_covers_max: 500
}
};
it('getHealth GETs /v1/admin/health', async () => {
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
const r = await getHealth();
expect(r.status).toBe('ok');
expect(r.checks[0].id).toBe('memory');
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/health$/);
});
it('updateHealthThresholds PUTs the thresholds body', async () => {
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
await updateHealthThresholds(healthFixture.thresholds);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/health\/thresholds$/);
const init = fetchSpy.mock.calls[0][1] as RequestInit;
expect(init.method).toBe('PUT');
expect(JSON.parse(init.body as string).dead_jobs_max).toBe(100);
});
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
const r = await runCrawlerPass();

View File

@@ -1018,3 +1018,46 @@ export async function listAuditLog(
const qs = params.toString();
return request<AuditPage>(`/v1/admin/audit${qs ? `?${qs}` : ''}`, init);
}
// ---- operational health checks ---------------------------------------------
export type CheckStatus = 'ok' | 'warn' | 'critical';
export type HealthCheck = {
id: string;
label: string;
status: CheckStatus;
value: string;
threshold: string | null;
hint: string | null;
/** Admin route that remediates this check, when one applies. */
action_href: string | null;
};
export type HealthThresholds = {
disk_pct: number;
mem_pct: number;
dead_jobs_max: number;
crawl_fail_pct: number;
analysis_fail_pct: number;
cron_freshness_hours: number;
missing_covers_max: number;
};
export type HealthReport = {
status: CheckStatus;
checks: HealthCheck[];
thresholds: HealthThresholds;
};
export async function getHealth(init?: RequestInit): Promise<HealthReport> {
return request<HealthReport>('/v1/admin/health', init);
}
export async function updateHealthThresholds(t: HealthThresholds): Promise<HealthReport> {
return request<HealthReport>('/v1/admin/health/thresholds', {
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(t)
});
}

View File

@@ -0,0 +1,306 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
import {
getHealth,
updateHealthThresholds,
type HealthReport,
type HealthThresholds,
type CheckStatus
} from '$lib/api/admin';
const POLL_MS = 30_000;
let report = $state<HealthReport | null>(null);
let loading = $state(true);
let error = $state<string | null>(null);
let timer: ReturnType<typeof setInterval> | null = null;
let inflight: AbortController | null = null;
// Threshold editor state.
let editing = $state(false);
let form = $state<HealthThresholds | null>(null);
let saving = $state(false);
let saveError = $state<string | null>(null);
let saved = $state(false);
async function load() {
inflight?.abort();
const ctrl = new AbortController();
inflight = ctrl;
try {
const r = await getHealth({ signal: ctrl.signal });
report = r;
error = null;
} catch (e) {
if ((e as Error)?.name === 'AbortError') return;
error = e instanceof Error ? e.message : 'Failed to load health.';
} finally {
if (inflight === ctrl) loading = false;
}
}
onMount(() => {
load();
timer = setInterval(load, POLL_MS);
});
onDestroy(() => {
if (timer) clearInterval(timer);
inflight?.abort();
});
function startEdit() {
// Snapshot current thresholds into an editable copy.
form = report ? { ...report.thresholds } : null;
saveError = null;
saved = false;
editing = true;
}
async function save() {
if (!form) return;
saving = true;
saveError = null;
saved = false;
try {
report = await updateHealthThresholds(form);
saved = true;
editing = false;
} catch (e) {
saveError = e instanceof Error ? e.message : 'Failed to save thresholds.';
} finally {
saving = false;
}
}
const STATUS_GLYPH: Record<CheckStatus, string> = {
ok: '✓',
warn: '⚠',
critical: '✗'
};
const STATUS_LABEL: Record<CheckStatus, string> = {
ok: 'Healthy',
warn: 'Needs attention',
critical: 'Critical'
};
const failing = $derived(
report ? report.checks.filter((c) => c.status !== 'ok').length : 0
);
// Editable threshold fields, with units for the form labels.
const FIELDS: { key: keyof HealthThresholds; label: string; unit: string }[] = [
{ key: 'disk_pct', label: 'Disk usage', unit: '%' },
{ key: 'mem_pct', label: 'Memory usage', unit: '%' },
{ key: 'dead_jobs_max', label: 'Dead jobs (max)', unit: '' },
{ key: 'crawl_fail_pct', label: 'Crawl failure rate', unit: '%' },
{ key: 'analysis_fail_pct', label: 'Analysis failure rate', unit: '%' },
{ key: 'cron_freshness_hours', label: 'Cron freshness', unit: 'h' },
{ key: 'missing_covers_max', label: 'Missing covers (max)', unit: '' }
];
</script>
<section class="health" data-testid="health-checks">
{#if error}
<p class="error" role="alert">{error}</p>
{/if}
{#if loading && !report}
<p class="muted" data-testid="health-loading">Loading…</p>
{:else if report}
<header class="overall status-{report.status}" data-testid="health-overall">
<span class="glyph">{STATUS_GLYPH[report.status]}</span>
<span class="overall-label">{STATUS_LABEL[report.status]}</span>
{#if failing > 0}
<span class="muted">· {failing} check{failing === 1 ? '' : 's'} need attention</span>
{/if}
</header>
<ul class="checks">
{#each report.checks as c (c.id)}
<li class="check status-{c.status}" data-testid={`health-check-${c.id}`}>
<span class="glyph" aria-hidden="true">{STATUS_GLYPH[c.status]}</span>
<span class="label">{c.label}</span>
<span class="value">
{c.value}{#if c.threshold}<span class="muted"> / {c.threshold}</span>{/if}
</span>
<span class="meta">
{#if c.hint}<span class="hint muted">{c.hint}</span>{/if}
{#if c.action_href && c.status !== 'ok'}
<a class="fix" href={c.action_href}>Fix →</a>
{/if}
</span>
</li>
{/each}
</ul>
<div class="thresholds">
<button
type="button"
class="disclosure"
onclick={() => (editing ? (editing = false) : startEdit())}
aria-expanded={editing}
data-testid="health-thresholds-toggle"
>
{editing ? '▾' : '▸'} Configure thresholds
</button>
{#if saved && !editing}<span class="ok-note">Saved ✓</span>{/if}
{#if editing && form}
<form
class="threshold-form"
onsubmit={(e) => {
e.preventDefault();
save();
}}
>
{#each FIELDS as f (f.key)}
<label>
<span>{f.label}{f.unit ? ` (${f.unit})` : ''}</span>
<input
type="number"
step={f.unit === '%' ? '1' : '1'}
min="0"
bind:value={form[f.key]}
data-testid={`threshold-${f.key}`}
/>
</label>
{/each}
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
<div class="actions">
<button type="submit" disabled={saving} data-testid="threshold-save">
{saving ? 'Saving…' : 'Save'}
</button>
<button type="button" class="secondary" onclick={() => (editing = false)}>
Cancel
</button>
</div>
</form>
{/if}
</div>
{/if}
</section>
<style>
.overall {
display: flex;
align-items: center;
gap: var(--space-2);
font-size: var(--font-md);
font-weight: var(--weight-semibold);
margin-bottom: var(--space-3);
}
.overall .glyph {
font-size: var(--font-lg);
}
.checks {
list-style: none;
margin: 0 0 var(--space-4) 0;
padding: 0;
border: 1px solid var(--border);
border-radius: var(--radius-md);
}
.check {
display: grid;
grid-template-columns: 1.4rem 1fr auto;
grid-template-areas: 'glyph label value' '. meta meta';
align-items: center;
gap: var(--space-1) var(--space-2);
padding: var(--space-2) var(--space-3);
border-bottom: 1px solid var(--border);
font-size: var(--font-sm);
}
.check:last-child {
border-bottom: none;
}
.check .glyph {
grid-area: glyph;
font-weight: var(--weight-semibold);
}
.label {
grid-area: label;
}
.value {
grid-area: value;
text-align: right;
font-variant-numeric: tabular-nums;
}
.meta {
grid-area: meta;
display: flex;
justify-content: space-between;
gap: var(--space-2);
font-size: var(--font-xs);
}
.hint {
font-size: var(--font-xs);
}
.fix {
white-space: nowrap;
}
/* Status colors on the glyph + left accent. */
.status-ok .glyph {
color: var(--success, #16a34a);
}
.status-warn {
box-shadow: inset 3px 0 0 var(--warning, #d97706);
}
.status-warn .glyph {
color: var(--warning, #d97706);
}
.status-critical {
box-shadow: inset 3px 0 0 var(--danger, #dc2626);
}
.status-critical .glyph {
color: var(--danger, #dc2626);
}
.muted {
color: var(--text-muted);
}
.disclosure {
background: none;
border: none;
color: var(--text);
cursor: pointer;
font-size: var(--font-sm);
padding: 0;
}
.ok-note {
margin-left: var(--space-2);
color: var(--success, #16a34a);
font-size: var(--font-sm);
}
.threshold-form {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
gap: var(--space-3);
margin-top: var(--space-3);
max-width: 40rem;
}
.threshold-form label {
display: flex;
flex-direction: column;
gap: 2px;
font-size: var(--font-xs);
color: var(--text-muted);
}
.threshold-form input {
height: 34px;
padding: 0 var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
.threshold-form .actions {
grid-column: 1 / -1;
display: flex;
gap: var(--space-2);
}
button.secondary {
background: var(--surface-elevated);
}
.error {
color: var(--danger, #dc2626);
}
</style>

View File

@@ -0,0 +1,81 @@
import { describe, it, expect, vi, afterEach } from 'vitest';
import { render, screen, cleanup, waitFor, fireEvent } from '@testing-library/svelte';
import HealthChecks from './HealthChecks.svelte';
import * as adminApi from '$lib/api/admin';
afterEach(() => {
cleanup();
vi.restoreAllMocks();
});
function thresholds(): adminApi.HealthThresholds {
return {
disk_pct: 90,
mem_pct: 90,
dead_jobs_max: 100,
crawl_fail_pct: 20,
analysis_fail_pct: 20,
cron_freshness_hours: 26,
missing_covers_max: 500
};
}
function report(over: Partial<adminApi.HealthReport> = {}): adminApi.HealthReport {
return {
status: 'warn',
checks: [
{
id: 'dead_jobs',
label: 'Dead jobs',
status: 'warn',
value: '137',
threshold: '100',
hint: 'jobs that exhausted their retries',
action_href: '/admin/crawler'
},
{
id: 'memory',
label: 'Memory usage',
status: 'ok',
value: '48%',
threshold: '90%',
hint: null,
action_href: null
}
],
thresholds: thresholds(),
...over
};
}
describe('HealthChecks', () => {
it('renders the overall status and each check, with a Fix link on failing ones', async () => {
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
render(HealthChecks);
await waitFor(() => expect(screen.getByTestId('health-overall')).toBeTruthy());
expect(screen.getByTestId('health-check-dead_jobs')).toBeTruthy();
expect(screen.getByText('137')).toBeTruthy();
// Failing check exposes a remediation link; the ok one does not.
const fix = screen.getByRole('link', { name: /fix/i }) as HTMLAnchorElement;
expect(fix.getAttribute('href')).toBe('/admin/crawler');
});
it('saves edited thresholds via updateHealthThresholds', async () => {
vi.spyOn(adminApi, 'getHealth').mockResolvedValue(report());
const update = vi
.spyOn(adminApi, 'updateHealthThresholds')
.mockResolvedValue(report({ status: 'ok' }));
render(HealthChecks);
await waitFor(() => expect(screen.getByTestId('health-thresholds-toggle')).toBeTruthy());
await fireEvent.click(screen.getByTestId('health-thresholds-toggle'));
const input = screen.getByTestId('threshold-dead_jobs_max') as HTMLInputElement;
await fireEvent.input(input, { target: { value: '250' } });
await fireEvent.click(screen.getByTestId('threshold-save'));
await waitFor(() => expect(update).toHaveBeenCalledTimes(1));
expect(update.mock.calls[0][0].dead_jobs_max).toBe(250);
});
});

View File

@@ -4,6 +4,7 @@
const tabs = [
{ href: '/admin', label: 'Overview' },
{ href: '/admin/health', label: 'Health' },
{ href: '/admin/users', label: 'Users' },
{ href: '/admin/mangas', label: 'Mangas' },
{ href: '/admin/crawler', label: 'Crawler' },

View File

@@ -9,11 +9,13 @@
analysisStatusStreamUrl,
getCrawlerSettings,
getAnalysisSettings,
getHealth,
type SystemStats,
type OverviewStats,
type CrawlerStatus,
type AnalysisMetrics,
type AnalysisEvent
type AnalysisEvent,
type HealthReport
} from '$lib/api/admin';
import { formatBytes as fmtBytes } from '$lib/upload-validation';
import type { LayoutData } from './$types';
@@ -44,6 +46,8 @@
let liveCrawler = $state(false);
let liveAnalysis = $state(false);
let health = $state<HealthReport | null>(null);
let sysTimer: ReturnType<typeof setInterval> | null = null;
let overviewTimer: ReturnType<typeof setInterval> | null = null;
let crawlerSource: EventSource | null = null;
@@ -86,6 +90,17 @@
// ignore
}
}
async function refreshHealth() {
try {
health = await getHealth();
} catch {
// keep the last good snapshot; retried next tick
}
}
const healthFailing = $derived(
health ? health.checks.filter((c) => c.status !== 'ok').length : 0
);
async function loadSettings() {
try {
const c = await getCrawlerSettings();
@@ -183,6 +198,7 @@
refreshSys();
refreshOverview();
refreshMetrics();
refreshHealth();
loadSettings();
getCrawlerStatus()
.then((s) => (crawler = s))
@@ -196,6 +212,7 @@
overviewTimer = setInterval(() => {
refreshOverview();
refreshMetrics();
refreshHealth();
}, 60000);
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', onVisibilityChange);
@@ -237,6 +254,26 @@
<h1>Overview</h1>
{#if health}
<a
class="health-summary status-{health.status}"
href="/admin/health"
data-testid="overview-health"
>
<span class="glyph" aria-hidden="true"
>{health.status === 'ok' ? '✓' : health.status === 'warn' ? '⚠' : '✗'}</span
>
<span class="hs-label">
{#if health.status === 'ok'}
All health checks passing
{:else}
{healthFailing} health check{healthFailing === 1 ? '' : 's'} need attention
{/if}
</span>
<span class="hs-view">View →</span>
</a>
{/if}
{#if sys.alerts.length > 0 || extraAlerts.length > 0}
<section class="alerts" data-testid="admin-alerts">
{#each sys.alerts as a (a.message)}
@@ -434,6 +471,44 @@
h1 {
margin: 0 0 var(--space-4) 0;
}
.health-summary {
display: flex;
align-items: center;
gap: var(--space-2);
padding: var(--space-2) var(--space-3);
margin-bottom: var(--space-3);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
font-size: var(--font-sm);
}
.health-summary:hover {
background: var(--surface-elevated);
text-decoration: none;
}
.health-summary .glyph {
font-weight: var(--weight-semibold);
}
.health-summary .hs-view {
margin-left: auto;
color: var(--text-muted);
}
.health-summary.status-ok .glyph {
color: var(--success, #16a34a);
}
.health-summary.status-warn {
border-left: 4px solid var(--warning, #d97706);
}
.health-summary.status-warn .glyph {
color: var(--warning, #d97706);
}
.health-summary.status-critical {
border-left: 4px solid var(--danger, #dc2626);
}
.health-summary.status-critical .glyph {
color: var(--danger, #dc2626);
}
.alerts {
display: flex;
flex-direction: column;

View File

@@ -0,0 +1,26 @@
<script lang="ts">
import HealthChecks from '$lib/components/admin/HealthChecks.svelte';
</script>
<svelte:head><title>Health · Admin</title></svelte:head>
<h1>Health</h1>
<p class="lead">
Operational checks rolled up from system sensors, the job queue, recent
failure rates and the crawler daemon. Thresholds are editable below and
apply immediately.
</p>
<HealthChecks />
<style>
h1 {
margin-bottom: var(--space-2);
}
.lead {
color: var(--text-muted);
font-size: var(--font-sm);
margin-bottom: var(--space-4);
max-width: 44rem;
}
</style>