feat(admin): time-series trend charts on the metrics tabs (0.87.0)

Turn the dormant timing data in crawl_metrics / page_analysis into "is it
healthy over time" views. New bucketed series queries (GROUP BY date_trunc,
hour|day via a closed Bucket enum so the unit can't be attacker-controlled)
behind GET /v1/admin/{crawler,analysis}/metrics/series, with a shared
SeriesParams/resolve_bucket helper (bad bucket → 400) and migration 0030
indexing page_analysis(analyzed_at) for the analysis scan.

Frontend: a dependency-free SVG TrendChart (line+area, null buckets render as
gaps, empty-state, role=img) embedded above the per-op tables in Crawler and
Analysis → Metrics, driven by each panel's existing window selector with
AbortController-cancelled fetches. A buildSeries() util fills the continuous
bucket axis (throughput 0 for empty buckets, success/duration null) — unit
tested alongside the chart and the series API client.

Closes the Phase-1 observability set (audit log · health checks · trends).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-19 11:48:56 +02:00
parent 314fc8738b
commit c6a6d1690d
18 changed files with 856 additions and 10 deletions

View File

@@ -0,0 +1,51 @@
import { describe, it, expect } from 'vitest';
import { buildSeries, unitForDays, truncUtc } from './series';
import type { MetricsBucket } from '$lib/api/admin';
function bucket(t: string, n: number, ok: number, avg: number | null): MetricsBucket {
return { t, n, ok, failed: n - ok, avg_ms: avg };
}
describe('series helpers', () => {
it('picks hour vs day by window', () => {
expect(unitForDays(1)).toBe('hour');
expect(unitForDays(2)).toBe('hour');
expect(unitForDays(7)).toBe('day');
expect(unitForDays(0)).toBe('day');
});
it('truncUtc snaps to hour/day boundaries', () => {
const ms = Date.parse('2026-06-18T09:37:42Z');
expect(new Date(truncUtc(ms, 'hour')).toISOString()).toBe('2026-06-18T09:00:00.000Z');
expect(new Date(truncUtc(ms, 'day')).toISOString()).toBe('2026-06-18T00:00:00.000Z');
});
it('fills empty buckets: throughput 0, success/duration null', () => {
// days=2 → hourly. Provide data only at 10:00; 11:00 must be a gap.
const now = Date.parse('2026-06-18T11:30:00Z');
const buckets = [bucket('2026-06-18T10:00:00Z', 4, 3, 200)];
const s = buildSeries(buckets, 2, now);
// Find the 10:00 and 11:00 slots.
const at10 = s.throughput.findIndex((p) => p.t === '2026-06-18T10:00:00.000Z');
expect(at10).toBeGreaterThanOrEqual(0);
expect(s.throughput[at10].value).toBe(4);
expect(s.success[at10].value).toBe(75); // 3/4
expect(s.duration[at10].value).toBe(200);
const at11 = s.throughput.findIndex((p) => p.t === '2026-06-18T11:00:00.000Z');
expect(s.throughput[at11].value).toBe(0); // no ops → zero throughput
expect(s.success[at11].value).toBeNull(); // no denominator → gap
expect(s.duration[at11].value).toBeNull();
});
it('all-time spans only the data present', () => {
const buckets = [
bucket('2026-06-10T00:00:00Z', 1, 1, 100),
bucket('2026-06-12T00:00:00Z', 1, 0, 100)
];
const s = buildSeries(buckets, 0);
// 10th, 11th (gap), 12th → 3 day slots.
expect(s.throughput.length).toBe(3);
expect(s.throughput[1].value).toBe(0);
});
});

View File

@@ -0,0 +1,75 @@
// Turn the backend's sparse, ordered metric buckets into the three
// continuous point series the trend charts render. Empty intervals become
// explicit points so the chart shows real gaps rather than interpolating
// across missing time: throughput is 0 (no ops happened), while success-rate
// and duration are null (no denominator / no samples) and render as a gap.
import type { MetricsBucket } from '$lib/api/admin';
export type SeriesPoint = { t: string; value: number | null };
export type TrendSeries = {
throughput: SeriesPoint[];
success: SeriesPoint[];
duration: SeriesPoint[];
};
/** Bucket unit the backend uses for a given window (kept in lockstep with
* `resolve_bucket` on the server: short windows → hour, else day). */
export function unitForDays(days: number): 'hour' | 'day' {
return days > 0 && days <= 2 ? 'hour' : 'day';
}
function stepMs(unit: 'hour' | 'day'): number {
return unit === 'hour' ? 3_600_000 : 86_400_000;
}
/** Truncate a UTC timestamp to the start of its hour/day, matching Postgres
* `date_trunc` (which the bucket `t` values already are). */
export function truncUtc(ms: number, unit: 'hour' | 'day'): number {
const d = new Date(ms);
if (unit === 'hour') {
d.setUTCMinutes(0, 0, 0);
} else {
d.setUTCHours(0, 0, 0, 0);
}
return d.getTime();
}
export function buildSeries(
buckets: MetricsBucket[],
days: number,
now: number = Date.now()
): TrendSeries {
const unit = unitForDays(days);
const step = stepMs(unit);
const byT = new Map<number, MetricsBucket>();
for (const b of buckets) byT.set(truncUtc(new Date(b.t).getTime(), unit), b);
let startMs: number;
let endMs: number;
if (days > 0) {
endMs = truncUtc(now, unit);
startMs = truncUtc(now - days * 86_400_000, unit);
} else {
// All-time: span only the data we have.
if (buckets.length === 0) return { throughput: [], success: [], duration: [] };
const times = buckets.map((b) => truncUtc(new Date(b.t).getTime(), unit));
startMs = Math.min(...times);
endMs = Math.max(...times);
}
const throughput: SeriesPoint[] = [];
const success: SeriesPoint[] = [];
const duration: SeriesPoint[] = [];
// Guard against an absurd point count (defensive; backend caps days at 365).
let guard = 0;
for (let t = startMs; t <= endMs && guard < 5000; t += step, guard++) {
const b = byT.get(t);
const iso = new Date(t).toISOString();
throughput.push({ t: iso, value: b ? b.n : 0 });
success.push({ t: iso, value: b && b.n > 0 ? (b.ok / b.n) * 100 : null });
duration.push({ t: iso, value: b && b.avg_ms != null ? b.avg_ms : null });
}
return { throughput, success, duration };
}

View File

@@ -45,7 +45,9 @@ import {
getAnalysisMetrics,
listAuditLog,
getHealth,
updateHealthThresholds
updateHealthThresholds,
getCrawlerMetricsSeries,
getAnalysisMetricsSeries
} from './admin';
function ok(body: unknown, status = 200): Response {
@@ -532,6 +534,20 @@ describe('admin crawler api client', () => {
}
};
it('getCrawlerMetricsSeries GETs the crawler series with days', async () => {
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
await getCrawlerMetricsSeries(7);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/crawler\/metrics\/series\?days=7$/);
});
it('getAnalysisMetricsSeries omits days=0 (all-time)', async () => {
fetchSpy.mockResolvedValueOnce(ok({ buckets: [] }));
await getAnalysisMetricsSeries(0);
const url = fetchSpy.mock.calls[0][0] as string;
expect(url).toMatch(/\/v1\/admin\/analysis\/metrics\/series$/);
});
it('getHealth GETs /v1/admin/health', async () => {
fetchSpy.mockResolvedValueOnce(ok(healthFixture));
const r = await getHealth();

View File

@@ -1061,3 +1061,37 @@ export async function updateHealthThresholds(t: HealthThresholds): Promise<Healt
body: JSON.stringify(t)
});
}
// ---- metrics trend series (crawler + analysis charts) ----------------------
export type MetricsBucket = {
/** Bucket start (ISO timestamp). */
t: string;
n: number;
ok: number;
failed: number;
avg_ms: number | null;
};
export type MetricsSeries = { buckets: MetricsBucket[] };
function seriesQs(days: number): string {
const params = new URLSearchParams();
if (days > 0) params.set('days', String(days));
const qs = params.toString();
return qs ? `?${qs}` : '';
}
export async function getCrawlerMetricsSeries(
days = 7,
init?: RequestInit
): Promise<MetricsSeries> {
return request<MetricsSeries>(`/v1/admin/crawler/metrics/series${seriesQs(days)}`, init);
}
export async function getAnalysisMetricsSeries(
days = 7,
init?: RequestInit
): Promise<MetricsSeries> {
return request<MetricsSeries>(`/v1/admin/analysis/metrics/series${seriesQs(days)}`, init);
}

View File

@@ -1,12 +1,20 @@
<script lang="ts">
import { onMount } from 'svelte';
import { fmtDuration } from '$lib/format';
import { getAnalysisMetrics, type AnalysisMetrics } from '$lib/api/admin';
import TrendChart from '$lib/components/charts/TrendChart.svelte';
import { buildSeries, type TrendSeries } from '$lib/admin/series';
import {
getAnalysisMetrics,
getAnalysisMetricsSeries,
type AnalysisMetrics
} from '$lib/api/admin';
let days = $state(7);
let metrics = $state<AnalysisMetrics | null>(null);
let loading = $state(true);
let error = $state<string | null>(null);
let series = $state<TrendSeries | null>(null);
let seriesCtrl: AbortController | null = null;
const successPct = $derived(
metrics && metrics.n > 0 ? Math.round((metrics.ok / metrics.n) * 100) : null
@@ -22,6 +30,20 @@
} finally {
loading = false;
}
loadSeries();
}
async function loadSeries() {
seriesCtrl?.abort();
const ctrl = new AbortController();
seriesCtrl = ctrl;
try {
const resp = await getAnalysisMetricsSeries(days, { signal: ctrl.signal });
series = buildSeries(resp.buckets, days);
} catch (e) {
if ((e as Error)?.name === 'AbortError') return;
// Chart failure is non-fatal — keep the tiles/table usable.
}
}
onMount(load);
@@ -45,6 +67,24 @@
{:else if loading}
<p class="muted">Loading…</p>
{:else if metrics}
{#if series}
<div class="charts" data-testid="analysis-metrics-charts">
<TrendChart points={series.throughput} label="Pages analyzed (per bucket)" />
<TrendChart
points={series.success}
label="Success rate"
format={(v) => `${v.toFixed(0)}%`}
accent="var(--success, #16a34a)"
/>
<TrendChart
points={series.duration}
label="Avg duration"
format={(v) => fmtDuration(v)}
accent="var(--warning, #d97706)"
/>
</div>
{/if}
<div class="tiles">
<div class="tile">
<span class="label">Pages analyzed</span>
@@ -112,6 +152,13 @@
color: var(--text);
font-size: var(--font-sm);
}
.charts {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
gap: var(--space-3) var(--space-4);
margin-bottom: var(--space-4);
max-width: 60rem;
}
.tiles {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));

View File

@@ -0,0 +1,168 @@
<script lang="ts">
// A small, dependency-free line+area sparkline for admin trend series.
// Null values render as gaps (a bucket with no data), so a missing point
// never draws a misleading line to zero.
type Point = { t: string; value: number | null };
let {
points,
label,
format = (v: number) => v.toFixed(0),
height = 72,
accent = 'var(--primary, #2563eb)'
}: {
points: Point[];
label: string;
format?: (v: number) => string;
height?: number;
accent?: string;
} = $props();
// Internal coordinate system; the SVG scales to its container width.
const W = 600;
const PAD = 6;
const values = $derived(
points.map((p) => p.value).filter((v): v is number => v != null)
);
const hasData = $derived(values.length > 0);
const max = $derived(hasData ? Math.max(...values) : 0);
const min = $derived(hasData ? Math.min(...values) : 0);
const latest = $derived.by(() => {
for (let i = points.length - 1; i >= 0; i--) {
if (points[i].value != null) return points[i].value as number;
}
return null;
});
function x(i: number): number {
if (points.length <= 1) return W / 2;
return PAD + (i / (points.length - 1)) * (W - 2 * PAD);
}
function y(v: number): number {
const range = max > min ? max - min : 1;
const norm = max > min ? (v - min) / range : 0.5;
const inner = height - 2 * PAD;
return PAD + (1 - norm) * inner;
}
// Break the series into runs of consecutive non-null points so gaps stay
// gaps. Each run becomes one line path and one area path.
type Run = { i: number; v: number }[];
const runs = $derived.by<Run[]>(() => {
const out: Run[] = [];
let cur: Run = [];
points.forEach((p, i) => {
if (p.value == null) {
if (cur.length) out.push(cur);
cur = [];
} else {
cur.push({ i, v: p.value });
}
});
if (cur.length) out.push(cur);
return out;
});
function linePath(run: Run): string {
return run.map((pt, k) => `${k === 0 ? 'M' : 'L'} ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
}
function areaPath(run: Run): string {
if (run.length === 0) return '';
const base = height - PAD;
const top = run.map((pt) => `L ${x(pt.i).toFixed(1)} ${y(pt.v).toFixed(1)}`).join(' ');
const x0 = x(run[0].i).toFixed(1);
const xn = x(run[run.length - 1].i).toFixed(1);
return `M ${x0} ${base} ${top} L ${xn} ${base} Z`;
}
function shortTime(iso: string): string {
const d = new Date(iso);
return d.toLocaleString(undefined, { month: 'short', day: 'numeric', hour: 'numeric' });
}
</script>
<figure class="trend" data-testid="trend-chart">
<figcaption>
<span class="lbl">{label}</span>
{#if hasData}
<span class="muted"
>peak {format(max)} · latest {latest != null ? format(latest) : '—'}</span
>
{/if}
</figcaption>
{#if !hasData}
<div class="empty muted" style={`height:${height}px`} data-testid="trend-empty">
No data in this window
</div>
{:else}
<svg
viewBox={`0 0 ${W} ${height}`}
preserveAspectRatio="none"
role="img"
aria-label={`${label}: ${values.length} points, latest ${latest != null ? format(latest) : 'none'}, peak ${format(max)}`}
>
{#each runs as run, ri (ri)}
<path class="area" d={areaPath(run)} fill={accent} />
<path class="line" d={linePath(run)} stroke={accent} />
{#if run.length === 1}
<circle cx={x(run[0].i)} cy={y(run[0].v)} r="3" fill={accent} />
{/if}
{/each}
</svg>
<div class="xaxis muted">
<span>{shortTime(points[0].t)}</span>
<span>{shortTime(points[points.length - 1].t)}</span>
</div>
{/if}
</figure>
<style>
.trend {
margin: 0 0 var(--space-3) 0;
}
figcaption {
display: flex;
justify-content: space-between;
align-items: baseline;
gap: var(--space-2);
font-size: var(--font-sm);
margin-bottom: 2px;
}
.lbl {
font-weight: var(--weight-medium);
}
svg {
width: 100%;
display: block;
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
}
.area {
opacity: 0.12;
stroke: none;
}
.line {
fill: none;
stroke-width: 1.5;
vector-effect: non-scaling-stroke;
}
.xaxis {
display: flex;
justify-content: space-between;
font-size: var(--font-xs);
margin-top: 2px;
}
.empty {
display: flex;
align-items: center;
justify-content: center;
border: 1px dashed var(--border);
border-radius: var(--radius-sm);
font-size: var(--font-sm);
}
.muted {
color: var(--text-muted);
}
</style>

View File

@@ -0,0 +1,53 @@
import { describe, it, expect, afterEach } from 'vitest';
import { render, screen, cleanup } from '@testing-library/svelte';
import TrendChart from './TrendChart.svelte';
afterEach(() => cleanup());
describe('TrendChart', () => {
it('renders an empty state when there is no data', () => {
render(TrendChart, {
props: {
points: [
{ t: '2026-06-18T09:00:00Z', value: null },
{ t: '2026-06-18T10:00:00Z', value: null }
],
label: 'Throughput'
}
});
expect(screen.getByTestId('trend-empty')).toBeTruthy();
});
it('draws a path and labels peak/latest when data is present', () => {
const { container } = render(TrendChart, {
props: {
points: [
{ t: '2026-06-18T09:00:00Z', value: 2 },
{ t: '2026-06-18T10:00:00Z', value: 6 },
{ t: '2026-06-18T11:00:00Z', value: 4 }
],
label: 'Throughput'
}
});
const line = container.querySelector('path.line') as SVGPathElement;
expect(line).toBeTruthy();
expect(line.getAttribute('d')).toMatch(/^M /);
// Header summarizes peak + latest.
expect(screen.getByText(/peak 6 · latest 4/)).toBeTruthy();
});
it('breaks the line across null gaps into separate paths', () => {
const { container } = render(TrendChart, {
props: {
points: [
{ t: '2026-06-18T09:00:00Z', value: 2 },
{ t: '2026-06-18T10:00:00Z', value: null },
{ t: '2026-06-18T11:00:00Z', value: 4 }
],
label: 'Success'
}
});
// Two runs → two line paths (and two area paths).
expect(container.querySelectorAll('path.line').length).toBe(2);
});
});

View File

@@ -2,8 +2,11 @@
import { onMount } from 'svelte';
import Pager from '$lib/components/Pager.svelte';
import { fmtDuration } from '$lib/format';
import TrendChart from '$lib/components/charts/TrendChart.svelte';
import { buildSeries, type TrendSeries } from '$lib/admin/series';
import {
getCrawlerMetrics,
getCrawlerMetricsSeries,
listCrawlerOps,
type OpSummary,
type OpRow,
@@ -15,6 +18,8 @@
let days = $state(7);
let summary = $state<OpSummary[]>([]);
let summaryLoading = $state(true);
let series = $state<TrendSeries | null>(null);
let seriesCtrl: AbortController | null = null;
let rows = $state<OpRow[]>([]);
let total = $state(0);
@@ -65,6 +70,19 @@
}
}
async function loadSeries() {
seriesCtrl?.abort();
const ctrl = new AbortController();
seriesCtrl = ctrl;
try {
const resp = await getCrawlerMetricsSeries(days, { signal: ctrl.signal });
series = buildSeries(resp.buckets, days);
} catch (e) {
if ((e as Error)?.name === 'AbortError') return;
// A chart failure shouldn't blank the whole panel; leave prior series.
}
}
async function loadOps() {
opsLoading = true;
try {
@@ -86,12 +104,14 @@
onMount(() => {
loadSummary();
loadSeries();
loadOps();
});
function onWindowChange() {
page = 1;
loadSummary();
loadSeries();
loadOps();
}
function onOpsFilter() {
@@ -138,6 +158,24 @@
<p class="error" role="alert">{error}</p>
{/if}
{#if series}
<div class="charts" data-testid="crawler-metrics-charts">
<TrendChart points={series.throughput} label="Throughput (ops/bucket)" />
<TrendChart
points={series.success}
label="Success rate"
format={(v) => `${v.toFixed(0)}%`}
accent="var(--success, #16a34a)"
/>
<TrendChart
points={series.duration}
label="Avg duration"
format={(v) => fmtDuration(v)}
accent="var(--warning, #d97706)"
/>
</div>
{/if}
<h2>Average durations by type</h2>
{#if summaryLoading}
<p class="muted">Loading…</p>
@@ -246,6 +284,13 @@
justify-content: flex-end;
margin-bottom: var(--space-2);
}
.charts {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
gap: var(--space-3) var(--space-4);
margin-bottom: var(--space-4);
max-width: 60rem;
}
label {
font-size: var(--font-sm);
color: var(--text-muted);