feat(admin): time-series trend charts on the metrics tabs (0.87.0)
Turn the dormant timing data in crawl_metrics / page_analysis into "is it
healthy over time" views. New bucketed series queries (GROUP BY date_trunc,
hour|day via a closed Bucket enum so the unit can't be attacker-controlled)
behind GET /v1/admin/{crawler,analysis}/metrics/series, with a shared
SeriesParams/resolve_bucket helper (bad bucket → 400) and migration 0030
indexing page_analysis(analyzed_at) for the analysis scan.
Frontend: a dependency-free SVG TrendChart (line+area, null buckets render as
gaps, empty-state, role=img) embedded above the per-op tables in Crawler and
Analysis → Metrics, driven by each panel's existing window selector with
AbortController-cancelled fetches. A buildSeries() util fills the continuous
bucket axis (throughput 0 for empty buckets, success/duration null) — unit
tested alongside the chart and the series API client.
Closes the Phase-1 observability set (audit log · health checks · trends).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -44,6 +44,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||
.route("/admin/analysis/history", get(list_history))
|
||||
.route("/admin/analysis/metrics", get(metrics))
|
||||
.route("/admin/analysis/metrics/series", get(metrics_series))
|
||||
.route("/admin/analysis/status/stream", get(stream_status))
|
||||
}
|
||||
|
||||
@@ -215,6 +216,46 @@ pub(super) fn window_since(days: i64) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
}
|
||||
}
|
||||
|
||||
// --- shared trend-series plumbing (crawler + analysis charts) ---------------
|
||||
|
||||
/// Query params for the bucketed trend-series endpoints. `bucket` is
|
||||
/// optional; when absent it's derived from `days`.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct SeriesParams {
|
||||
#[serde(default)]
|
||||
pub days: i64,
|
||||
#[serde(default)]
|
||||
pub bucket: Option<String>,
|
||||
}
|
||||
|
||||
/// Envelope for a trend series.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SeriesResponse<T> {
|
||||
pub buckets: Vec<T>,
|
||||
}
|
||||
|
||||
/// Resolve the `date_trunc` granularity: an explicit `bucket` (validated to
|
||||
/// `hour`|`day`, else 400) wins; otherwise short windows bucket by hour and
|
||||
/// longer ones by day so the point count stays chart-friendly.
|
||||
pub(super) fn resolve_bucket(
|
||||
days: i64,
|
||||
explicit: Option<&str>,
|
||||
) -> Result<crate::repo::crawl_metrics::Bucket, AppError> {
|
||||
use crate::repo::crawl_metrics::Bucket;
|
||||
match explicit {
|
||||
Some("hour") => Ok(Bucket::Hour),
|
||||
Some("day") => Ok(Bucket::Day),
|
||||
Some(other) => Err(AppError::InvalidInput(format!(
|
||||
"invalid bucket '{other}' (expected 'hour' or 'day')"
|
||||
))),
|
||||
None => Ok(if days > 0 && days <= 2 {
|
||||
Bucket::Hour
|
||||
} else {
|
||||
Bucket::Day
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn metrics(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
@@ -225,6 +266,17 @@ async fn metrics(
|
||||
Ok(Json(m))
|
||||
}
|
||||
|
||||
async fn metrics_series(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SeriesParams>,
|
||||
) -> AppResult<Json<SeriesResponse<crate::domain::crawl_metrics::MetricsBucket>>> {
|
||||
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||
let buckets =
|
||||
repo::page_analysis::analysis_series(&state.db, bucket, window_since(params.days)).await?;
|
||||
Ok(Json(SeriesResponse { buckets }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
|
||||
@@ -11,20 +11,31 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawl_metrics::OpFilter;
|
||||
|
||||
use super::default_limit;
|
||||
// Reuse the analysis handler's window helper so both endpoints clamp `days`
|
||||
// identically.
|
||||
use crate::api::admin::analysis::window_since;
|
||||
// Reuse the analysis handler's window + bucket helpers so all the metrics
|
||||
// endpoints clamp `days` and resolve `bucket` identically.
|
||||
use crate::api::admin::analysis::{resolve_bucket, window_since, SeriesParams, SeriesResponse};
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/metrics", get(summary))
|
||||
.route("/admin/crawler/metrics/ops", get(list_ops))
|
||||
.route("/admin/crawler/metrics/series", get(series))
|
||||
}
|
||||
|
||||
async fn series(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<SeriesParams>,
|
||||
) -> AppResult<Json<SeriesResponse<MetricsBucket>>> {
|
||||
let bucket = resolve_bucket(params.days, params.bucket.as_deref())?;
|
||||
let buckets = repo::crawl_metrics::series(&state.db, bucket, window_since(params.days)).await?;
|
||||
Ok(Json(SeriesResponse { buckets }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
|
||||
@@ -11,6 +11,20 @@ use serde::Serialize;
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// One time bucket of a metrics series — count, ok/failed split, and mean
|
||||
/// duration over the bucket. Shared by the crawl-ops and analysis trend
|
||||
/// charts (both are `{t, n, ok, failed, avg_ms}` over a `date_trunc` window).
|
||||
/// Empty intervals are simply absent; the client fills the continuous axis.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MetricsBucket {
|
||||
/// Bucket start (`date_trunc(unit, finished_at|analyzed_at)`).
|
||||
pub t: DateTime<Utc>,
|
||||
pub n: i64,
|
||||
pub ok: i64,
|
||||
pub failed: i64,
|
||||
pub avg_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// Per-operation-type average roll-up over a time window.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OpSummary {
|
||||
|
||||
@@ -8,7 +8,24 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::crawl_metrics::{OpRow, OpSummary};
|
||||
use crate::domain::crawl_metrics::{MetricsBucket, OpRow, OpSummary};
|
||||
|
||||
/// Time-bucket granularity for trend series. A closed enum (not a free
|
||||
/// string) so the `date_trunc` unit can never be attacker-controlled.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Bucket {
|
||||
Hour,
|
||||
Day,
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Bucket::Hour => "hour",
|
||||
Bucket::Day => "day",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Operation kinds, kept as consts so call sites and the CHECK constraint
|
||||
/// don't drift from a free-typed literal.
|
||||
@@ -75,6 +92,34 @@ pub async fn summary(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Bucketed throughput/success/duration series over an optional window.
|
||||
/// Plain `GROUP BY date_trunc(...)` — empty intervals are absent and the
|
||||
/// client fills the continuous axis. Uses `crawl_metrics_time_idx`.
|
||||
pub async fn series(
|
||||
pool: &PgPool,
|
||||
bucket: Bucket,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> sqlx::Result<Vec<MetricsBucket>> {
|
||||
sqlx::query_as::<_, MetricsBucket>(
|
||||
r#"
|
||||
SELECT
|
||||
date_trunc($1, finished_at) AS t,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE outcome = 'ok') AS ok,
|
||||
COUNT(*) FILTER (WHERE outcome = 'failed') AS failed,
|
||||
AVG(duration_ms)::float8 AS avg_ms
|
||||
FROM crawl_metrics
|
||||
WHERE ($2::timestamptz IS NULL OR finished_at >= $2)
|
||||
GROUP BY t
|
||||
ORDER BY t
|
||||
"#,
|
||||
)
|
||||
.bind(bucket.as_str())
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Filters for [`list_ops`]. `None`/empty widens the scope.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct OpFilter<'a> {
|
||||
|
||||
@@ -495,6 +495,39 @@ pub async fn analysis_metrics(
|
||||
Ok(AnalysisMetrics { n, ok, failed, avg_ms, by_model })
|
||||
}
|
||||
|
||||
/// Bucketed analysis throughput/success/duration series over an optional
|
||||
/// window, for the Analysis-tab trend charts. `ok`/`failed` follow the
|
||||
/// `done`/`failed` statuses; mean duration is over completed pages only.
|
||||
/// Empty intervals are absent (client fills the axis). The
|
||||
/// `page_analysis_analyzed_at_idx` (migration 0030) backs the scan.
|
||||
pub async fn analysis_series(
|
||||
pool: &PgPool,
|
||||
bucket: crate::repo::crawl_metrics::Bucket,
|
||||
since: Option<DateTime<Utc>>,
|
||||
) -> AppResult<Vec<crate::domain::crawl_metrics::MetricsBucket>> {
|
||||
let rows = sqlx::query_as::<_, crate::domain::crawl_metrics::MetricsBucket>(
|
||||
r#"
|
||||
SELECT
|
||||
date_trunc($1, analyzed_at) AS t,
|
||||
COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE status = 'done') AS ok,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||
AVG(duration_ms) FILTER (WHERE status = 'done')::float8 AS avg_ms
|
||||
FROM page_analysis
|
||||
WHERE status <> 'pending'
|
||||
AND analyzed_at IS NOT NULL
|
||||
AND ($2::timestamptz IS NULL OR analyzed_at >= $2)
|
||||
GROUP BY t
|
||||
ORDER BY t
|
||||
"#,
|
||||
)
|
||||
.bind(bucket.as_str())
|
||||
.bind(since)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Persist a completed analysis for a page, replacing any previous result.
|
||||
///
|
||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||
|
||||
Reference in New Issue
Block a user