feat(metrics): per-app observability dashboard over execution_logs (A2)
Aggregates the existing execution_logs table (no hot-path instrumentation):
ExecutionLogRepository::summarize_for_app returns counts, error rate, latency
avg/p50/p95 (percentile_cont), by-status/by-source breakdowns, and an hourly
series over a trailing window (clamped 1h..90d). New metrics_api router at
GET /api/v1/admin/apps/{id}/metrics?window= (AppLogRead), and a dashboard
Metrics tab (summary cards + an inline-SVG per-hour bar chart, no JS dep).
Pinned by a manager-core test asserting the exact rollup (percentiles included)
and the empty-app zero case.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -78,6 +78,7 @@ pub mod kv_service;
|
||||
pub mod log_sink;
|
||||
pub mod login_rate_limit;
|
||||
pub mod materialize;
|
||||
pub mod metrics_api;
|
||||
pub mod migrations;
|
||||
pub mod module_source;
|
||||
pub mod outbox_event_emitter;
|
||||
@@ -231,6 +232,7 @@ pub use kv_api::{kv_admin_router, KvAdminState};
|
||||
pub use kv_repo::{KvRepo, KvRepoError, PostgresKvRepo};
|
||||
pub use kv_service::KvServiceImpl;
|
||||
pub use log_sink::PostgresExecutionLogSink;
|
||||
pub use metrics_api::{metrics_router, MetricsState};
|
||||
pub use module_source::PostgresModuleSource;
|
||||
pub use outbox_event_emitter::OutboxEventEmitter;
|
||||
pub use outbox_repo::{
|
||||
|
||||
118
crates/manager-core/src/metrics_api.rs
Normal file
118
crates/manager-core/src/metrics_api.rs
Normal file
@@ -0,0 +1,118 @@
|
||||
//! `/api/v1/admin/apps/{id}/metrics` — read-only observability rollup (A2).
|
||||
//!
|
||||
//! Aggregates the existing `execution_logs` table (counts, error rate, latency
|
||||
//! percentiles, an hourly series) for one app over a trailing window. No
|
||||
//! hot-path instrumentation: the data plane already persists `duration_ms` /
|
||||
//! `status` / `source` / `created_at` per run, so this is a pure read model
|
||||
//! behind the dashboard's Metrics tab.
|
||||
//!
|
||||
//! Capability: `AppLogRead` (the same tier as the per-script log view),
|
||||
//! resolved against the app loaded from the path.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use axum::routing::get;
|
||||
use axum::{Extension, Router};
|
||||
use picloud_shared::{AppId, Principal};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::app_repo::AppRepository;
|
||||
use crate::authz::{require, AuthzDenied, AuthzRepo, Capability};
|
||||
use crate::repo::{ExecutionLogRepository, MetricsSummary};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MetricsState {
|
||||
pub logs: Arc<dyn ExecutionLogRepository>,
|
||||
pub apps: Arc<dyn AppRepository>,
|
||||
pub authz: Arc<dyn AuthzRepo>,
|
||||
}
|
||||
|
||||
pub fn metrics_router(state: MetricsState) -> Router {
|
||||
Router::new()
|
||||
.route("/apps/{app_id}/metrics", get(summary))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// Default rollup window when `?window=` is absent (last 24 hours).
|
||||
const DEFAULT_WINDOW_HOURS: i32 = 24;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct MetricsQuery {
|
||||
/// Trailing window in hours (clamped to `[1, 2160]` by the repo).
|
||||
#[serde(default)]
|
||||
pub window: Option<i32>,
|
||||
}
|
||||
|
||||
async fn summary(
|
||||
State(s): State<MetricsState>,
|
||||
Extension(principal): Extension<Principal>,
|
||||
Path(id_or_slug): Path<String>,
|
||||
Query(q): Query<MetricsQuery>,
|
||||
) -> Result<Json<MetricsSummary>, MetricsApiError> {
|
||||
let app_id = resolve_app(&*s.apps, &id_or_slug).await?;
|
||||
require(s.authz.as_ref(), &principal, Capability::AppLogRead(app_id)).await?;
|
||||
let window = q.window.unwrap_or(DEFAULT_WINDOW_HOURS);
|
||||
let summary = s
|
||||
.logs
|
||||
.summarize_for_app(app_id, window)
|
||||
.await
|
||||
.map_err(|e| MetricsApiError::Backend(e.to_string()))?;
|
||||
Ok(Json(summary))
|
||||
}
|
||||
|
||||
async fn resolve_app(apps: &dyn AppRepository, ident: &str) -> Result<AppId, MetricsApiError> {
|
||||
crate::app_repo::resolve_app(apps, ident)
|
||||
.await
|
||||
.map_err(|e| MetricsApiError::Backend(e.to_string()))?
|
||||
.map(|l| l.app.id)
|
||||
.ok_or(MetricsApiError::AppNotFound)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MetricsApiError {
|
||||
#[error("app not found")]
|
||||
AppNotFound,
|
||||
#[error("forbidden")]
|
||||
Forbidden,
|
||||
#[error("authorization repo error: {0}")]
|
||||
AuthzRepo(String),
|
||||
#[error("metrics backend: {0}")]
|
||||
Backend(String),
|
||||
}
|
||||
|
||||
impl From<AuthzDenied> for MetricsApiError {
|
||||
fn from(d: AuthzDenied) -> Self {
|
||||
match d {
|
||||
AuthzDenied::Denied => Self::Forbidden,
|
||||
AuthzDenied::Repo(e) => Self::AuthzRepo(e.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for MetricsApiError {
|
||||
fn into_response(self) -> Response {
|
||||
use axum::http::StatusCode;
|
||||
let (status, body) = match &self {
|
||||
Self::AppNotFound => (StatusCode::NOT_FOUND, json!({ "error": self.to_string() })),
|
||||
Self::Forbidden => (StatusCode::FORBIDDEN, json!({ "error": self.to_string() })),
|
||||
Self::AuthzRepo(e) => {
|
||||
tracing::error!(error = %e, "metrics admin authz error");
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
json!({ "error": "internal error" }),
|
||||
)
|
||||
}
|
||||
Self::Backend(e) => {
|
||||
tracing::error!(error = %e, "metrics admin backend error");
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
json!({ "error": "internal error" }),
|
||||
)
|
||||
}
|
||||
};
|
||||
(status, Json(body)).into_response()
|
||||
}
|
||||
}
|
||||
@@ -731,6 +731,40 @@ impl ExecutionLogCursor {
|
||||
}
|
||||
}
|
||||
|
||||
/// One `(key, count)` row of a metrics breakdown (by status or by source).
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct MetricsCount {
|
||||
pub key: String,
|
||||
pub count: i64,
|
||||
}
|
||||
|
||||
/// One hour-bucket of the metrics time series.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct MetricsBucket {
|
||||
pub ts: chrono::DateTime<chrono::Utc>,
|
||||
pub total: i64,
|
||||
pub errors: i64,
|
||||
}
|
||||
|
||||
/// Aggregate execution metrics for one app over a trailing window — the read
|
||||
/// model behind the observability dashboard. Sourced entirely from
|
||||
/// `execution_logs` (no hot-path instrumentation): counts + error rate +
|
||||
/// latency percentiles + an hourly series.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct MetricsSummary {
|
||||
pub window_hours: i32,
|
||||
pub total: i64,
|
||||
pub errors: i64,
|
||||
/// Errors / total in `[0, 1]` (0 when there are no executions).
|
||||
pub error_rate: f64,
|
||||
pub avg_duration_ms: f64,
|
||||
pub p50_duration_ms: f64,
|
||||
pub p95_duration_ms: f64,
|
||||
pub by_status: Vec<MetricsCount>,
|
||||
pub by_source: Vec<MetricsCount>,
|
||||
pub series: Vec<MetricsBucket>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ExecutionLogRepository: Send + Sync {
|
||||
/// F-P-005: keyset cursor on `(created_at, id)` — `cursor` is the
|
||||
@@ -743,6 +777,14 @@ pub trait ExecutionLogRepository: Send + Sync {
|
||||
cursor: Option<ExecutionLogCursor>,
|
||||
source: Option<ExecutionSource>,
|
||||
) -> Result<Vec<ExecutionLog>, ScriptRepositoryError>;
|
||||
|
||||
/// A2: aggregate metrics for `app_id` over the trailing `window_hours`.
|
||||
/// Read-only rollup over `execution_logs`.
|
||||
async fn summarize_for_app(
|
||||
&self,
|
||||
app_id: AppId,
|
||||
window_hours: i32,
|
||||
) -> Result<MetricsSummary, ScriptRepositoryError>;
|
||||
}
|
||||
|
||||
pub struct PostgresExecutionLogRepository {
|
||||
@@ -813,6 +855,129 @@ impl ExecutionLogRepository for PostgresExecutionLogRepository {
|
||||
|
||||
Ok(rows.into_iter().map(Into::into).collect())
|
||||
}
|
||||
|
||||
async fn summarize_for_app(
|
||||
&self,
|
||||
app_id: AppId,
|
||||
window_hours: i32,
|
||||
) -> Result<MetricsSummary, ScriptRepositoryError> {
|
||||
// Clamp the window to something sane (1 hour … 90 days) so a crafted
|
||||
// `?window=` can't ask for an unbounded scan.
|
||||
let window_hours = window_hours.clamp(1, 24 * 90);
|
||||
let app = app_id.into_inner();
|
||||
|
||||
// Headline aggregate: totals, error count, latency mean + percentiles.
|
||||
// `<> 'success'` counts every non-success terminal state as an error.
|
||||
let agg = sqlx::query_as::<_, AggRow>(
|
||||
"SELECT count(*)::int8 AS total, \
|
||||
count(*) FILTER (WHERE status <> 'success')::int8 AS errors, \
|
||||
COALESCE(avg(duration_ms), 0)::float8 AS avg_ms, \
|
||||
COALESCE(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 AS p50, \
|
||||
COALESCE(percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 AS p95 \
|
||||
FROM execution_logs \
|
||||
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour')",
|
||||
)
|
||||
.bind(app)
|
||||
.bind(window_hours)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
let by_status = sqlx::query_as::<_, CountRow>(
|
||||
"SELECT status AS key, count(*)::int8 AS count \
|
||||
FROM execution_logs \
|
||||
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
|
||||
GROUP BY status ORDER BY count DESC",
|
||||
)
|
||||
.bind(app)
|
||||
.bind(window_hours)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
let by_source = sqlx::query_as::<_, CountRow>(
|
||||
"SELECT source AS key, count(*)::int8 AS count \
|
||||
FROM execution_logs \
|
||||
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
|
||||
GROUP BY source ORDER BY count DESC",
|
||||
)
|
||||
.bind(app)
|
||||
.bind(window_hours)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
let series = sqlx::query_as::<_, BucketRow>(
|
||||
"SELECT date_trunc('hour', created_at) AS ts, count(*)::int8 AS total, \
|
||||
count(*) FILTER (WHERE status <> 'success')::int8 AS errors \
|
||||
FROM execution_logs \
|
||||
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
|
||||
GROUP BY ts ORDER BY ts",
|
||||
)
|
||||
.bind(app)
|
||||
.bind(window_hours)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
// Counts are execution tallies over a bounded window — always far below
|
||||
// f64's 2^53 exact-integer range, so the cast is lossless in practice.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let error_rate = if agg.total > 0 {
|
||||
agg.errors as f64 / agg.total as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
Ok(MetricsSummary {
|
||||
window_hours,
|
||||
total: agg.total,
|
||||
errors: agg.errors,
|
||||
error_rate,
|
||||
avg_duration_ms: agg.avg_ms,
|
||||
p50_duration_ms: agg.p50,
|
||||
p95_duration_ms: agg.p95,
|
||||
by_status: by_status
|
||||
.into_iter()
|
||||
.map(|r| MetricsCount {
|
||||
key: r.key,
|
||||
count: r.count,
|
||||
})
|
||||
.collect(),
|
||||
by_source: by_source
|
||||
.into_iter()
|
||||
.map(|r| MetricsCount {
|
||||
key: r.key,
|
||||
count: r.count,
|
||||
})
|
||||
.collect(),
|
||||
series: series
|
||||
.into_iter()
|
||||
.map(|r| MetricsBucket {
|
||||
ts: r.ts,
|
||||
total: r.total,
|
||||
errors: r.errors,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
struct AggRow {
|
||||
total: i64,
|
||||
errors: i64,
|
||||
avg_ms: f64,
|
||||
p50: f64,
|
||||
p95: f64,
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
struct CountRow {
|
||||
key: String,
|
||||
count: i64,
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
struct BucketRow {
|
||||
ts: chrono::DateTime<chrono::Utc>,
|
||||
total: i64,
|
||||
errors: i64,
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
|
||||
Reference in New Issue
Block a user