fabi/PiCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(metrics): per-app observability dashboard over execution_logs (A2)

Browse Source 
Aggregates the existing execution_logs table (no hot-path instrumentation):
ExecutionLogRepository::summarize_for_app returns counts, error rate, latency
avg/p50/p95 (percentile_cont), by-status/by-source breakdowns, and an hourly
series over a trailing window (clamped 1h..90d). New metrics_api router at
GET /api/v1/admin/apps/{id}/metrics?window= (AppLogRead), and a dashboard
Metrics tab (summary cards + an inline-SVG per-hour bar chart, no JS dep).

Pinned by a manager-core test asserting the exact rollup (percentiles included)
and the empty-app zero case.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-15 21:42:18 +02:00
parent 35345e62de
commit 6b2dcd41a6
10 changed files with 774 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/manager-core/src/lib.rs
View File

@@ -78,6 +78,7 @@ pub mod kv_service;
pub mod log_sink;
pub mod login_rate_limit;
pub mod materialize;
pub mod metrics_api;
pub mod migrations;
pub mod module_source;
pub mod outbox_event_emitter;
@@ -231,6 +232,7 @@ pub use kv_api::{kv_admin_router, KvAdminState};
pub use kv_repo::{KvRepo, KvRepoError, PostgresKvRepo};
pub use kv_service::KvServiceImpl;
pub use log_sink::PostgresExecutionLogSink;
pub use metrics_api::{metrics_router, MetricsState};
pub use module_source::PostgresModuleSource;
pub use outbox_event_emitter::OutboxEventEmitter;
pub use outbox_repo::{

118
crates/manager-core/src/metrics_api.rs Normal file
View File

@@ -0,0 +1,118 @@
//! `/api/v1/admin/apps/{id}/metrics` — read-only observability rollup (A2).
//!
//! Aggregates the existing `execution_logs` table (counts, error rate, latency
//! percentiles, an hourly series) for one app over a trailing window. No
//! hot-path instrumentation: the data plane already persists `duration_ms` /
//! `status` / `source` / `created_at` per run, so this is a pure read model
//! behind the dashboard's Metrics tab.
//!
//! Capability: `AppLogRead` (the same tier as the per-script log view),
//! resolved against the app loaded from the path.
use std::sync::Arc;
use axum::extract::{Path, Query, State};
use axum::response::{IntoResponse, Json, Response};
use axum::routing::get;
use axum::{Extension, Router};
use picloud_shared::{AppId, Principal};
use serde::Deserialize;
use serde_json::json;
use crate::app_repo::AppRepository;
use crate::authz::{require, AuthzDenied, AuthzRepo, Capability};
use crate::repo::{ExecutionLogRepository, MetricsSummary};
#[derive(Clone)]
pub struct MetricsState {
pub logs: Arc<dyn ExecutionLogRepository>,
pub apps: Arc<dyn AppRepository>,
pub authz: Arc<dyn AuthzRepo>,
}
pub fn metrics_router(state: MetricsState) -> Router {
Router::new()
.route("/apps/{app_id}/metrics", get(summary))
.with_state(state)
}
/// Default rollup window when `?window=` is absent (last 24 hours).
const DEFAULT_WINDOW_HOURS: i32 = 24;
#[derive(Debug, Deserialize)]
pub struct MetricsQuery {
/// Trailing window in hours (clamped to `[1, 2160]` by the repo).
#[serde(default)]
pub window: Option<i32>,
}
async fn summary(
State(s): State<MetricsState>,
Extension(principal): Extension<Principal>,
Path(id_or_slug): Path<String>,
Query(q): Query<MetricsQuery>,
) -> Result<Json<MetricsSummary>, MetricsApiError> {
let app_id = resolve_app(&*s.apps, &id_or_slug).await?;
require(s.authz.as_ref(), &principal, Capability::AppLogRead(app_id)).await?;
let window = q.window.unwrap_or(DEFAULT_WINDOW_HOURS);
let summary = s
.logs
.summarize_for_app(app_id, window)
.await
.map_err(|e| MetricsApiError::Backend(e.to_string()))?;
Ok(Json(summary))
}
async fn resolve_app(apps: &dyn AppRepository, ident: &str) -> Result<AppId, MetricsApiError> {
crate::app_repo::resolve_app(apps, ident)
.await
.map_err(|e| MetricsApiError::Backend(e.to_string()))?
.map(|l| l.app.id)
.ok_or(MetricsApiError::AppNotFound)
}
#[derive(Debug, thiserror::Error)]
pub enum MetricsApiError {
#[error("app not found")]
AppNotFound,
#[error("forbidden")]
Forbidden,
#[error("authorization repo error: {0}")]
AuthzRepo(String),
#[error("metrics backend: {0}")]
Backend(String),
}
impl From<AuthzDenied> for MetricsApiError {
fn from(d: AuthzDenied) -> Self {
match d {
AuthzDenied::Denied => Self::Forbidden,
AuthzDenied::Repo(e) => Self::AuthzRepo(e.to_string()),
}
}
}
impl IntoResponse for MetricsApiError {
fn into_response(self) -> Response {
use axum::http::StatusCode;
let (status, body) = match &self {
Self::AppNotFound => (StatusCode::NOT_FOUND, json!({ "error": self.to_string() })),
Self::Forbidden => (StatusCode::FORBIDDEN, json!({ "error": self.to_string() })),
Self::AuthzRepo(e) => {
tracing::error!(error = %e, "metrics admin authz error");
(
StatusCode::INTERNAL_SERVER_ERROR,
json!({ "error": "internal error" }),
)
}
Self::Backend(e) => {
tracing::error!(error = %e, "metrics admin backend error");
(
StatusCode::INTERNAL_SERVER_ERROR,
json!({ "error": "internal error" }),
)
}
};
(status, Json(body)).into_response()
}
}

165
crates/manager-core/src/repo.rs
View File

@@ -731,6 +731,40 @@ impl ExecutionLogCursor {
}
}
/// One `(key, count)` row of a metrics breakdown (by status or by source).
#[derive(Debug, Clone, serde::Serialize)]
pub struct MetricsCount {
pub key: String,
pub count: i64,
}
/// One hour-bucket of the metrics time series.
#[derive(Debug, Clone, serde::Serialize)]
pub struct MetricsBucket {
pub ts: chrono::DateTime<chrono::Utc>,
pub total: i64,
pub errors: i64,
}
/// Aggregate execution metrics for one app over a trailing window — the read
/// model behind the observability dashboard. Sourced entirely from
/// `execution_logs` (no hot-path instrumentation): counts + error rate +
/// latency percentiles + an hourly series.
#[derive(Debug, Clone, serde::Serialize)]
pub struct MetricsSummary {
pub window_hours: i32,
pub total: i64,
pub errors: i64,
/// Errors / total in `[0, 1]` (0 when there are no executions).
pub error_rate: f64,
pub avg_duration_ms: f64,
pub p50_duration_ms: f64,
pub p95_duration_ms: f64,
pub by_status: Vec<MetricsCount>,
pub by_source: Vec<MetricsCount>,
pub series: Vec<MetricsBucket>,
}
#[async_trait]
pub trait ExecutionLogRepository: Send + Sync {
/// F-P-005: keyset cursor on `(created_at, id)` — `cursor` is the
@@ -743,6 +777,14 @@ pub trait ExecutionLogRepository: Send + Sync {
cursor: Option<ExecutionLogCursor>,
source: Option<ExecutionSource>,
) -> Result<Vec<ExecutionLog>, ScriptRepositoryError>;
/// A2: aggregate metrics for `app_id` over the trailing `window_hours`.
/// Read-only rollup over `execution_logs`.
async fn summarize_for_app(
&self,
app_id: AppId,
window_hours: i32,
) -> Result<MetricsSummary, ScriptRepositoryError>;
}
pub struct PostgresExecutionLogRepository {
@@ -813,6 +855,129 @@ impl ExecutionLogRepository for PostgresExecutionLogRepository {
Ok(rows.into_iter().map(Into::into).collect())
}
async fn summarize_for_app(
&self,
app_id: AppId,
window_hours: i32,
) -> Result<MetricsSummary, ScriptRepositoryError> {
// Clamp the window to something sane (1 hour … 90 days) so a crafted
// `?window=` can't ask for an unbounded scan.
let window_hours = window_hours.clamp(1, 24 * 90);
let app = app_id.into_inner();
// Headline aggregate: totals, error count, latency mean + percentiles.
// `<> 'success'` counts every non-success terminal state as an error.
let agg = sqlx::query_as::<_, AggRow>(
"SELECT count(*)::int8 AS total, \
count(*) FILTER (WHERE status <> 'success')::int8 AS errors, \
COALESCE(avg(duration_ms), 0)::float8 AS avg_ms, \
COALESCE(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 AS p50, \
COALESCE(percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 AS p95 \
FROM execution_logs \
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour')",
)
.bind(app)
.bind(window_hours)
.fetch_one(&self.pool)
.await?;
let by_status = sqlx::query_as::<_, CountRow>(
"SELECT status AS key, count(*)::int8 AS count \
FROM execution_logs \
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
GROUP BY status ORDER BY count DESC",
)
.bind(app)
.bind(window_hours)
.fetch_all(&self.pool)
.await?;
let by_source = sqlx::query_as::<_, CountRow>(
"SELECT source AS key, count(*)::int8 AS count \
FROM execution_logs \
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
GROUP BY source ORDER BY count DESC",
)
.bind(app)
.bind(window_hours)
.fetch_all(&self.pool)
.await?;
let series = sqlx::query_as::<_, BucketRow>(
"SELECT date_trunc('hour', created_at) AS ts, count(*)::int8 AS total, \
count(*) FILTER (WHERE status <> 'success')::int8 AS errors \
FROM execution_logs \
WHERE app_id = $1 AND created_at >= now() - ($2 * interval '1 hour') \
GROUP BY ts ORDER BY ts",
)
.bind(app)
.bind(window_hours)
.fetch_all(&self.pool)
.await?;
// Counts are execution tallies over a bounded window — always far below
// f64's 2^53 exact-integer range, so the cast is lossless in practice.
#[allow(clippy::cast_precision_loss)]
let error_rate = if agg.total > 0 {
agg.errors as f64 / agg.total as f64
} else {
0.0
};
Ok(MetricsSummary {
window_hours,
total: agg.total,
errors: agg.errors,
error_rate,
avg_duration_ms: agg.avg_ms,
p50_duration_ms: agg.p50,
p95_duration_ms: agg.p95,
by_status: by_status
.into_iter()
.map(|r| MetricsCount {
key: r.key,
count: r.count,
})
.collect(),
by_source: by_source
.into_iter()
.map(|r| MetricsCount {
key: r.key,
count: r.count,
})
.collect(),
series: series
.into_iter()
.map(|r| MetricsBucket {
ts: r.ts,
total: r.total,
errors: r.errors,
})
.collect(),
})
}
}
#[derive(sqlx::FromRow)]
struct AggRow {
total: i64,
errors: i64,
avg_ms: f64,
p50: f64,
p95: f64,
}
#[derive(sqlx::FromRow)]
struct CountRow {
key: String,
count: i64,
}
#[derive(sqlx::FromRow)]
struct BucketRow {
ts: chrono::DateTime<chrono::Utc>,
total: i64,
errors: i64,
}
#[derive(sqlx::FromRow)]

150
crates/manager-core/tests/metrics_summary.rs Normal file
View File

@@ -0,0 +1,150 @@
//! A2: `ExecutionLogRepository::summarize_for_app` — the read model behind the
//! observability dashboard. Seeds a known set of `execution_logs` rows for one
//! app and asserts the rollup: totals, error rate, latency mean + percentiles,
//! the by-status / by-source breakdowns, and the hourly series. Scoped to a
//! unique `app_id`, so it is unaffected by any other rows in a shared DB.
//! Skips cleanly when `DATABASE_URL` is unset.
use picloud_manager_core::repo::{ExecutionLogRepository, PostgresExecutionLogRepository};
use picloud_shared::AppId;
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
use uuid::Uuid;
async fn pool_or_skip() -> Option<PgPool> {
let Ok(url) = std::env::var("DATABASE_URL") else {
picloud_test_support::abort_if_db_required("metrics_summary");
eprintln!("metrics_summary: DATABASE_URL unset — skipping");
return None;
};
let pool = PgPoolOptions::new()
.max_connections(2)
.connect(&url)
.await
.expect("connect to DATABASE_URL");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("apply migrations");
Some(pool)
}
async fn make_app(pool: &PgPool) -> Uuid {
// `apps.group_id` is NOT NULL, so anchor the app under a fresh root group.
let gslug = format!("metrics-g-{}", Uuid::new_v4());
let group: (Uuid,) =
sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
.bind(&gslug)
.fetch_one(pool)
.await
.expect("group insert");
let slug = format!("metrics-{}", Uuid::new_v4());
let row: (Uuid,) =
sqlx::query_as("INSERT INTO apps (slug, name, group_id) VALUES ($1, $1, $2) RETURNING id")
.bind(&slug)
.bind(group.0)
.fetch_one(pool)
.await
.expect("app insert");
row.0
}
async fn seed(pool: &PgPool, app: Uuid, duration_ms: i32, status: &str, source: &str) {
sqlx::query(
"INSERT INTO execution_logs (app_id, request_id, duration_ms, status, source) \
VALUES ($1, gen_random_uuid(), $2, $3, $4)",
)
.bind(app)
.bind(duration_ms)
.bind(status)
.bind(source)
.execute(pool)
.await
.expect("seed execution_logs row");
}
fn count_of(counts: &[picloud_manager_core::repo::MetricsCount], key: &str) -> i64 {
counts.iter().find(|c| c.key == key).map_or(0, |c| c.count)
}
#[tokio::test]
async fn summarize_rolls_up_counts_error_rate_and_percentiles() {
let Some(pool) = pool_or_skip().await else {
return;
};
let app = make_app(&pool).await;
// 3 successes (http), 1 error (kv), 1 timeout (cron). Durations chosen so
// the percentiles are exact: sorted [10, 20, 30, 50, 100].
seed(&pool, app, 10, "success", "http").await;
seed(&pool, app, 20, "success", "http").await;
seed(&pool, app, 30, "success", "http").await;
seed(&pool, app, 100, "error", "kv").await;
seed(&pool, app, 50, "timeout", "cron").await;
let repo = PostgresExecutionLogRepository::new(pool.clone());
let s = repo
.summarize_for_app(AppId::from(app), 24)
.await
.expect("summarize");
assert_eq!(s.total, 5, "total executions");
// Every non-success terminal state counts as an error: error + timeout = 2.
assert_eq!(s.errors, 2, "error count");
assert!(
(s.error_rate - 0.4).abs() < 1e-9,
"error rate: {}",
s.error_rate
);
// mean of [10,20,30,100,50] = 42.
assert!(
(s.avg_duration_ms - 42.0).abs() < 1e-6,
"avg: {}",
s.avg_duration_ms
);
// percentile_cont(0.5) of the 5 sorted values = 30.
assert!(
(s.p50_duration_ms - 30.0).abs() < 1e-6,
"p50: {}",
s.p50_duration_ms
);
// percentile_cont(0.95): 0.95*(n-1)=3.8 → interpolate 50..100 → 90.
assert!(
(s.p95_duration_ms - 90.0).abs() < 1e-6,
"p95: {}",
s.p95_duration_ms
);
assert_eq!(count_of(&s.by_status, "success"), 3);
assert_eq!(count_of(&s.by_status, "error"), 1);
assert_eq!(count_of(&s.by_status, "timeout"), 1);
assert_eq!(count_of(&s.by_source, "http"), 3);
assert_eq!(count_of(&s.by_source, "kv"), 1);
assert_eq!(count_of(&s.by_source, "cron"), 1);
// All rows land in the current hour bucket; the series sums back to the total.
assert!(!s.series.is_empty(), "series has at least one bucket");
assert_eq!(s.series.iter().map(|b| b.total).sum::<i64>(), 5);
assert_eq!(s.series.iter().map(|b| b.errors).sum::<i64>(), 2);
}
#[tokio::test]
async fn summarize_of_an_empty_app_is_all_zero() {
let Some(pool) = pool_or_skip().await else {
return;
};
let app = make_app(&pool).await;
let repo = PostgresExecutionLogRepository::new(pool.clone());
let s = repo
.summarize_for_app(AppId::from(app), 24)
.await
.expect("summarize");
assert_eq!(s.total, 0);
assert_eq!(s.errors, 0);
assert!(
(s.error_rate - 0.0).abs() < 1e-9,
"no executions → zero rate"
);
assert!(s.series.is_empty());
}

44
crates/picloud/src/lib.rs
View File

@@ -13,8 +13,8 @@ use picloud_manager_core::{
admin_router, admins_router, api_keys_router, app_members_router, apply_router, apps_api,
apps_router, attach_principal_if_present, auth_router, dead_letters_router, dev_emails_router,
docs_admin_router, email_inbound_router, files_admin_router, groups_router, kv_admin_router,
migrations, projects_router, rebuild_route_table, require_authenticated, route_admin_router,
secrets_router, topics_router, triggers_router, vars_router, AbandonedRepo,
metrics_router, migrations, projects_router, rebuild_route_table, require_authenticated,
route_admin_router, secrets_router, topics_router, triggers_router, vars_router, AbandonedRepo,
AdminPrincipalResolver, AdminSessionRepository, AdminState, AdminUserRepository, AdminsState,
ApiKeyRepository, ApiKeysState, AppDomainRepository, AppMembersRepository, AppMembersState,
AppRepository, ApplyService, AppsState, AuthState, AuthzRepo, DeadLetterRepo, DeadLettersState,
@@ -23,22 +23,22 @@ use picloud_manager_core::{
FsGroupFilesRepo, GroupDocsServiceImpl, GroupFilesServiceImpl, GroupKvServiceImpl,
GroupMembersRepository, GroupPubsubServiceImpl, GroupQueueServiceImpl, GroupRepository,
GroupsState, HttpConfig, HttpServiceImpl, InboundNonceDedup, KvAdminState, KvServiceImpl,
OutboxEventEmitter, OutboxRepo, PostgresAbandonedRepo, PostgresAdminSessionRepository,
PostgresAdminUserRepository, PostgresApiKeyRepository, PostgresAppDomainRepository,
PostgresAppMembersRepository, PostgresAppRepository, PostgresAppSecretsRepo,
PostgresAppUserInvitationRepo, PostgresAppUserPasswordResetRepo, PostgresAppUserRepository,
PostgresAppUserRoleRepo, PostgresAppUserSessionRepository, PostgresAppUserVerificationRepo,
PostgresDeadLetterRepo, PostgresDeadLetterService, PostgresDocsRepo,
PostgresExecutionLogRepository, PostgresExecutionLogSink, PostgresGroupCollectionResolver,
PostgresGroupDocsRepo, PostgresGroupKvRepo, PostgresGroupMembersRepository,
PostgresGroupQueueRepo, PostgresGroupRepository, PostgresKvRepo, PostgresOutboxRepo,
PostgresProjectRepository, PostgresPubsubRepo, PostgresRouteRepository,
PostgresScriptRepository, PostgresSecretsRepo, PostgresTopicRepo, PostgresTriggerRepo,
PostgresVarsRepo, PrincipalResolver, ProjectRepository, ProjectsState, PubsubServiceImpl,
RealtimeAuthorityImpl, RepoResolver, RouteAdminState, SandboxCeiling, ScriptRepository,
SecretsConfig, SecretsServiceImpl, SecretsState, SubscriberTokenConfig, TopicRepo, TopicsState,
TriggerConfig, TriggerRepo, TriggersState, UsersServiceConfig, UsersServiceImpl, VarsApiState,
VarsServiceImpl,
MetricsState, OutboxEventEmitter, OutboxRepo, PostgresAbandonedRepo,
PostgresAdminSessionRepository, PostgresAdminUserRepository, PostgresApiKeyRepository,
PostgresAppDomainRepository, PostgresAppMembersRepository, PostgresAppRepository,
PostgresAppSecretsRepo, PostgresAppUserInvitationRepo, PostgresAppUserPasswordResetRepo,
PostgresAppUserRepository, PostgresAppUserRoleRepo, PostgresAppUserSessionRepository,
PostgresAppUserVerificationRepo, PostgresDeadLetterRepo, PostgresDeadLetterService,
PostgresDocsRepo, PostgresExecutionLogRepository, PostgresExecutionLogSink,
PostgresGroupCollectionResolver, PostgresGroupDocsRepo, PostgresGroupKvRepo,
PostgresGroupMembersRepository, PostgresGroupQueueRepo, PostgresGroupRepository,
PostgresKvRepo, PostgresOutboxRepo, PostgresProjectRepository, PostgresPubsubRepo,
PostgresRouteRepository, PostgresScriptRepository, PostgresSecretsRepo, PostgresTopicRepo,
PostgresTriggerRepo, PostgresVarsRepo, PrincipalResolver, ProjectRepository, ProjectsState,
PubsubServiceImpl, RealtimeAuthorityImpl, RepoResolver, RouteAdminState, SandboxCeiling,
ScriptRepository, SecretsConfig, SecretsServiceImpl, SecretsState, SubscriberTokenConfig,
TopicRepo, TopicsState, TriggerConfig, TriggerRepo, TriggersState, UsersServiceConfig,
UsersServiceImpl, VarsApiState, VarsServiceImpl,
};
use picloud_orchestrator_core::realtime::DEFAULT_GC_INTERVAL_SECS;
use picloud_orchestrator_core::routing::{AppDomainTable, RouteTable};
@@ -553,7 +553,7 @@ pub async fn build_app(
let admin = AdminState {
repo: script_repo.clone(),
logs: log_repo,
logs: log_repo.clone(),
apps: apps_repo.clone(),
authz: authz.clone(),
validator: engine.clone(),
@@ -810,6 +810,12 @@ pub async fn build_app(
apps: apps_repo.clone(),
authz: authz.clone(),
}))
// A2: read-only observability rollup over execution_logs.
.merge(metrics_router(MetricsState {
logs: log_repo.clone(),
apps: apps_repo.clone(),
authz: authz.clone(),
}))
// §11.6 M4: read-only operator inspection of a group's shared collections.
.merge(picloud_manager_core::group_blobs_api::group_blobs_router(
picloud_manager_core::group_blobs_api::GroupBlobsState {