fabi/PiCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v1.1.6): realtime channels + v1.1.5 follow-ups + version bumps

Browse Source 
Server-side realtime SSE on per-app pub/sub topics, plus the three
v1.1.5 follow-ups and the version bumps.

Realtime:
- topics registry (0021) + admin endpoints + Capability::AppTopicManage
  (-> app:admin; no new scope).
- GET /realtime/topics/{topic} SSE endpoint (orchestrator-core data
  plane): Host -> app, RealtimeAuthority gate (404 missing/internal,
  401 bad/absent token), broadcast::Receiver stream + heartbeat.
- RealtimeBroadcaster / RealtimeEvent / RealtimeAuthority traits
  (picloud-shared); InProcessBroadcaster + GC (orchestrator-core);
  DB-backed RealtimeAuthorityImpl (manager-core). Publish path fans out
  to in-process subscribers after the durable outbox commit (best-effort,
  panic-isolated).
- HMAC subscriber tokens (subscriber_token.rs) + app_secrets table (0022)
  + pubsub::subscriber_token SDK (schema 1.6 -> 1.7). TTL clamp + env
  overrides.
- Dashboard Topics tab (register/list/edit/delete, prominent external
  badge, flip confirmation).

v1.1.5 follow-ups:
- Empty blobs accepted (NewFile/FileUpdate::validate) + round-trip test.
- Orphan *.tmp.* sweeper (spawn_files_orphan_sweep).
- Dispatcher e2e tests, one per trigger kind (DATABASE_URL-gated).

Versions: workspace 1.1.6, SDK 1.7, dashboard 0.12.0. Schema-snapshot
golden re-blessed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-04 20:18:50 +02:00
parent d064681c49
commit fcbcc576a2
35 changed files with 4333 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
crates/picloud/src/lib.rs
View File

@@ -12,29 +12,33 @@ use picloud_executor_core::{Engine, Limits};
use picloud_manager_core::{
admin_router, admins_router, api_keys_router, app_members_router, apps_api, apps_router,
attach_principal_if_present, auth_router, compile_routes, dead_letters_router,
files_admin_router, migrations, require_authenticated, route_admin_router, triggers_router,
AbandonedRepo, AdminPrincipalResolver, AdminSessionRepository, AdminState, AdminUserRepository,
AdminsState, ApiKeyRepository, ApiKeysState, AppDomainRepository, AppMembersRepository,
AppMembersState, AppRepository, AppsState, AuthState, AuthzRepo, DeadLetterRepo,
DeadLettersState, Dispatcher, DocsServiceImpl, FilesAdminState, FilesConfig, FilesServiceImpl,
FsFilesRepo, HttpConfig, HttpServiceImpl, KvServiceImpl, OutboxEventEmitter, OutboxRepo,
PostgresAbandonedRepo, PostgresAdminSessionRepository, PostgresAdminUserRepository,
files_admin_router, migrations, require_authenticated, route_admin_router, topics_router,
triggers_router, AbandonedRepo, AdminPrincipalResolver, AdminSessionRepository, AdminState,
AdminUserRepository, AdminsState, ApiKeyRepository, ApiKeysState, AppDomainRepository,
AppMembersRepository, AppMembersState, AppRepository, AppsState, AuthState, AuthzRepo,
DeadLetterRepo, DeadLettersState, Dispatcher, DocsServiceImpl, FilesAdminState, FilesConfig,
FilesServiceImpl, FsFilesRepo, HttpConfig, HttpServiceImpl, KvServiceImpl, OutboxEventEmitter,
OutboxRepo, PostgresAbandonedRepo, PostgresAdminSessionRepository, PostgresAdminUserRepository,
PostgresApiKeyRepository, PostgresAppDomainRepository, PostgresAppMembersRepository,
PostgresAppRepository, PostgresDeadLetterRepo, PostgresDeadLetterService, PostgresDocsRepo,
PostgresExecutionLogRepository, PostgresExecutionLogSink, PostgresKvRepo, PostgresOutboxRepo,
PostgresPubsubRepo, PostgresRouteRepository, PostgresScriptRepository, PostgresTriggerRepo,
PrincipalResolver, PubsubServiceImpl, RepoResolver, RouteAdminState, RouteRepository,
SandboxCeiling, ScriptRepository, TriggerConfig, TriggerRepo, TriggersState,
PostgresAppRepository, PostgresAppSecretsRepo, PostgresDeadLetterRepo,
PostgresDeadLetterService, PostgresDocsRepo, PostgresExecutionLogRepository,
PostgresExecutionLogSink, PostgresKvRepo, PostgresOutboxRepo, PostgresPubsubRepo,
PostgresRouteRepository, PostgresScriptRepository, PostgresTopicRepo, PostgresTriggerRepo,
PrincipalResolver, PubsubServiceImpl, RealtimeAuthorityImpl, RepoResolver, RouteAdminState,
RouteRepository, SandboxCeiling, ScriptRepository, SubscriberTokenConfig, TopicRepo,
TopicsState, TriggerConfig, TriggerRepo, TriggersState,
};
use picloud_orchestrator_core::realtime::DEFAULT_GC_INTERVAL_SECS;
use picloud_orchestrator_core::routing::{AppDomainTable, RouteTable};
use picloud_orchestrator_core::{
data_plane_router, user_routes_router, DataPlaneState, ExecutionGate, InboxRegistry,
LocalExecutorClient,
data_plane_router, realtime_router, spawn_realtime_gc, user_routes_router, DataPlaneState,
ExecutionGate, InProcessBroadcaster, InboxRegistry, LocalExecutorClient, RealtimeState,
};
use picloud_shared::{
DeadLetterService, DocsService, ExecutionLogSink, FilesService, HttpService, InboxResolver,
KvService, OutboxWriter, PubsubService, ScriptValidator, ServiceEventEmitter, Services,
API_VERSION, PRODUCT_VERSION, SDK_VERSION, WIRE_VERSION,
KvService, OutboxWriter, PubsubService, RealtimeAuthority, RealtimeBroadcaster,
ScriptValidator, ServiceEventEmitter, Services, API_VERSION, PRODUCT_VERSION, SDK_VERSION,
WIRE_VERSION,
};
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
@@ -162,6 +166,8 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
// the bytes live on disk under `PICLOUD_FILES_ROOT` (default ./data).
let files_config = FilesConfig::from_env();
let files_max_size = files_config.max_file_size_bytes;
// Kept for the v1.1.6 orphan sweeper (cleans stale `*.tmp.*` files).
let files_root = files_config.root.clone();
let files_repo = Arc::new(FsFilesRepo::new(pool.clone(), files_config));
let files: Arc<dyn FilesService> = Arc::new(FilesServiceImpl::new(
files_repo.clone(),
@@ -169,12 +175,34 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
events.clone(),
files_max_size,
));
// v1.1.5 durable pub/sub. Publishes fan out to matching pubsub
// triggers at publish time (one outbox row each), delivered by the
// same dispatcher as every other async trigger.
// v1.1.6 realtime: the in-process broadcaster is shared between the
// publish path (PubsubServiceImpl fans out to SSE subscribers after
// the durable outbox fan-out) and the SSE endpoint (subscribe side).
// The topic registry + app-secrets repo back the subscriber-token
// mint + SSE subscribe-authorization.
let broadcaster_concrete = Arc::new(InProcessBroadcaster::from_env());
let broadcaster: Arc<dyn RealtimeBroadcaster> = broadcaster_concrete.clone();
let topic_repo: Arc<dyn TopicRepo> = Arc::new(PostgresTopicRepo::new(pool.clone()));
let app_secrets_repo = Arc::new(PostgresAppSecretsRepo::new(pool.clone()));
let realtime_authority: Arc<dyn RealtimeAuthority> = Arc::new(RealtimeAuthorityImpl::new(
topic_repo.clone(),
app_secrets_repo.clone(),
));
// v1.1.5 durable pub/sub, extended in v1.1.6 with the realtime
// broadcast + subscriber-token mint. Publishes fan out to matching
// pubsub triggers at publish time (one outbox row each, delivered by
// the same dispatcher as every other async trigger) AND, best-effort,
// to in-process SSE subscribers.
let pubsub_repo = Arc::new(PostgresPubsubRepo::new(pool.clone()));
let pubsub: Arc<dyn PubsubService> =
Arc::new(PubsubServiceImpl::new(pubsub_repo, authz.clone()));
let pubsub: Arc<dyn PubsubService> = Arc::new(
PubsubServiceImpl::new(pubsub_repo, authz.clone()).with_realtime(
broadcaster.clone(),
topic_repo.clone(),
app_secrets_repo,
SubscriberTokenConfig::from_env(),
),
);
let services = Services::new(
kv,
docs,
@@ -284,6 +312,10 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
// enqueues due triggers into the outbox; the dispatcher above
// delivers them like any other async trigger.
picloud_manager_core::spawn_cron_scheduler(pool, trigger_config.cron_tick_interval_ms);
// v1.1.6: GC empty realtime broadcast channels (one-shot subscribers)
// and sweep orphaned `*.tmp.*` blobs left by crashed file writes.
spawn_realtime_gc(broadcaster_concrete, DEFAULT_GC_INTERVAL_SECS);
picloud_manager_core::spawn_files_orphan_sweep(files_root);
let triggers_state = TriggersState {
triggers: trigger_repo,
apps: apps_repo.clone(),
@@ -302,11 +334,17 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
apps: apps_repo.clone(),
authz: authz.clone(),
};
let topics_state = TopicsState {
topics: topic_repo,
apps: apps_repo.clone(),
authz: authz.clone(),
broadcaster: broadcaster.clone(),
};
let apps_state = AppsState {
apps: apps_repo,
domains: domains_repo,
routes: route_repo,
domain_table: app_domain_table,
domain_table: app_domain_table.clone(),
authz: authz.clone(),
};
@@ -345,6 +383,7 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
.merge(api_keys_router(api_keys_state))
.merge(triggers_router(triggers_state))
.merge(files_admin_router(files_admin_state))
.merge(topics_router(topics_state))
.merge(dead_letters_router(dead_letters_state))
.layer(from_fn_with_state(
auth_state.clone(),
@@ -375,10 +414,21 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result<Router> {
.nest("/admin", guarded_admin)
.merge(data_plane_routed);
// v1.1.6 SSE realtime surface, merged at the root (deliberately NOT
// under /api/ — realtime is its own versioning surface). Public auth
// is per-topic; no principal middleware (token verification is the
// gate, handled inside the authority).
let realtime = realtime_router(RealtimeState::new(
app_domain_table,
broadcaster,
realtime_authority,
));
Ok(Router::new()
.route("/healthz", get(healthz))
.route("/version", get(version))
.nest(&format!("/api/v{API_VERSION}"), api_v1)
.merge(realtime)
.merge(user_routes)
.layer(TraceLayer::new_for_http()))
}

353
crates/picloud/tests/dispatcher_e2e.rs Normal file
View File

@@ -0,0 +1,353 @@
//! End-to-end dispatcher tests — one per trigger kind (v1.1.5 follow-up,
//! landed in v1.1.6). Each test wires the full all-in-one app via
//! `build_app` (which spawns the real dispatcher + cron scheduler +
//! executor), creates an app + a logging handler script + a trigger,
//! causes the originating event, and polls for the handler's side effect.
//!
//! ## Gating
//!
//! These need a Postgres reachable via `DATABASE_URL`. They follow the
//! `schema_snapshot` pattern (NOT `#[ignore]`): when `DATABASE_URL` is
//! unset the test prints a notice and returns early, so plain
//! `cargo test` stays green locally while CI (which sets `DATABASE_URL`)
//! runs them.
//!
//! ## How "the handler fired" is observed
//!
//! The dispatcher does not write `execution_log` rows for trigger
//! handlers, so each handler instead records its `ctx.event` into a KV
//! marker (`collection = "e2e_markers"`, which no trigger watches — no
//! recursion). The test polls `kv_entries` for that marker and asserts
//! the event shape. See HANDBACK §deviations for why this lives in
//! `picloud/tests/` rather than `manager-core/tests/` (build_app lives in
//! the `picloud` crate) and for the `dead_letter` reinterpretation.
#![allow(clippy::needless_pass_by_value)]
use std::time::Duration;
use axum_test::TestServer;
use serde_json::{json, Value};
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
use uuid::Uuid;
/// Connect + migrate, or return `None` (printing a skip notice) when
/// `DATABASE_URL` is unset — mirrors `schema_snapshot.rs`.
async fn pool_or_skip() -> Option<PgPool> {
let Ok(url) = std::env::var("DATABASE_URL") else {
eprintln!("dispatcher_e2e: DATABASE_URL unset — skipping");
return None;
};
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.expect("connect to DATABASE_URL");
sqlx::migrate!("../manager-core/migrations")
.run(&pool)
.await
.expect("apply migrations");
Some(pool)
}
/// Build the app over the shared pool with a uniquely-named owner admin,
/// log in, and create a fresh app. `suffix` must be unique per test (the
/// pool is shared, so names must not collide).
async fn server_for(pool: PgPool, suffix: &str) -> (TestServer, String) {
use picloud_manager_core::auth::hash_password;
use picloud_shared::InstanceRole;
let unique = format!("{suffix}-{}", Uuid::new_v4().simple());
let auth = picloud::AuthDeps::from_pool(pool.clone());
let username = format!("e2e-{unique}");
let hash = hash_password("pw").expect("hash");
auth.users
.create(&username, &hash, InstanceRole::Owner, None)
.await
.expect("seed admin");
let app = picloud::build_app(pool, auth).await.expect("build_app");
let mut server = TestServer::new(app).expect("TestServer");
let resp = server
.post("/api/v1/admin/auth/login")
.json(&json!({ "username": username, "password": "pw" }))
.await;
resp.assert_status_ok();
let token = resp.json::<Value>()["token"]
.as_str()
.expect("login token")
.to_string();
server.add_header("authorization", format!("Bearer {token}"));
// A fresh app keeps each test's KV / events isolated from siblings.
let slug = format!("e2e-{unique}");
let created: Value = server
.post("/api/v1/admin/apps")
.json(&json!({ "slug": slug, "name": slug }))
.await
.json();
let app_id = created["id"].as_str().expect("app id").to_string();
(server, app_id)
}
async fn create_script(server: &TestServer, app_id: &str, name: &str, source: &str) -> String {
let created: Value = server
.post("/api/v1/admin/scripts")
.json(&json!({ "app_id": app_id, "name": name, "source": source }))
.await
.json();
created["id"].as_str().expect("script id").to_string()
}
/// A handler that records its `ctx.event` into a KV marker the test can
/// observe. The marker collection is watched by no trigger.
const MARKER_HANDLER: &str = r#"
let e = ctx.event;
kv::collection("e2e_markers").set("marker", e);
#{ ok: true }
"#;
/// Poll the marker KV key until present (or ~10s timeout).
async fn poll_marker(pool: &PgPool, app_id: &str) -> Option<Value> {
poll_marker_n(pool, app_id, 100).await
}
/// Poll the marker KV key for `iters` × 100ms.
async fn poll_marker_n(pool: &PgPool, app_id: &str, iters: u32) -> Option<Value> {
let app_uuid = Uuid::parse_str(app_id).expect("app uuid");
for _ in 0..iters {
let row: Option<(Value,)> = sqlx::query_as(
"SELECT value FROM kv_entries \
WHERE app_id = $1 AND collection = 'e2e_markers' AND key = 'marker'",
)
.bind(app_uuid)
.fetch_optional(pool)
.await
.expect("query marker");
if let Some((value,)) = row {
return Some(value);
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
None
}
async fn execute(server: &TestServer, script_id: &str) {
server
.post(&format!("/api/v1/execute/{script_id}"))
.json(&json!({}))
.await
.assert_status_ok();
}
#[tokio::test]
async fn dispatcher_delivers_kv_to_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "kv").await;
let handler = create_script(&server, &app_id, "kv-handler", MARKER_HANDLER).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/kv"))
.json(&json!({ "script_id": handler, "collection_glob": "src" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
let source = create_script(
&server,
&app_id,
"kv-source",
r#"kv::collection("src").set("k", 42); #{ ok: true }"#,
)
.await;
execute(&server, &source).await;
let event = poll_marker(&pool, &app_id).await.expect("kv handler fired");
assert_eq!(event["source"], "kv");
assert_eq!(event["op"], "insert");
assert_eq!(event["kv"]["collection"], "src");
assert_eq!(event["kv"]["key"], "k");
}
#[tokio::test]
async fn dispatcher_delivers_docs_to_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "docs").await;
let handler = create_script(&server, &app_id, "docs-handler", MARKER_HANDLER).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/docs"))
.json(&json!({ "script_id": handler, "collection_glob": "src" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
let source = create_script(
&server,
&app_id,
"docs-source",
r#"docs::collection("src").create(#{ x: 1 }); #{ ok: true }"#,
)
.await;
execute(&server, &source).await;
let event = poll_marker(&pool, &app_id)
.await
.expect("docs handler fired");
assert_eq!(event["source"], "docs");
assert_eq!(event["op"], "create");
assert_eq!(event["docs"]["collection"], "src");
}
#[tokio::test]
async fn dispatcher_delivers_cron_to_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "cron").await;
let handler = create_script(&server, &app_id, "cron-handler", MARKER_HANDLER).await;
// Fire every second (6-field cron, seconds-resolution).
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/cron"))
.json(&json!({ "script_id": handler, "schedule": "* * * * * *", "timezone": "UTC" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
// No source — the scheduler enqueues the due tick on its own. The
// scheduler skips its first tick and then ticks every
// PICLOUD_CRON_TICK_INTERVAL_MS (default 30s), so poll past that
// (set the env var lower to speed CI up if desired).
let event = poll_marker_n(&pool, &app_id, 450)
.await
.expect("cron handler fired");
assert_eq!(event["source"], "cron");
assert_eq!(event["op"], "tick");
assert_eq!(event["cron"]["timezone"], "UTC");
}
#[tokio::test]
async fn dispatcher_delivers_files_to_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "files").await;
let handler = create_script(&server, &app_id, "files-handler", MARKER_HANDLER).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/files"))
.json(&json!({ "script_id": handler, "collection_glob": "src" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
let source = create_script(
&server,
&app_id,
"files-source",
r#"
let data = base64::decode("aGk=");
files::collection("src").create(#{ name: "f.txt", content_type: "text/plain", data: data });
#{ ok: true }
"#,
)
.await;
execute(&server, &source).await;
let event = poll_marker(&pool, &app_id)
.await
.expect("files handler fired");
assert_eq!(event["source"], "files");
assert_eq!(event["op"], "create");
assert_eq!(event["files"]["collection"], "src");
assert_eq!(event["files"]["name"], "f.txt");
}
#[tokio::test]
async fn dispatcher_delivers_pubsub_to_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "pubsub").await;
let handler = create_script(&server, &app_id, "pubsub-handler", MARKER_HANDLER).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/pubsub"))
.json(&json!({ "script_id": handler, "topic_pattern": "e2e.topic" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
let source = create_script(
&server,
&app_id,
"pubsub-source",
r#"pubsub::publish_durable("e2e.topic", #{ hello: 1 }); #{ ok: true }"#,
)
.await;
execute(&server, &source).await;
let event = poll_marker(&pool, &app_id)
.await
.expect("pubsub handler fired");
assert_eq!(event["source"], "pubsub");
assert_eq!(event["op"], "publish");
assert_eq!(event["pubsub"]["topic"], "e2e.topic");
assert_eq!(event["pubsub"]["message"]["hello"], 1);
}
#[tokio::test]
async fn dispatcher_delivers_dead_letter_to_handler() {
// NOTE: the dead-letter creation path (`dispatcher::handle_failure` →
// `DeadLetterRepo::insert`) writes the `dead_letters` row but does not
// appear to enqueue deliveries for `dead_letter`-kind triggers
// (`TriggerRepo::list_matching_dead_letter` has no production caller —
// see HANDBACK latent-findings). So this test asserts the wired
// behavior: a failing handler that exhausts its (single) attempt
// produces a dead-letter row. If/when DL→handler fan-out lands, this
// can be upgraded to assert the handler marker like the others.
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "dl").await;
// A handler that always throws, with a single attempt so it
// dead-letters immediately (no retry backoff).
let failing = create_script(&server, &app_id, "dl-failing", r#"throw "boom";"#).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/kv"))
.json(&json!({
"script_id": failing,
"collection_glob": "dlsrc",
"retry_max_attempts": 1,
"retry_base_ms": 0
}))
.await
.assert_status(axum::http::StatusCode::CREATED);
let source = create_script(
&server,
&app_id,
"dl-source",
r#"kv::collection("dlsrc").set("k", 1); #{ ok: true }"#,
)
.await;
execute(&server, &source).await;
// Poll the dead_letters table for this app.
let app_uuid = Uuid::parse_str(&app_id).unwrap();
let mut count: i64 = 0;
for _ in 0..100 {
count = sqlx::query_scalar("SELECT COUNT(*) FROM dead_letters WHERE app_id = $1")
.bind(app_uuid)
.fetch_one(&pool)
.await
.expect("count dead_letters");
if count > 0 {
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
assert!(count > 0, "a dead-letter row should have been produced");
}