diff --git a/.gitignore b/.gitignore index c6478c9..6c14189 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ Cargo.lock.bak # Local config overrides config.local.toml /data +# Files-root blob storage created when integration tests run build_app +# from the picloud crate dir (PICLOUD_FILES_ROOT default ./data). +/crates/picloud/data /postgres-data # Dashboard diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a0593..44f3dfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,91 @@ # PiCloud Changelog +## v1.1.6 — Realtime Channels & Client Library (unreleased) + +The first **external realtime surface** and the first **frontend +library**, co-shipped per the §5/§6 design-notes decisions. Browser +clients can subscribe over SSE to per-app pub/sub topics that have been +explicitly externalized; everything else stays internal-only. The +`@picloud/client` TypeScript package wraps typed HTTP, SSE, auth, and +React/Svelte hooks. Plus three v1.1.5 follow-ups. + +### Added — Realtime + +- **`topics` registry** (`migrations/0021_topics.sql`) — pub/sub topics + are internal-only by default; a `topics` row with + `external_subscribable = true` opts one into external SSE subscription. + `auth_mode` is `'public'` or `'token'`. +- **Topic admin endpoints** under `/api/v1/admin/apps/{id}/topics` — + `POST` (register), `GET` (list), `PATCH /{name}` (flip + external/auth_mode — its own audited surface), `DELETE /{name}` + (unregister + disconnect live subscribers). Gated by the new + `Capability::AppTopicManage` → `app:admin` scope (no new scope; the + seven-scope commitment holds). +- **SSE endpoint `GET /realtime/topics/{topic}`** — data-plane surface + (deliberately not under `/api/`). Resolves `Host` → app, authorizes + via the `RealtimeAuthority` (404 for missing/internal topics, 401 for + bad/absent tokens), then streams `data: {topic,message,published_at}` + events with a configurable heartbeat (`PICLOUD_REALTIME_HEARTBEAT_SEC`, + default 30). Token via `Authorization: Bearer` or `?token=`. +- **`RealtimeBroadcaster` + `RealtimeEvent` + `RealtimeAuthority`** + traits (`picloud-shared`); in-process `InProcessBroadcaster` + (`tokio::sync::broadcast`, per-channel capacity + `PICLOUD_REALTIME_BROADCAST_CAPACITY` default 64, periodic empty-channel + GC) and the DB-backed `RealtimeAuthorityImpl` (orchestrator-core / + manager-core respectively). The publish path now also fans out to + in-process SSE subscribers, best-effort, after the durable outbox + fan-out commits — a broadcast failure never fails the publish. +- **`pubsub::subscriber_token(topics, ttl)`** Rhai SDK (SDK schema + 1.6 → 1.7) — mints an HMAC-SHA256 subscriber token (URL-safe + `payload.signature`) scoped to externally-subscribable topics. + Requires an authenticated principal + the pub/sub publish capability. + TTL clamped to `[10s, 24h]` (default 1h), env-overridable via + `PICLOUD_SUBSCRIBER_TOKEN_TTL_{MIN,MAX,DEFAULT}_SEC`. Per-app signing + keys persist in the new `app_secrets` table + (`migrations/0022_app_secrets.sql`), created lazily on first mint. No + per-token revocation (rotation invalidates wholesale; short TTL is the + safety mechanism). +- **Dashboard Topics tab** — register/list/edit/delete topics with a + prominent external/internal badge, auth-mode radio (conditional on + external), and a confirmation when flipping a topic external. + +### Added — `@picloud/client` (TypeScript, v1.0.0) + +- New top-level package `clients/typescript/` (tsup dual ESM+CJS + + `.d.ts`, vitest). Typed HTTP via `endpoint(path).get()/.post()` + with auth-token injection and structured errors; SSE `subscribe(topic, + cb, {token, onTokenExpired})` with exponential-backoff reconnect, + 401 token-refresh, and `Last-Event-ID` resume; `auth.login/logout/token` + over dev-defined endpoints; React (`useTopic`/`useEndpoint` + + `PicloudProvider`) and Svelte (`topicStore`/`endpointStore`) subpath + exports. Optional zod/valibot runtime validation via a `{ parse }` + adapter (no hard dep). Hybrid model: no direct service access from the + browser. + +### Changed / Fixed — v1.1.5 follow-ups + +- **Empty blobs accepted** — `NewFile::validate` / `FileUpdate::validate` + no longer reject zero-length `data`; empty files are a valid stored + state (sentinels, placeholders). Non-breaking. +- **Orphan `*.tmp.*` sweeper** — a startup tokio task + (`spawn_files_orphan_sweep`) walks the files root every + `PICLOUD_FILES_ORPHAN_SWEEP_INTERVAL_SEC` (default 6h) and unlinks temp + blobs older than `PICLOUD_FILES_ORPHAN_TMP_TTL_SEC` (default 1h). No DB + cross-check (that full reconciler is v1.3+). +- **Dispatcher end-to-end tests** — `crates/picloud/tests/dispatcher_e2e.rs`, + one per trigger kind (kv/docs/cron/files/pubsub/dead_letter), + DATABASE_URL-gated (skip cleanly when unset). + +### Notes + +- New deps: `hmac` (token signing, picloud-shared), `tokio-stream` (SSE + body stream, orchestrator-core). +- New env vars: `PICLOUD_REALTIME_HEARTBEAT_SEC`, + `PICLOUD_REALTIME_BROADCAST_CAPACITY`, + `PICLOUD_SUBSCRIBER_TOKEN_TTL_{MIN,MAX,DEFAULT}_SEC`, + `PICLOUD_FILES_ORPHAN_SWEEP_INTERVAL_SEC`, + `PICLOUD_FILES_ORPHAN_TMP_TTL_SEC`. + ## v1.1.5 — Files & Pub/Sub (unreleased) Two stateful services + two trigger kinds. **`files::*`** is diff --git a/Cargo.lock b/Cargo.lock index 8d989b3..203e733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1610,7 +1610,7 @@ dependencies = [ [[package]] name = "picloud" -version = "1.1.5" +version = "1.1.6" dependencies = [ "anyhow", "async-trait", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "picloud-cli" -version = "1.1.5" +version = "1.1.6" dependencies = [ "anyhow", "assert_cmd", @@ -1657,7 +1657,7 @@ dependencies = [ [[package]] name = "picloud-executor" -version = "1.1.5" +version = "1.1.6" dependencies = [ "anyhow", "picloud-executor-core", @@ -1669,7 +1669,7 @@ dependencies = [ [[package]] name = "picloud-executor-core" -version = "1.1.5" +version = "1.1.6" dependencies = [ "async-trait", "base64", @@ -1693,7 +1693,7 @@ dependencies = [ [[package]] name = "picloud-manager" -version = "1.1.5" +version = "1.1.6" dependencies = [ "anyhow", "picloud-manager-core", @@ -1705,7 +1705,7 @@ dependencies = [ [[package]] name = "picloud-manager-core" -version = "1.1.5" +version = "1.1.6" dependencies = [ "argon2", "async-trait", @@ -1733,7 +1733,7 @@ dependencies = [ [[package]] name = "picloud-orchestrator" -version = "1.1.5" +version = "1.1.6" dependencies = [ "anyhow", "picloud-orchestrator-core", @@ -1745,7 +1745,7 @@ dependencies = [ [[package]] name = "picloud-orchestrator-core" -version = "1.1.5" +version = "1.1.6" dependencies = [ "async-trait", "axum", @@ -1759,6 +1759,8 @@ dependencies = [ "serde_json", "thiserror 1.0.69", "tokio", + "tokio-stream", + "tower", "tracing", "urlencoding", "uuid", @@ -1766,13 +1768,17 @@ dependencies = [ [[package]] name = "picloud-shared" -version = "1.1.5" +version = "1.1.6" dependencies = [ "async-trait", + "base64", "chrono", + "hmac", "serde", "serde_json", + "sha2", "thiserror 1.0.69", + "tokio", "uuid", ] @@ -2990,6 +2996,20 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 36e112e..de58d5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "1.1.5" +version = "1.1.6" edition = "2021" rust-version = "1.92" license = "MIT OR Apache-2.0" @@ -29,6 +29,8 @@ picloud-manager-core = { path = "crates/manager-core" } # Async + HTTP tokio = { version = "1.40", features = ["full"] } +# Wraps a broadcast::Receiver into a Stream for the SSE endpoint (v1.1.6). +tokio-stream = { version = "0.1", features = ["sync"] } axum = "0.8" tower = "0.5" tower-http = { version = "0.6", features = ["trace", "cors"] } @@ -75,6 +77,8 @@ urlencoding = "2" argon2 = "0.5" rand = { version = "0.8", features = ["getrandom"] } sha2 = "0.10" +# HMAC-SHA256 for realtime subscriber tokens (v1.1.6). +hmac = "0.12" base64 = "0.22" data-encoding = "2.6" diff --git a/crates/executor-core/src/sdk/pubsub.rs b/crates/executor-core/src/sdk/pubsub.rs index dcc185c..fc35387 100644 --- a/crates/executor-core/src/sdk/pubsub.rs +++ b/crates/executor-core/src/sdk/pubsub.rs @@ -40,9 +40,85 @@ pub(super) fn register(engine: &mut RhaiEngine, services: &Services, cx: Arc Result> { + mint_token(&svc, &cx, topics, None) + }, + ); + } + // `pubsub::subscriber_token(topics, ttl)` — `ttl` is an integer + // (seconds) or `()` for the default. + { + let svc = svc.clone(); + let cx = cx.clone(); + module.set_native_fn( + "subscriber_token", + move |topics: Array, ttl: Dynamic| -> Result> { + let ttl = ttl_from_dynamic(&ttl)?; + mint_token(&svc, &cx, topics, ttl) + }, + ); + } engine.register_static_module("pubsub", module.into()); } +/// Interpret the optional `ttl` argument: `()` → use the default, +/// integer → that many seconds, anything else → throw. +fn ttl_from_dynamic(ttl: &Dynamic) -> Result, Box> { + if ttl.is_unit() { + return Ok(None); + } + ttl.as_int().map(Some).map_err(|_| -> Box { + EvalAltResult::ErrorRuntime( + "pubsub::subscriber_token: ttl must be an integer (seconds) or ()".into(), + rhai::Position::NONE, + ) + .into() + }) +} + +fn mint_token( + svc: &Arc, + cx: &Arc, + topics: Array, + ttl: Option, +) -> Result> { + // Every element must be a string; surface a clear error otherwise. + let mut names = Vec::with_capacity(topics.len()); + for t in topics { + if !t.is_string() { + return Err(EvalAltResult::ErrorRuntime( + "pubsub::subscriber_token: topics must be an array of strings".into(), + rhai::Position::NONE, + ) + .into()); + } + names.push(t.into_string().unwrap_or_default()); + } + let svc = svc.clone(); + let cx = cx.clone(); + let handle = TokioHandle::try_current().map_err(|e| -> Box { + EvalAltResult::ErrorRuntime( + format!("pubsub: no tokio runtime available: {e}").into(), + rhai::Position::NONE, + ) + .into() + })?; + // SubscriberToken errors already carry the full + // "pubsub::subscriber_token: …" wording, so surface them verbatim. + handle + .block_on(async move { svc.mint_subscriber_token(&cx, names, ttl).await }) + .map_err(|err| -> Box { + EvalAltResult::ErrorRuntime(format!("{err}").into(), rhai::Position::NONE).into() + }) +} + /// Convert a Rhai `Dynamic` message into JSON, base64-encoding any /// `Blob` (at any nesting depth). Mirrors `bridge::dynamic_to_json` but /// adds the blob arm the pub/sub wire contract requires. diff --git a/crates/executor-core/tests/sdk_subscriber_token.rs b/crates/executor-core/tests/sdk_subscriber_token.rs new file mode 100644 index 0000000..0768c2c --- /dev/null +++ b/crates/executor-core/tests/sdk_subscriber_token.rs @@ -0,0 +1,242 @@ +//! `pubsub::subscriber_token` SDK bridge integration tests (v1.1.6). +//! +//! Runs a real Rhai engine against a fake `PubsubService` whose +//! `mint_subscriber_token` mirrors the production validation (principal +//! required, non-empty topics, ttl clamp, externally-subscribable check) +//! and signs a real token. These cover the bridge surface: array → +//! `Vec` forwarding, the omitted/`()`/integer ttl handling, and +//! errors surfacing as thrown Rhai errors. The authoritative validation +//! logic is unit-tested in `manager-core::pubsub_service`. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use picloud_executor_core::{Engine, ExecRequest, InvocationType, Limits}; +use picloud_shared::subscriber_token::{self, TokenClaims}; +use picloud_shared::{ + AdminUserId, AppId, ExecutionId, InstanceRole, NoopDeadLetterService, NoopDocsService, + NoopEventEmitter, NoopFilesService, NoopHttpService, NoopKvService, NoopModuleSource, + Principal, PubsubError, PubsubService, RequestId, ScriptId, ScriptSandbox, SdkCallCx, Services, +}; +use serde_json::Value; + +const FAKE_KEY: [u8; 32] = [7u8; 32]; +const MIN_TTL: i64 = 10; +const MAX_TTL: i64 = 86_400; +const DEFAULT_TTL: i64 = 3_600; + +/// Fake that mirrors the production mint rules and signs with FAKE_KEY. +#[derive(Default)] +struct FakeMintPubsub; + +#[async_trait] +impl PubsubService for FakeMintPubsub { + async fn publish_durable( + &self, + _cx: &SdkCallCx, + _topic: &str, + _message: Value, + ) -> Result<(), PubsubError> { + Ok(()) + } + + async fn mint_subscriber_token( + &self, + cx: &SdkCallCx, + topics: Vec, + ttl_seconds: Option, + ) -> Result { + if cx.principal.is_none() { + return Err(PubsubError::SubscriberToken( + "pubsub::subscriber_token: requires an authenticated principal".into(), + )); + } + if topics.is_empty() { + return Err(PubsubError::SubscriberToken( + "pubsub::subscriber_token: topics list must not be empty".into(), + )); + } + let ttl = ttl_seconds.unwrap_or(DEFAULT_TTL); + if !(MIN_TTL..=MAX_TTL).contains(&ttl) { + return Err(PubsubError::SubscriberToken(format!( + "pubsub::subscriber_token: ttl_seconds must be between {MIN_TTL} and {MAX_TTL}" + ))); + } + for name in &topics { + // Only "chat" and "notify" are "registered" in this fake. + if name != "chat" && name != "notify" { + return Err(PubsubError::SubscriberToken(format!( + "pubsub::subscriber_token: topic {name} is not externally subscribable" + ))); + } + } + let now = 1_000_000; + Ok(subscriber_token::sign( + &FAKE_KEY, + &TokenClaims { + app_id: cx.app_id, + topics, + exp: now + ttl, + iat: now, + }, + )) + } +} + +fn make_engine() -> Arc { + let services = Services::new( + Arc::new(NoopKvService), + Arc::new(NoopDocsService), + Arc::new(NoopDeadLetterService), + Arc::new(NoopEventEmitter), + Arc::new(NoopModuleSource), + Arc::new(NoopHttpService), + Arc::new(NoopFilesService), + Arc::new(FakeMintPubsub), + ); + Arc::new(Engine::new(Limits::default(), services)) +} + +fn request(app_id: AppId, with_principal: bool) -> ExecRequest { + let execution_id = ExecutionId::new(); + ExecRequest { + execution_id, + request_id: RequestId::new(), + script_id: ScriptId::new(), + script_name: "token-test".into(), + invocation_type: InvocationType::Http, + path: "/token-test".into(), + headers: BTreeMap::new(), + body: Value::Null, + params: BTreeMap::new(), + query: BTreeMap::new(), + rest: String::new(), + sandbox_overrides: ScriptSandbox::default(), + app_id, + principal: with_principal.then(|| Principal { + user_id: AdminUserId::new(), + instance_role: InstanceRole::Owner, + scopes: None, + app_binding: None, + }), + trigger_depth: 0, + root_execution_id: execution_id, + is_dead_letter_handler: false, + event: None, + } +} + +async fn run_ok(engine: Arc, src: &str, req: ExecRequest) -> Value { + let src = src.to_string(); + tokio::task::spawn_blocking(move || engine.execute(&src, req)) + .await + .expect("spawn_blocking should not panic") + .expect("script execution should succeed") + .body +} + +async fn run_err(engine: Arc, src: &str, req: ExecRequest) { + let src = src.to_string(); + let res = tokio::task::spawn_blocking(move || engine.execute(&src, req)) + .await + .expect("spawn_blocking should not panic"); + assert!(res.is_err(), "expected script to throw"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn token_contains_topics_and_expiry() { + let app = AppId::new(); + let body = run_ok( + make_engine(), + r#"#{ token: pubsub::subscriber_token(["chat", "notify"], 120) }"#, + request(app, true), + ) + .await; + let token = body["token"].as_str().expect("token string"); + let claims = subscriber_token::verify(&FAKE_KEY, token, 1_000_001).unwrap(); + assert_eq!(claims.app_id, app); + assert_eq!( + claims.topics, + vec!["chat".to_string(), "notify".to_string()] + ); + assert_eq!(claims.exp - claims.iat, 120); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn omitted_ttl_uses_default() { + let app = AppId::new(); + let body = run_ok( + make_engine(), + r#"#{ token: pubsub::subscriber_token(["chat"]) }"#, + request(app, true), + ) + .await; + let token = body["token"].as_str().unwrap(); + let claims = subscriber_token::verify(&FAKE_KEY, token, 1_000_001).unwrap(); + assert_eq!(claims.exp - claims.iat, DEFAULT_TTL); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unit_ttl_uses_default() { + let app = AppId::new(); + let body = run_ok( + make_engine(), + r#"#{ token: pubsub::subscriber_token(["chat"], ()) }"#, + request(app, true), + ) + .await; + let token = body["token"].as_str().unwrap(); + let claims = subscriber_token::verify(&FAKE_KEY, token, 1_000_001).unwrap(); + assert_eq!(claims.exp - claims.iat, DEFAULT_TTL); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn empty_topics_throws() { + run_err( + make_engine(), + r#"pubsub::subscriber_token([], 60)"#, + request(AppId::new(), true), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ttl_below_min_throws() { + run_err( + make_engine(), + r#"pubsub::subscriber_token(["chat"], 5)"#, + request(AppId::new(), true), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ttl_above_max_throws() { + run_err( + make_engine(), + r#"pubsub::subscriber_token(["chat"], 90000)"#, + request(AppId::new(), true), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn anonymous_principal_throws() { + run_err( + make_engine(), + r#"pubsub::subscriber_token(["chat"], 60)"#, + request(AppId::new(), false), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unregistered_topic_throws() { + run_err( + make_engine(), + r#"pubsub::subscriber_token(["chat", "secret"], 60)"#, + request(AppId::new(), true), + ) + .await; +} diff --git a/crates/manager-core/migrations/0021_topics.sql b/crates/manager-core/migrations/0021_topics.sql new file mode 100644 index 0000000..51202c1 --- /dev/null +++ b/crates/manager-core/migrations/0021_topics.sql @@ -0,0 +1,31 @@ +-- v1.1.6: Explicit registration for externally-subscribable topics. +-- +-- Internal-only topics remain implicit per the §5 design-notes +-- decision: anyone can publish_durable("any.topic", msg) and triggers +-- can subscribe without a row here. This table only holds topics that +-- have been explicitly externalized — external SSE subscribers can +-- only subscribe to topics with a row here AND external_subscribable +-- = TRUE. +-- +-- The publish path (v1.1.5's publish_durable) does NOT consult this +-- table: publishing to a topic with no row still fans out to triggers +-- and to any in-process external subscribers (none exist for an +-- unregistered topic, since external subscribers can't subscribe to +-- one). The topics table is read by the SSE subscribe path only. +-- +-- auth_mode values: 'public' + 'token' in v1.1.6. 'session' arrives in +-- v1.1.8 (users-SDK); 'script' arrives in v1.2 (script-mediated auth). +-- The CHECK constraint extends in those releases. +CREATE TABLE topics ( + app_id UUID NOT NULL REFERENCES apps(id) ON DELETE CASCADE, + name TEXT NOT NULL, + external_subscribable BOOL NOT NULL DEFAULT FALSE, + auth_mode TEXT NOT NULL DEFAULT 'public' + CHECK (auth_mode IN ('public', 'token')), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (app_id, name) +); + +-- Hot lookup: "is topic T in app X externally subscribable?" The PK +-- (app_id, name) already covers this; an explicit index is redundant. diff --git a/crates/manager-core/migrations/0022_app_secrets.sql b/crates/manager-core/migrations/0022_app_secrets.sql new file mode 100644 index 0000000..b3a8815 --- /dev/null +++ b/crates/manager-core/migrations/0022_app_secrets.sql @@ -0,0 +1,19 @@ +-- v1.1.6: per-app secret material. Currently holds the HMAC signing key +-- used to mint + verify realtime subscriber tokens +-- (pubsub::subscriber_token → SSE /realtime/topics handshake). +-- +-- The key is: +-- * stable across restarts (issued tokens stay valid until expiry), +-- * per-app (a token signed by app A is rejected by app B), +-- * never script-accessible (scripts can't print/exfiltrate it — the +-- SDK only mints tokens, it never returns the key). +-- +-- The row is created lazily on the first pubsub::subscriber_token call +-- for an app (32 random bytes). This table is the natural home for +-- v1.1.7's encrypted per-app secrets work. +CREATE TABLE app_secrets ( + app_id UUID PRIMARY KEY REFERENCES apps(id) ON DELETE CASCADE, + realtime_signing_key BYTEA NOT NULL, -- 32 random bytes + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/crates/manager-core/src/app_secrets_repo.rs b/crates/manager-core/src/app_secrets_repo.rs new file mode 100644 index 0000000..ad0648c --- /dev/null +++ b/crates/manager-core/src/app_secrets_repo.rs @@ -0,0 +1,91 @@ +//! `AppSecretsRepo` — per-app secret material (v1.1.6). +//! +//! Today this holds only the HMAC signing key for realtime subscriber +//! tokens. The key is generated lazily (32 random bytes) on the first +//! `pubsub::subscriber_token` call for an app and never changes +//! thereafter in v1.1.6 (no rotation API yet — rotation is the +//! key-invalidation mechanism, deferred). The key is never exposed to +//! scripts: the SDK mints tokens, it never returns the key. +//! +//! This table is the natural home for v1.1.7's encrypted per-app +//! secrets work. + +use async_trait::async_trait; +use picloud_shared::AppId; +use rand::RngCore; +use sqlx::PgPool; + +/// Length of a freshly-generated realtime signing key. +pub const SIGNING_KEY_LEN: usize = 32; + +#[derive(Debug, thiserror::Error)] +pub enum AppSecretsRepoError { + #[error("database error: {0}")] + Db(#[from] sqlx::Error), +} + +#[async_trait] +pub trait AppSecretsRepo: Send + Sync { + /// Fetch the app's realtime signing key, generating + persisting one + /// (32 random bytes) if absent. Idempotent under concurrency: a + /// racing creator's `ON CONFLICT DO NOTHING` insert is a no-op and + /// the existing key is returned. + async fn get_or_create_signing_key( + &self, + app_id: AppId, + ) -> Result, AppSecretsRepoError>; + + /// Fetch the signing key if it exists, WITHOUT creating one. The SSE + /// verify path uses this: a missing key means no token was ever + /// minted for the app, so any presented token must be rejected. + async fn signing_key(&self, app_id: AppId) -> Result>, AppSecretsRepoError>; +} + +pub struct PostgresAppSecretsRepo { + pool: PgPool, +} + +impl PostgresAppSecretsRepo { + #[must_use] + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl AppSecretsRepo for PostgresAppSecretsRepo { + async fn get_or_create_signing_key( + &self, + app_id: AppId, + ) -> Result, AppSecretsRepoError> { + let mut fresh = vec![0u8; SIGNING_KEY_LEN]; + rand::thread_rng().fill_bytes(&mut fresh); + + // Insert-if-absent then read: the racing-creator's insert is a + // no-op, and the SELECT always returns the winning key. + sqlx::query( + "INSERT INTO app_secrets (app_id, realtime_signing_key) \ + VALUES ($1, $2) ON CONFLICT (app_id) DO NOTHING", + ) + .bind(app_id.into_inner()) + .bind(&fresh) + .execute(&self.pool) + .await?; + + let key: (Vec,) = + sqlx::query_as("SELECT realtime_signing_key FROM app_secrets WHERE app_id = $1") + .bind(app_id.into_inner()) + .fetch_one(&self.pool) + .await?; + Ok(key.0) + } + + async fn signing_key(&self, app_id: AppId) -> Result>, AppSecretsRepoError> { + let row: Option<(Vec,)> = + sqlx::query_as("SELECT realtime_signing_key FROM app_secrets WHERE app_id = $1") + .bind(app_id.into_inner()) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.0)) + } +} diff --git a/crates/manager-core/src/authz.rs b/crates/manager-core/src/authz.rs index 64615c4..b7844d2 100644 --- a/crates/manager-core/src/authz.rs +++ b/crates/manager-core/src/authz.rs @@ -97,6 +97,12 @@ pub enum Capability { /// to `app:admin` on API keys. Public-HTTP scripts (principal None) /// fail this check — managing dead letters is an admin act. AppDeadLetterManage(AppId), + /// Register / list / update / delete externally-subscribable topics + /// for this app (v1.1.6). Maps to `app:admin` on API keys — + /// externalizing a topic is an app-configuration act with security + /// weight (it opens an internal pub/sub topic to outside SSE + /// subscribers). Granted to `app_admin`+. + AppTopicManage(AppId), } impl Capability { @@ -123,7 +129,8 @@ impl Capability { | Self::AppFilesWrite(id) | Self::AppPubsubPublish(id) | Self::AppManageTriggers(id) - | Self::AppDeadLetterManage(id) => Some(id), + | Self::AppDeadLetterManage(id) + | Self::AppTopicManage(id) => Some(id), } } @@ -150,9 +157,10 @@ impl Capability { | Self::AppPubsubPublish(_) => Scope::ScriptWrite, Self::AppWriteRoute(_) => Scope::RouteWrite, Self::AppManageDomains(_) => Scope::DomainManage, - Self::AppAdmin(_) | Self::AppManageTriggers(_) | Self::AppDeadLetterManage(_) => { - Scope::AppAdmin - } + Self::AppAdmin(_) + | Self::AppManageTriggers(_) + | Self::AppDeadLetterManage(_) + | Self::AppTopicManage(_) => Scope::AppAdmin, Self::AppLogRead(_) => Scope::LogRead, } } @@ -316,6 +324,7 @@ const fn role_satisfies(role: AppRole, cap: Capability) -> bool { | Capability::AppAdmin(_) | Capability::AppManageTriggers(_) | Capability::AppDeadLetterManage(_) + | Capability::AppTopicManage(_) ); match role { AppRole::Viewer => in_viewer, @@ -659,6 +668,35 @@ mod tests { ); } + #[tokio::test] + async fn topic_manage_requires_app_admin() { + let repo = InMemoryAuthzRepo::default(); + let app = AppId::new(); + // Maps to the app:admin scope, not a new one. + assert_eq!( + Capability::AppTopicManage(app).required_scope(), + Scope::AppAdmin + ); + + // Member with only Editor role cannot manage topics. + let p = principal(InstanceRole::Member); + repo.grant(p.user_id, app, AppRole::Editor).await; + assert_eq!( + can(&repo, &p, Capability::AppTopicManage(app)) + .await + .unwrap(), + Decision::Deny, + ); + + // App-admin role can. + let admin = principal(InstanceRole::Member); + repo.grant(admin.user_id, app, AppRole::AppAdmin).await; + assert!(can(&repo, &admin, Capability::AppTopicManage(app)) + .await + .unwrap() + .is_allow()); + } + #[test] fn capability_app_id_extraction() { let app = AppId::new(); diff --git a/crates/manager-core/src/files_service.rs b/crates/manager-core/src/files_service.rs index 2d60da6..443f0bd 100644 --- a/crates/manager-core/src/files_service.rs +++ b/crates/manager-core/src/files_service.rs @@ -633,20 +633,36 @@ mod tests { .await .unwrap_err(); assert!(matches!(err, FilesError::MissingField("content_type"))); - // data - let err = files + // Empty `data` is NO LONGER rejected (v1.1.6 relaxation) — see + // `empty_file_round_trips`. + } + + #[tokio::test] + async fn empty_file_round_trips() { + // v1.1.6: a zero-byte blob is a valid stored state (sentinels, + // placeholders). Create with empty data, then read it back. + let files = svc(); + let cx = anon_cx(AppId::new()); + let id = files .create( &cx, "c", NewFile { - name: "f".into(), - content_type: "text/plain".into(), + name: "empty.bin".into(), + content_type: "application/octet-stream".into(), data: vec![], }, ) .await - .unwrap_err(); - assert!(matches!(err, FilesError::MissingField("data"))); + .expect("empty file create should succeed"); + let bytes = files.get(&cx, "c", &id.to_string()).await.unwrap(); + assert_eq!(bytes, Some(Vec::new())); + let meta = files + .head(&cx, "c", &id.to_string()) + .await + .unwrap() + .expect("metadata present"); + assert_eq!(meta.size, 0); } #[tokio::test] diff --git a/crates/manager-core/src/files_sweep.rs b/crates/manager-core/src/files_sweep.rs new file mode 100644 index 0000000..2ccb77e --- /dev/null +++ b/crates/manager-core/src/files_sweep.rs @@ -0,0 +1,185 @@ +//! Orphan `*.tmp.*` blob sweeper (v1.1.6, v1.1.5 follow-up). +//! +//! The files repo writes blobs atomically: it streams into a +//! `.tmp.-` temp file, fsyncs, then renames to the final +//! `` path. A crash between create and rename leaves an orphan temp +//! file that is never read and never reclaimed. This sweeper deletes +//! those: every `PICLOUD_FILES_ORPHAN_SWEEP_INTERVAL_SEC` (default 6h) it +//! walks `/files/` and unlinks any `*.tmp.*` file older than +//! `PICLOUD_FILES_ORPHAN_TMP_TTL_SEC` (default 1h). +//! +//! Deliberately bounded: it does NOT cross-check on-disk files against DB +//! rows (the full reconciling sweeper is v1.3+). It only targets the temp +//! files, which are unambiguously orphans once past the TTL — no live +//! writer keeps one around for an hour. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +const ENV_INTERVAL: &str = "PICLOUD_FILES_ORPHAN_SWEEP_INTERVAL_SEC"; +const ENV_TMP_TTL: &str = "PICLOUD_FILES_ORPHAN_TMP_TTL_SEC"; +const DEFAULT_INTERVAL_SECS: u64 = 21_600; // 6h +const DEFAULT_TMP_TTL_SECS: u64 = 3_600; // 1h + +/// Marker that identifies a temp blob (`.tmp.-`). A final +/// blob is named just `` (a UUID), so it never contains this. +const TMP_MARKER: &str = ".tmp."; + +#[derive(Debug, Default, Clone, Copy)] +pub struct SweepStats { + pub dirs_walked: u64, + pub files_deleted: u64, + pub bytes_reclaimed: u64, +} + +/// Spawn the periodic orphan sweep. Spawned at startup alongside the +/// cron scheduler and the realtime/cache GC tasks. +pub fn spawn_files_orphan_sweep(files_root: PathBuf) { + let interval = Duration::from_secs(read_secs(ENV_INTERVAL, DEFAULT_INTERVAL_SECS)); + let ttl = Duration::from_secs(read_secs(ENV_TMP_TTL, DEFAULT_TMP_TTL_SECS)); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.tick().await; // skip the immediate first fire + loop { + ticker.tick().await; + let root = files_root.clone(); + // Blocking filesystem walk off the async worker. + let stats = tokio::task::spawn_blocking(move || sweep_orphan_tmp_files(&root, ttl)) + .await + .unwrap_or_default(); + tracing::info!( + dirs_walked = stats.dirs_walked, + files_deleted = stats.files_deleted, + bytes_reclaimed = stats.bytes_reclaimed, + "files orphan sweep complete" + ); + } + }); +} + +/// Walk `/files/` and delete `*.tmp.*` files older than +/// `ttl`. Missing root is not an error (returns zeroed stats). Pure + +/// synchronous so it's unit-testable without a runtime. +#[must_use] +pub fn sweep_orphan_tmp_files(files_root: &Path, ttl: Duration) -> SweepStats { + let mut stats = SweepStats::default(); + let blobs_dir = files_root.join("files"); + if !blobs_dir.is_dir() { + return stats; + } + let now = SystemTime::now(); + walk(&blobs_dir, ttl, now, &mut stats); + stats +} + +fn walk(dir: &Path, ttl: Duration, now: SystemTime, stats: &mut SweepStats) { + stats.dirs_walked += 1; + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let Ok(ft) = entry.file_type() else { + continue; + }; + let path = entry.path(); + if ft.is_dir() { + walk(&path, ttl, now, stats); + continue; + } + if !ft.is_file() { + continue; + } + if !entry.file_name().to_string_lossy().contains(TMP_MARKER) { + continue; + } + let Ok(meta) = entry.metadata() else { + continue; + }; + let age = meta + .modified() + .ok() + .and_then(|m| now.duration_since(m).ok()) + .unwrap_or(Duration::ZERO); + if age >= ttl { + let size = meta.len(); + if std::fs::remove_file(&path).is_ok() { + stats.files_deleted += 1; + stats.bytes_reclaimed += size; + } + } + } +} + +fn read_secs(key: &str, default: u64) -> u64 { + match std::env::var(key) { + Err(_) => default, + Ok(v) => match v.parse::() { + Ok(n) if n > 0 => n, + _ => { + tracing::warn!(env = key, value = %v, "invalid; using default"); + default + } + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + static SEQ: AtomicU64 = AtomicU64::new(0); + + fn tmp_root() -> PathBuf { + let n = SEQ.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("picloud-sweep-test-{}-{n}", std::process::id())); + std::fs::create_dir_all(dir.join("files").join("ab")).unwrap(); + dir + } + + fn touch(path: &Path) { + std::fs::write(path, b"x").unwrap(); + } + + #[test] + fn deletes_old_tmp_files() { + let root = tmp_root(); + let tmp = root.join("files/ab/uuid.tmp.123-0"); + touch(&tmp); + // ttl 0 → any tmp file counts as old. + let stats = sweep_orphan_tmp_files(&root, Duration::ZERO); + assert_eq!(stats.files_deleted, 1); + assert!(!tmp.exists()); + assert!(stats.bytes_reclaimed >= 1); + } + + #[test] + fn keeps_young_tmp_files() { + let root = tmp_root(); + let tmp = root.join("files/ab/uuid.tmp.123-0"); + touch(&tmp); + // Large TTL → the just-created file is too young to reap. + let stats = sweep_orphan_tmp_files(&root, Duration::from_secs(3600)); + assert_eq!(stats.files_deleted, 0); + assert!(tmp.exists()); + } + + #[test] + fn keeps_non_tmp_files() { + let root = tmp_root(); + let blob = root.join("files/ab/0123456789abcdef"); + touch(&blob); + let stats = sweep_orphan_tmp_files(&root, Duration::ZERO); + assert_eq!(stats.files_deleted, 0); + assert!(blob.exists()); + } + + #[test] + fn missing_root_does_not_panic() { + let root = std::env::temp_dir().join("picloud-sweep-nonexistent-xyz"); + let stats = sweep_orphan_tmp_files(&root, Duration::ZERO); + assert_eq!(stats.files_deleted, 0); + assert_eq!(stats.dirs_walked, 0); + } +} diff --git a/crates/manager-core/src/lib.rs b/crates/manager-core/src/lib.rs index b39bf20..561594d 100644 --- a/crates/manager-core/src/lib.rs +++ b/crates/manager-core/src/lib.rs @@ -16,6 +16,7 @@ pub mod app_domain_repo; pub mod app_members_api; pub mod app_members_repo; pub mod app_repo; +pub mod app_secrets_repo; pub mod apps_api; pub mod auth; pub mod auth_api; @@ -33,6 +34,7 @@ pub mod docs_service; pub mod files_api; pub mod files_repo; pub mod files_service; +pub mod files_sweep; pub mod gc; pub mod http_service; pub mod kv_repo; @@ -45,12 +47,15 @@ pub mod outbox_repo; pub mod principal_resolver; pub mod pubsub_repo; pub mod pubsub_service; +pub mod realtime_authority; pub mod repo; pub mod route_admin; pub mod route_repo; pub mod sandbox; pub mod scheduler; pub mod ssrf; +pub mod topic_repo; +pub mod topics_api; pub mod trigger_config; pub mod trigger_repo; pub mod triggers_api; @@ -81,6 +86,9 @@ pub use app_members_repo::{ PostgresAppMembersRepository, }; pub use app_repo::{resolve_app, AppLookup, AppRepository, PostgresAppRepository}; +pub use app_secrets_repo::{ + AppSecretsRepo, AppSecretsRepoError, PostgresAppSecretsRepo, SIGNING_KEY_LEN, +}; pub use apps_api::{apps_router, AppsState}; pub use auth_api::auth_router; pub use auth_bootstrap::{ @@ -104,6 +112,7 @@ pub use docs_service::DocsServiceImpl; pub use files_api::{files_admin_router, FilesAdminState}; pub use files_repo::{FilesConfig, FilesRepo, FilesRepoError, FsFilesRepo}; pub use files_service::FilesServiceImpl; +pub use files_sweep::{spawn_files_orphan_sweep, sweep_orphan_tmp_files, SweepStats}; pub use gc::{spawn_abandoned_gc, spawn_dead_letter_gc}; pub use http_service::{HttpConfig, HttpServiceImpl}; pub use kv_repo::{KvRepo, KvRepoError, PostgresKvRepo}; @@ -116,7 +125,8 @@ pub use outbox_repo::{ }; pub use principal_resolver::{AdminPrincipalResolver, PrincipalResolver, PrincipalResolverError}; pub use pubsub_repo::{PostgresPubsubRepo, PublishCtx, PubsubRepo, PubsubRepoError}; -pub use pubsub_service::PubsubServiceImpl; +pub use pubsub_service::{PubsubServiceImpl, SubscriberTokenConfig}; +pub use realtime_authority::RealtimeAuthorityImpl; pub use repo::{ ExecutionLogRepository, NewScript, PostgresExecutionLogRepository, PostgresScriptRepository, RepoResolver, ScriptPatch, ScriptRepository, ScriptRepositoryError, @@ -124,6 +134,8 @@ pub use repo::{ pub use route_admin::{compile_routes, route_admin_router, RouteAdminState}; pub use route_repo::{NewRoute, PostgresRouteRepository, RouteRepository}; pub use sandbox::{CeilingError, SandboxCeiling}; +pub use topic_repo::{PostgresTopicRepo, Topic, TopicAuthMode, TopicRepo, TopicRepoError}; +pub use topics_api::{topics_router, TopicsApiError, TopicsState}; pub use trigger_config::{BackoffShape, TriggerConfig}; pub use trigger_repo::{ collection_matches, CreateDeadLetterTrigger, CreateDocsTrigger, CreateFilesTrigger, diff --git a/crates/manager-core/src/pubsub_service.rs b/crates/manager-core/src/pubsub_service.rs index 71a45d4..1190ea8 100644 --- a/crates/manager-core/src/pubsub_service.rs +++ b/crates/manager-core/src/pubsub_service.rs @@ -11,20 +11,106 @@ use std::sync::Arc; use async_trait::async_trait; -use picloud_shared::{PubsubError, PubsubService, SdkCallCx, TriggerEvent}; +use picloud_shared::subscriber_token::{self, TokenClaims}; +use picloud_shared::{ + PubsubError, PubsubService, RealtimeBroadcaster, RealtimeEvent, SdkCallCx, TriggerEvent, +}; +use crate::app_secrets_repo::AppSecretsRepo; use crate::authz::{self, AuthzRepo, Capability}; use crate::pubsub_repo::{PublishCtx, PubsubRepo, PubsubRepoError}; +use crate::topic_repo::TopicRepo; + +/// TTL bounds (seconds) for `pubsub::subscriber_token`. Env-overridable +/// via `PICLOUD_SUBSCRIBER_TOKEN_TTL_{MIN,MAX,DEFAULT}_SEC`. +#[derive(Debug, Clone, Copy)] +pub struct SubscriberTokenConfig { + pub min_ttl: i64, + pub max_ttl: i64, + pub default_ttl: i64, +} + +impl SubscriberTokenConfig { + #[must_use] + pub const fn conservative() -> Self { + Self { + min_ttl: 10, + max_ttl: 86_400, + default_ttl: 3_600, + } + } + + /// Load from env, falling back to the conservative defaults for any + /// missing / invalid value. + #[must_use] + pub fn from_env() -> Self { + let mut c = Self::conservative(); + load_i64(&mut c.min_ttl, "PICLOUD_SUBSCRIBER_TOKEN_TTL_MIN_SEC"); + load_i64(&mut c.max_ttl, "PICLOUD_SUBSCRIBER_TOKEN_TTL_MAX_SEC"); + load_i64( + &mut c.default_ttl, + "PICLOUD_SUBSCRIBER_TOKEN_TTL_DEFAULT_SEC", + ); + c + } +} + +impl Default for SubscriberTokenConfig { + fn default() -> Self { + Self::conservative() + } +} + +fn load_i64(dst: &mut i64, key: &str) { + if let Ok(v) = std::env::var(key) { + match v.parse::() { + Ok(n) if n > 0 => *dst = n, + _ => tracing::warn!(env = key, value = %v, "ignoring invalid token-ttl value"), + } + } +} pub struct PubsubServiceImpl { repo: Arc, authz: Arc, + // Realtime extras (v1.1.6) — optional so the existing two-arg + // constructor (and its unit tests) keep working without them. The + // production binary attaches them via `with_realtime`. + realtime: Option>, + topics: Option>, + secrets: Option>, + token_config: SubscriberTokenConfig, } impl PubsubServiceImpl { #[must_use] pub fn new(repo: Arc, authz: Arc) -> Self { - Self { repo, authz } + Self { + repo, + authz, + realtime: None, + topics: None, + secrets: None, + token_config: SubscriberTokenConfig::conservative(), + } + } + + /// Attach the v1.1.6 realtime surface: the in-process broadcaster + /// (publish fan-out to SSE subscribers), the topic registry + + /// app-secrets repo (subscriber-token minting), and the TTL config. + #[must_use] + pub fn with_realtime( + mut self, + broadcaster: Arc, + topics: Arc, + secrets: Arc, + token_config: SubscriberTokenConfig, + ) -> Self { + self.realtime = Some(broadcaster); + self.topics = Some(topics); + self.secrets = Some(secrets); + self.token_config = token_config; + self } async fn check_publish(&self, cx: &SdkCallCx) -> Result<(), PubsubError> { @@ -60,12 +146,15 @@ impl PubsubService for PubsubServiceImpl { } self.check_publish(cx).await?; - // `published_at` is stamped on the manager side so every - // delivery agrees on one instant. + // `published_at` is stamped once on the manager side so every + // delivery path — durable triggers AND the realtime broadcast — + // agrees on one instant. The message is cloned into the trigger + // event so the realtime path can reuse the original. + let published_at = chrono::Utc::now(); let event = TriggerEvent::Pubsub { topic: topic.to_string(), - message, - published_at: chrono::Utc::now(), + message: message.clone(), + published_at, }; let payload = serde_json::to_value(&event) .map_err(|e| PubsubError::Rejected(format!("event serialize: {e}")))?; @@ -76,12 +165,115 @@ impl PubsubService for PubsubServiceImpl { trigger_depth: cx.trigger_depth, root_execution_id: cx.root_execution_id, }; + // Order (design notes §8): transactional outbox fan-out + commit + // FIRST; only then the best-effort realtime broadcast. If the + // fan-out fails, the publish throws and no broadcast happens. If + // the broadcast fails after a committed fan-out, trigger + // deliveries still happen and only SSE subscribers miss this + // event (no replay in v1.1.6). + // // No matching triggers → 0 rows written, publish still succeeds. self.repo .fan_out_publish(publish_ctx, topic, payload) .await?; + + // Non-transactional, best-effort fan-out to in-process SSE + // subscribers. Run on a child task so a panicking broadcaster + // (or a future cluster-mode resolver fault) becomes a warn log, + // never a failed publish — the durable deliveries already + // committed above. + if let Some(realtime) = self.realtime.clone() { + let app_id = cx.app_id; + let topic_owned = topic.to_string(); + let realtime_event = RealtimeEvent { + topic: topic_owned.clone(), + message, + published_at, + }; + let handle = tokio::spawn(async move { + realtime.publish(app_id, &topic_owned, realtime_event).await; + }); + if let Err(e) = handle.await { + tracing::warn!(error = %e, "realtime broadcast failed; publish unaffected"); + } + } Ok(()) } + + async fn mint_subscriber_token( + &self, + cx: &SdkCallCx, + topics: Vec, + ttl_seconds: Option, + ) -> Result { + // Anonymous (public-HTTP) scripts can't mint — that would bypass + // the token-minting authz boundary. + let Some(principal) = cx.principal.as_ref() else { + return Err(PubsubError::SubscriberToken( + "pubsub::subscriber_token: requires an authenticated principal \ + (a script on a public route cannot mint tokens)" + .into(), + )); + }; + // Minting reuses the existing pub/sub publish capability (no new + // scope — the seven-scope commitment holds). + authz::require( + &*self.authz, + principal, + Capability::AppPubsubPublish(cx.app_id), + ) + .await + .map_err(|_| PubsubError::Forbidden)?; + + let (Some(topic_repo), Some(secrets)) = (self.topics.as_ref(), self.secrets.as_ref()) + else { + return Err(PubsubError::Unavailable( + "subscriber tokens are not wired in".into(), + )); + }; + + if topics.is_empty() { + return Err(PubsubError::SubscriberToken( + "pubsub::subscriber_token: topics list must not be empty".into(), + )); + } + + let ttl = ttl_seconds.unwrap_or(self.token_config.default_ttl); + if ttl < self.token_config.min_ttl || ttl > self.token_config.max_ttl { + return Err(PubsubError::SubscriberToken(format!( + "pubsub::subscriber_token: ttl_seconds must be between {} and {}", + self.token_config.min_ttl, self.token_config.max_ttl + ))); + } + + // Every requested topic must be registered as externally + // subscribable in this app — fail fast rather than mint a token + // that won't work. + for name in &topics { + let registered = topic_repo + .get(cx.app_id, name) + .await + .map_err(|e| PubsubError::Unavailable(e.to_string()))?; + if !registered.map(|t| t.external_subscribable).unwrap_or(false) { + return Err(PubsubError::SubscriberToken(format!( + "pubsub::subscriber_token: topic {name} is not externally subscribable" + ))); + } + } + + let key = secrets + .get_or_create_signing_key(cx.app_id) + .await + .map_err(|e| PubsubError::Unavailable(e.to_string()))?; + let now = chrono::Utc::now().timestamp(); + let claims = TokenClaims { + app_id: cx.app_id, + topics, + exp: now.saturating_add(ttl), + iat: now, + }; + Ok(subscriber_token::sign(&key, &claims)) + } } // ---------------------------------------------------------------------------- @@ -317,4 +509,218 @@ mod tests { .await .unwrap(); } + + // ------------------------------------------------------------------ + // v1.1.6 realtime broadcast + subscriber-token minting + // ------------------------------------------------------------------ + + use crate::app_secrets_repo::{AppSecretsRepo, AppSecretsRepoError}; + use crate::topic_repo::{Topic, TopicAuthMode, TopicRepo, TopicRepoError}; + use picloud_orchestrator_core::InProcessBroadcaster; + use picloud_shared::{RealtimeBroadcaster, RealtimeEvent}; + + /// Topic repo fake: returns the configured topics as registered + + /// externally-subscribable (unless absent). + struct FakeTopicRepo(Vec); + #[async_trait] + impl TopicRepo for FakeTopicRepo { + async fn create( + &self, + _: AppId, + _: &str, + _: bool, + _: TopicAuthMode, + ) -> Result { + unimplemented!() + } + async fn list(&self, _: AppId) -> Result, TopicRepoError> { + unimplemented!() + } + async fn get(&self, _: AppId, name: &str) -> Result, TopicRepoError> { + Ok(self.0.iter().any(|t| t == name).then(|| Topic { + name: name.to_string(), + external_subscribable: true, + auth_mode: TopicAuthMode::Token, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + })) + } + async fn update( + &self, + _: AppId, + _: &str, + _: Option, + _: Option, + ) -> Result, TopicRepoError> { + unimplemented!() + } + async fn delete(&self, _: AppId, _: &str) -> Result { + unimplemented!() + } + } + + #[derive(Default)] + struct FakeSecrets; + #[async_trait] + impl AppSecretsRepo for FakeSecrets { + async fn get_or_create_signing_key( + &self, + _: AppId, + ) -> Result, AppSecretsRepoError> { + Ok(vec![42u8; 32]) + } + async fn signing_key(&self, _: AppId) -> Result>, AppSecretsRepoError> { + Ok(Some(vec![42u8; 32])) + } + } + + /// Broadcaster that panics on publish — proves a broadcast fault + /// can't fail the publish. + struct PanicBroadcaster; + #[async_trait] + impl RealtimeBroadcaster for PanicBroadcaster { + async fn subscribe( + &self, + _: AppId, + _: &str, + ) -> Result, picloud_shared::BroadcasterError> + { + unimplemented!() + } + async fn publish(&self, _: AppId, _: &str, _: RealtimeEvent) { + panic!("boom"); + } + async fn drop_topic(&self, _: AppId, _: &str) {} + } + + fn realtime_svc( + repo: Arc, + broadcaster: Arc, + topics: Vec, + ) -> PubsubServiceImpl { + PubsubServiceImpl::new(repo, Arc::new(EditorAuthzRepo)).with_realtime( + broadcaster, + Arc::new(FakeTopicRepo(topics)), + Arc::new(FakeSecrets), + SubscriberTokenConfig::conservative(), + ) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn publish_broadcasts_to_in_process_subscribers() { + let app = AppId::new(); + let broadcaster = Arc::new(InProcessBroadcaster::new(16)); + let mut rx = broadcaster.subscribe(app, "chat").await.unwrap(); + let svc = realtime_svc( + Arc::new(InMemoryPubsubRepo::new(vec![])), + broadcaster, + vec![], + ); + svc.publish_durable(&anon_cx(app), "chat", serde_json::json!({ "hi": 1 })) + .await + .unwrap(); + let ev = rx.recv().await.unwrap(); + assert_eq!(ev.topic, "chat"); + assert_eq!(ev.message, serde_json::json!({ "hi": 1 })); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn panicking_broadcaster_does_not_fail_publish() { + let app = AppId::new(); + let svc = realtime_svc( + Arc::new(InMemoryPubsubRepo::new(vec![])), + Arc::new(PanicBroadcaster), + vec![], + ); + // The outbox fan-out committed; the broadcast panic is swallowed. + svc.publish_durable(&anon_cx(app), "chat", serde_json::json!(1)) + .await + .expect("publish must succeed despite broadcast panic"); + } + + fn mint_svc(topics: Vec) -> PubsubServiceImpl { + realtime_svc( + Arc::new(InMemoryPubsubRepo::new(vec![])), + Arc::new(picloud_shared::NoopRealtimeBroadcaster), + topics, + ) + } + + #[tokio::test] + async fn mint_returns_token_scoped_to_topics() { + let app = AppId::new(); + let svc = mint_svc(vec!["chat".into(), "notify".into()]); + let token = svc + .mint_subscriber_token( + &member_cx(app), + vec!["chat".into(), "notify".into()], + Some(120), + ) + .await + .unwrap(); + // Verify with the fake key; claims carry the topics + expiry. + let claims = subscriber_token::verify(&[42u8; 32], &token, chrono::Utc::now().timestamp()) + .expect("token verifies"); + assert_eq!(claims.app_id, app); + assert!(claims.allows_topic("chat") && claims.allows_topic("notify")); + assert!(claims.exp > claims.iat); + } + + #[tokio::test] + async fn mint_anonymous_principal_throws() { + let app = AppId::new(); + let svc = mint_svc(vec!["chat".into()]); + let err = svc + .mint_subscriber_token(&anon_cx(app), vec!["chat".into()], None) + .await + .unwrap_err(); + assert!(matches!(err, PubsubError::SubscriberToken(_))); + } + + #[tokio::test] + async fn mint_empty_topics_throws() { + let app = AppId::new(); + let svc = mint_svc(vec!["chat".into()]); + let err = svc + .mint_subscriber_token(&member_cx(app), vec![], None) + .await + .unwrap_err(); + assert!(matches!(err, PubsubError::SubscriberToken(_))); + } + + #[tokio::test] + async fn mint_ttl_below_min_and_above_max_throw() { + let app = AppId::new(); + let svc = mint_svc(vec!["chat".into()]); + for bad in [Some(5), Some(90_000)] { + let err = svc + .mint_subscriber_token(&member_cx(app), vec!["chat".into()], bad) + .await + .unwrap_err(); + assert!( + matches!(err, PubsubError::SubscriberToken(_)), + "ttl {bad:?}" + ); + } + } + + #[tokio::test] + async fn mint_unregistered_topic_throws_with_message() { + let app = AppId::new(); + // "chat" registered; "secret" is not. + let svc = mint_svc(vec!["chat".into()]); + let err = svc + .mint_subscriber_token(&member_cx(app), vec!["chat".into(), "secret".into()], None) + .await + .unwrap_err(); + match err { + PubsubError::SubscriberToken(msg) => { + assert!( + msg.contains("topic secret is not externally subscribable"), + "got: {msg}" + ); + } + other => panic!("expected SubscriberToken, got {other:?}"), + } + } } diff --git a/crates/manager-core/src/realtime_authority.rs b/crates/manager-core/src/realtime_authority.rs new file mode 100644 index 0000000..848e34e --- /dev/null +++ b/crates/manager-core/src/realtime_authority.rs @@ -0,0 +1,338 @@ +//! `RealtimeAuthorityImpl` — the DB-backed SSE subscribe gate (v1.1.6). +//! +//! Backs the [`picloud_shared::RealtimeAuthority`] trait the SSE handler +//! in orchestrator-core calls. All `topics`-table reads and signing-key +//! material stay inside this impl so the data-plane crate never touches +//! the key. +//! +//! Verdict mapping (see [`SubscribeDenied`]): +//! * topic missing OR not externally subscribable → `NotFound` (404). +//! Both collapse to 404 so the endpoint can't probe internal topics. +//! * `auth_mode = 'public'` → allow. +//! * `auth_mode = 'token'` → verify the HMAC token (present, signed by +//! this app's key, unexpired, scoped to this topic) → allow, else +//! `Unauthorized` (401, generic — never says which check failed). +//! +//! Signing keys never change in v1.1.6 (no rotation API), so a small +//! in-memory cache avoids a per-subscribe DB read once an app's key has +//! been seen. The cache is purely an optimization — a cold miss reads +//! the row. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use picloud_shared::{subscriber_token, AppId, RealtimeAuthority, SubscribeDenied}; + +use crate::app_secrets_repo::AppSecretsRepo; +use crate::topic_repo::{TopicAuthMode, TopicRepo}; + +pub struct RealtimeAuthorityImpl { + topics: Arc, + secrets: Arc, + key_cache: Mutex>>, +} + +impl RealtimeAuthorityImpl { + #[must_use] + pub fn new(topics: Arc, secrets: Arc) -> Self { + Self { + topics, + secrets, + key_cache: Mutex::new(HashMap::new()), + } + } + + /// Fetch the app's signing key, consulting the cache first. Returns + /// `None` when the app has no key (no token ever minted) — which the + /// caller maps to `Unauthorized`. + async fn signing_key(&self, app_id: AppId) -> Result>, SubscribeDenied> { + if let Ok(cache) = self.key_cache.lock() { + if let Some(k) = cache.get(&app_id) { + return Ok(Some(k.clone())); + } + } + let key = self + .secrets + .signing_key(app_id) + .await + .map_err(|e| SubscribeDenied::Backend(e.to_string()))?; + if let Some(ref k) = key { + if let Ok(mut cache) = self.key_cache.lock() { + cache.insert(app_id, k.clone()); + } + } + Ok(key) + } +} + +#[async_trait] +impl RealtimeAuthority for RealtimeAuthorityImpl { + async fn authorize_subscribe( + &self, + app_id: AppId, + topic: &str, + token: Option<&str>, + ) -> Result<(), SubscribeDenied> { + let registered = self + .topics + .get(app_id, topic) + .await + .map_err(|e| SubscribeDenied::Backend(e.to_string()))?; + + // Missing topic AND internal-only topic both 404 — don't leak + // which internal topics exist. + let Some(t) = registered.filter(|t| t.external_subscribable) else { + return Err(SubscribeDenied::NotFound); + }; + + match t.auth_mode { + TopicAuthMode::Public => Ok(()), + TopicAuthMode::Token => { + let token = token.ok_or(SubscribeDenied::Unauthorized)?; + let key = self + .signing_key(app_id) + .await? + .ok_or(SubscribeDenied::Unauthorized)?; + let now = chrono::Utc::now().timestamp(); + let claims = subscriber_token::verify(&key, token, now) + .map_err(|_| SubscribeDenied::Unauthorized)?; + // Per-app key already makes a cross-app token fail the + // signature check; this is belt-and-suspenders. + if claims.app_id != app_id || !claims.allows_topic(topic) { + return Err(SubscribeDenied::Unauthorized); + } + Ok(()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::app_secrets_repo::AppSecretsRepoError; + use crate::topic_repo::{Topic, TopicRepoError}; + use chrono::Utc; + use picloud_shared::subscriber_token::{sign, TokenClaims}; + + struct FakeTopics(Vec<(AppId, Topic)>); + #[async_trait] + impl TopicRepo for FakeTopics { + async fn create( + &self, + _: AppId, + _: &str, + _: bool, + _: TopicAuthMode, + ) -> Result { + unimplemented!() + } + async fn list(&self, _: AppId) -> Result, TopicRepoError> { + unimplemented!() + } + async fn get(&self, app_id: AppId, name: &str) -> Result, TopicRepoError> { + Ok(self + .0 + .iter() + .find(|(a, t)| *a == app_id && t.name == name) + .map(|(_, t)| t.clone())) + } + async fn update( + &self, + _: AppId, + _: &str, + _: Option, + _: Option, + ) -> Result, TopicRepoError> { + unimplemented!() + } + async fn delete(&self, _: AppId, _: &str) -> Result { + unimplemented!() + } + } + + struct FakeSecrets(AppId, Vec); + #[async_trait] + impl AppSecretsRepo for FakeSecrets { + async fn get_or_create_signing_key( + &self, + _: AppId, + ) -> Result, AppSecretsRepoError> { + Ok(self.1.clone()) + } + async fn signing_key(&self, app_id: AppId) -> Result>, AppSecretsRepoError> { + Ok((app_id == self.0).then(|| self.1.clone())) + } + } + + fn topic(name: &str, external: bool, mode: TopicAuthMode) -> Topic { + Topic { + name: name.to_string(), + external_subscribable: external, + auth_mode: mode, + created_at: Utc::now(), + updated_at: Utc::now(), + } + } + + fn authority( + topics: Vec<(AppId, Topic)>, + key_app: AppId, + key: Vec, + ) -> RealtimeAuthorityImpl { + RealtimeAuthorityImpl::new( + Arc::new(FakeTopics(topics)), + Arc::new(FakeSecrets(key_app, key)), + ) + } + + #[tokio::test] + async fn missing_topic_is_not_found() { + let app = AppId::new(); + let a = authority(vec![], app, vec![0u8; 32]); + assert_eq!( + a.authorize_subscribe(app, "ghost", None).await, + Err(SubscribeDenied::NotFound) + ); + } + + #[tokio::test] + async fn internal_only_topic_is_not_found() { + let app = AppId::new(); + let a = authority( + vec![(app, topic("internal", false, TopicAuthMode::Public))], + app, + vec![0u8; 32], + ); + assert_eq!( + a.authorize_subscribe(app, "internal", None).await, + Err(SubscribeDenied::NotFound) + ); + } + + #[tokio::test] + async fn public_topic_allows_without_token() { + let app = AppId::new(); + let a = authority( + vec![(app, topic("news", true, TopicAuthMode::Public))], + app, + vec![0u8; 32], + ); + assert!(a.authorize_subscribe(app, "news", None).await.is_ok()); + } + + #[tokio::test] + async fn token_topic_without_token_is_unauthorized() { + let app = AppId::new(); + let a = authority( + vec![(app, topic("chat", true, TopicAuthMode::Token))], + app, + vec![7u8; 32], + ); + assert_eq!( + a.authorize_subscribe(app, "chat", None).await, + Err(SubscribeDenied::Unauthorized) + ); + } + + #[tokio::test] + async fn token_topic_with_valid_token_allows() { + let app = AppId::new(); + let key = vec![9u8; 32]; + let a = authority( + vec![(app, topic("chat", true, TopicAuthMode::Token))], + app, + key.clone(), + ); + let token = sign( + &key, + &TokenClaims { + app_id: app, + topics: vec!["chat".into()], + iat: Utc::now().timestamp(), + exp: Utc::now().timestamp() + 60, + }, + ); + assert!(a + .authorize_subscribe(app, "chat", Some(&token)) + .await + .is_ok()); + } + + #[tokio::test] + async fn token_for_other_topic_is_unauthorized() { + let app = AppId::new(); + let key = vec![9u8; 32]; + let a = authority( + vec![(app, topic("chat", true, TopicAuthMode::Token))], + app, + key.clone(), + ); + let token = sign( + &key, + &TokenClaims { + app_id: app, + topics: vec!["other".into()], + iat: Utc::now().timestamp(), + exp: Utc::now().timestamp() + 60, + }, + ); + assert_eq!( + a.authorize_subscribe(app, "chat", Some(&token)).await, + Err(SubscribeDenied::Unauthorized) + ); + } + + #[tokio::test] + async fn expired_token_is_unauthorized() { + let app = AppId::new(); + let key = vec![9u8; 32]; + let a = authority( + vec![(app, topic("chat", true, TopicAuthMode::Token))], + app, + key.clone(), + ); + let token = sign( + &key, + &TokenClaims { + app_id: app, + topics: vec!["chat".into()], + iat: Utc::now().timestamp() - 120, + exp: Utc::now().timestamp() - 60, + }, + ); + assert_eq!( + a.authorize_subscribe(app, "chat", Some(&token)).await, + Err(SubscribeDenied::Unauthorized) + ); + } + + #[tokio::test] + async fn token_signed_by_other_app_key_is_unauthorized() { + let app_a = AppId::new(); + let app_b = AppId::new(); + let key_a = vec![1u8; 32]; + let key_b = vec![2u8; 32]; + // Authority for app B; its key is key_b. + let a = authority( + vec![(app_b, topic("chat", true, TopicAuthMode::Token))], + app_b, + key_b, + ); + // Token signed by app A's key, claiming app A. + let token = sign( + &key_a, + &TokenClaims { + app_id: app_a, + topics: vec!["chat".into()], + iat: Utc::now().timestamp(), + exp: Utc::now().timestamp() + 60, + }, + ); + assert_eq!( + a.authorize_subscribe(app_b, "chat", Some(&token)).await, + Err(SubscribeDenied::Unauthorized) + ); + } +} diff --git a/crates/manager-core/src/topic_repo.rs b/crates/manager-core/src/topic_repo.rs new file mode 100644 index 0000000..207bf57 --- /dev/null +++ b/crates/manager-core/src/topic_repo.rs @@ -0,0 +1,212 @@ +//! `TopicRepo` — CRUD for the `topics` table (v1.1.6). +//! +//! This table holds ONLY topics that have been explicitly externalized +//! for SSE subscription (design notes §5). Internal-only pub/sub topics +//! stay implicit — they never get a row here, and the publish path never +//! consults this table. The two readers are the topic admin endpoints +//! ([`crate::topics_api`]) and the SSE subscribe authorization +//! ([`crate::realtime_authority`]). + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use picloud_shared::AppId; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; + +/// External-subscriber auth gate for a topic. `'public'` + `'token'` in +/// v1.1.6; `'session'` (v1.1.8) and `'script'` (v1.2) extend the DB +/// CHECK constraint and this enum later. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TopicAuthMode { + Public, + Token, +} + +impl TopicAuthMode { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Public => "public", + Self::Token => "token", + } + } + + fn from_db(s: &str) -> Result { + match s { + "public" => Ok(Self::Public), + "token" => Ok(Self::Token), + other => Err(TopicRepoError::Backend(format!( + "unknown auth_mode in DB: {other}" + ))), + } + } +} + +/// A registered, externally-subscribable topic row. +#[derive(Debug, Clone, Serialize)] +pub struct Topic { + pub name: String, + pub external_subscribable: bool, + pub auth_mode: TopicAuthMode, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, thiserror::Error)] +pub enum TopicRepoError { + #[error("a topic named {0:?} already exists in this app")] + AlreadyExists(String), + #[error("database error: {0}")] + Db(#[from] sqlx::Error), + #[error("topic backend error: {0}")] + Backend(String), +} + +#[async_trait] +pub trait TopicRepo: Send + Sync { + /// Register a topic. Errors `AlreadyExists` on PK conflict. + async fn create( + &self, + app_id: AppId, + name: &str, + external_subscribable: bool, + auth_mode: TopicAuthMode, + ) -> Result; + + /// List every registered topic in the app, ordered by name. + async fn list(&self, app_id: AppId) -> Result, TopicRepoError>; + + /// Fetch one topic by name, `None` if not registered. + async fn get(&self, app_id: AppId, name: &str) -> Result, TopicRepoError>; + + /// Update `external_subscribable` and/or `auth_mode` (each `None` + /// leaves the column unchanged). `None` return = no such topic. + async fn update( + &self, + app_id: AppId, + name: &str, + external_subscribable: Option, + auth_mode: Option, + ) -> Result, TopicRepoError>; + + /// Unregister a topic. Returns `true` if a row was removed. + async fn delete(&self, app_id: AppId, name: &str) -> Result; +} + +#[derive(sqlx::FromRow)] +struct TopicRow { + name: String, + external_subscribable: bool, + auth_mode: String, + created_at: DateTime, + updated_at: DateTime, +} + +impl TopicRow { + fn into_topic(self) -> Result { + Ok(Topic { + auth_mode: TopicAuthMode::from_db(&self.auth_mode)?, + name: self.name, + external_subscribable: self.external_subscribable, + created_at: self.created_at, + updated_at: self.updated_at, + }) + } +} + +const SELECT_COLS: &str = "name, external_subscribable, auth_mode, created_at, updated_at"; + +pub struct PostgresTopicRepo { + pool: PgPool, +} + +impl PostgresTopicRepo { + #[must_use] + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl TopicRepo for PostgresTopicRepo { + async fn create( + &self, + app_id: AppId, + name: &str, + external_subscribable: bool, + auth_mode: TopicAuthMode, + ) -> Result { + let row: Option = sqlx::query_as(&format!( + "INSERT INTO topics (app_id, name, external_subscribable, auth_mode) \ + VALUES ($1, $2, $3, $4) \ + ON CONFLICT (app_id, name) DO NOTHING \ + RETURNING {SELECT_COLS}" + )) + .bind(app_id.into_inner()) + .bind(name) + .bind(external_subscribable) + .bind(auth_mode.as_str()) + .fetch_optional(&self.pool) + .await?; + match row { + Some(r) => r.into_topic(), + None => Err(TopicRepoError::AlreadyExists(name.to_string())), + } + } + + async fn list(&self, app_id: AppId) -> Result, TopicRepoError> { + let rows: Vec = sqlx::query_as(&format!( + "SELECT {SELECT_COLS} FROM topics WHERE app_id = $1 ORDER BY name" + )) + .bind(app_id.into_inner()) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(TopicRow::into_topic).collect() + } + + async fn get(&self, app_id: AppId, name: &str) -> Result, TopicRepoError> { + let row: Option = sqlx::query_as(&format!( + "SELECT {SELECT_COLS} FROM topics WHERE app_id = $1 AND name = $2" + )) + .bind(app_id.into_inner()) + .bind(name) + .fetch_optional(&self.pool) + .await?; + row.map(TopicRow::into_topic).transpose() + } + + async fn update( + &self, + app_id: AppId, + name: &str, + external_subscribable: Option, + auth_mode: Option, + ) -> Result, TopicRepoError> { + // COALESCE leaves a column untouched when its bind is NULL. + let row: Option = sqlx::query_as(&format!( + "UPDATE topics SET \ + external_subscribable = COALESCE($3, external_subscribable), \ + auth_mode = COALESCE($4, auth_mode), \ + updated_at = NOW() \ + WHERE app_id = $1 AND name = $2 \ + RETURNING {SELECT_COLS}" + )) + .bind(app_id.into_inner()) + .bind(name) + .bind(external_subscribable) + .bind(auth_mode.map(|m| m.as_str())) + .fetch_optional(&self.pool) + .await?; + row.map(TopicRow::into_topic).transpose() + } + + async fn delete(&self, app_id: AppId, name: &str) -> Result { + let res = sqlx::query("DELETE FROM topics WHERE app_id = $1 AND name = $2") + .bind(app_id.into_inner()) + .bind(name) + .execute(&self.pool) + .await?; + Ok(res.rows_affected() > 0) + } +} diff --git a/crates/manager-core/src/topics_api.rs b/crates/manager-core/src/topics_api.rs new file mode 100644 index 0000000..2d5bc53 --- /dev/null +++ b/crates/manager-core/src/topics_api.rs @@ -0,0 +1,629 @@ +//! `/api/v1/admin/apps/{id}/topics*` — topic registration admin +//! endpoints (v1.1.6). +//! +//! These manage the `topics` table: the explicit registry of which +//! pub/sub topics are externally subscribable over SSE (design notes +//! §5). Internal-only topics never appear here. +//! +//! * `POST /apps/{id}/topics` — register a topic. +//! * `GET /apps/{id}/topics` — list registered topics. +//! * `PATCH /apps/{id}/topics/{name}` — flip external/auth_mode. +//! * `DELETE /apps/{id}/topics/{name}` — unregister + disconnect. +//! +//! The PATCH endpoint is deliberately its OWN surface (not folded into a +//! generic topic update) so every change to externally-subscribable +//! status is a discrete, watchable/auditable API call (§5 commitment). +//! +//! Create / update / delete are gated by `AppTopicManage` (→ `app:admin` +//! scope); list is gated by the existing `AppRead`. DELETE also drops +//! the topic's in-process broadcast channel so live SSE subscribers +//! disconnect cleanly. + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Json, Response}; +use axum::routing::{get, patch}; +use axum::{Extension, Router}; +use picloud_shared::{AppId, Principal, RealtimeBroadcaster}; +use serde::Deserialize; +use serde_json::json; + +use crate::app_repo::AppRepository; +use crate::authz::{require, AuthzDenied, AuthzError, AuthzRepo, Capability}; +use crate::topic_repo::{Topic, TopicAuthMode, TopicRepo, TopicRepoError}; + +#[derive(Clone)] +pub struct TopicsState { + pub topics: Arc, + pub apps: Arc, + pub authz: Arc, + pub broadcaster: Arc, +} + +pub fn topics_router(state: TopicsState) -> Router { + Router::new() + .route("/apps/{app_id}/topics", get(list_topics).post(create_topic)) + .route( + "/apps/{app_id}/topics/{name}", + patch(update_topic).delete(delete_topic), + ) + .with_state(state) +} + +#[derive(Debug, Deserialize)] +pub struct CreateTopicRequest { + pub name: String, + #[serde(default)] + pub external_subscribable: bool, + #[serde(default = "default_auth_mode")] + pub auth_mode: TopicAuthMode, +} + +const fn default_auth_mode() -> TopicAuthMode { + TopicAuthMode::Public +} + +#[derive(Debug, Deserialize)] +pub struct UpdateTopicRequest { + #[serde(default)] + pub external_subscribable: Option, + #[serde(default)] + pub auth_mode: Option, +} + +/// Topic names are concrete (external pattern subscription is v1.2), so +/// reject empties and `*` wildcards at registration. +fn validate_topic_name(name: &str) -> Result<(), TopicsApiError> { + if name.trim().is_empty() { + return Err(TopicsApiError::Invalid( + "topic name must not be empty".into(), + )); + } + if name.contains('*') { + return Err(TopicsApiError::Invalid( + "topic name must be a concrete topic, not a pattern (no '*')".into(), + )); + } + Ok(()) +} + +async fn create_topic( + State(s): State, + Extension(principal): Extension, + Path(app_id): Path, + Json(input): Json, +) -> Result<(StatusCode, Json), TopicsApiError> { + ensure_app_exists(&*s.apps, app_id).await?; + require( + s.authz.as_ref(), + &principal, + Capability::AppTopicManage(app_id), + ) + .await?; + validate_topic_name(&input.name)?; + let topic = s + .topics + .create( + app_id, + input.name.trim(), + input.external_subscribable, + input.auth_mode, + ) + .await?; + Ok((StatusCode::CREATED, Json(topic))) +} + +#[derive(Debug, serde::Serialize)] +struct ListTopicsResponse { + topics: Vec, +} + +async fn list_topics( + State(s): State, + Extension(principal): Extension, + Path(app_id): Path, +) -> Result, TopicsApiError> { + ensure_app_exists(&*s.apps, app_id).await?; + require(s.authz.as_ref(), &principal, Capability::AppRead(app_id)).await?; + let topics = s.topics.list(app_id).await?; + Ok(Json(ListTopicsResponse { topics })) +} + +async fn update_topic( + State(s): State, + Extension(principal): Extension, + Path((app_id, name)): Path<(AppId, String)>, + Json(input): Json, +) -> Result, TopicsApiError> { + ensure_app_exists(&*s.apps, app_id).await?; + require( + s.authz.as_ref(), + &principal, + Capability::AppTopicManage(app_id), + ) + .await?; + let topic = s + .topics + .update(app_id, &name, input.external_subscribable, input.auth_mode) + .await? + .ok_or(TopicsApiError::NotFound)?; + Ok(Json(topic)) +} + +async fn delete_topic( + State(s): State, + Extension(principal): Extension, + Path((app_id, name)): Path<(AppId, String)>, +) -> Result { + ensure_app_exists(&*s.apps, app_id).await?; + require( + s.authz.as_ref(), + &principal, + Capability::AppTopicManage(app_id), + ) + .await?; + if !s.topics.delete(app_id, &name).await? { + return Err(TopicsApiError::NotFound); + } + // Disconnect any live SSE subscribers for the now-unregistered topic. + s.broadcaster.drop_topic(app_id, &name).await; + Ok(StatusCode::NO_CONTENT) +} + +async fn ensure_app_exists(apps: &dyn AppRepository, app_id: AppId) -> Result<(), TopicsApiError> { + apps.get_by_id(app_id) + .await + .map_err(|e| TopicsApiError::Backend(e.to_string()))? + .ok_or(TopicsApiError::AppNotFound)?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum TopicsApiError { + #[error("app not found")] + AppNotFound, + #[error("topic not found")] + NotFound, + #[error("{0}")] + AlreadyExists(String), + #[error("invalid request: {0}")] + Invalid(String), + #[error("forbidden")] + Forbidden, + #[error("authorization repo error: {0}")] + AuthzRepo(String), + #[error("topics backend: {0}")] + Backend(String), +} + +impl From for TopicsApiError { + fn from(d: AuthzDenied) -> Self { + match d { + AuthzDenied::Denied => Self::Forbidden, + AuthzDenied::Repo(e) => Self::AuthzRepo(e.to_string()), + } + } +} + +impl From for TopicsApiError { + fn from(e: AuthzError) -> Self { + Self::AuthzRepo(e.to_string()) + } +} + +impl From for TopicsApiError { + fn from(e: TopicRepoError) -> Self { + match e { + TopicRepoError::AlreadyExists(name) => { + Self::AlreadyExists(format!("a topic named {name:?} already exists in this app")) + } + other => Self::Backend(other.to_string()), + } + } +} + +impl IntoResponse for TopicsApiError { + fn into_response(self) -> Response { + let (status, body) = match &self { + Self::AppNotFound | Self::NotFound => { + (StatusCode::NOT_FOUND, json!({ "error": self.to_string() })) + } + Self::AlreadyExists(_) => (StatusCode::CONFLICT, json!({ "error": self.to_string() })), + Self::Invalid(_) => ( + StatusCode::UNPROCESSABLE_ENTITY, + json!({ "error": self.to_string() }), + ), + Self::Forbidden => (StatusCode::FORBIDDEN, json!({ "error": self.to_string() })), + Self::AuthzRepo(e) => { + tracing::error!(error = %e, "topics admin authz repo error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + json!({ "error": "internal error" }), + ) + } + Self::Backend(e) => { + tracing::error!(error = %e, "topics admin backend error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + json!({ "error": "internal error" }), + ) + } + }; + (status, Json(body)).into_response() + } +} + +#[cfg(test)] +mod tests { + //! In-memory handler tests: capability enforcement, the + //! `external_subscribable` default, the flip being its own endpoint, + //! cross-app isolation, and DELETE disconnecting subscribers. The + //! Postgres repo is exercised by the schema + integration suites. + + use super::*; + use crate::repo::ScriptRepositoryError; + use async_trait::async_trait; + use chrono::Utc; + use picloud_shared::{ + AdminUserId, App, AppRole, BroadcasterError, InstanceRole, RealtimeEvent, UserId, + }; + use std::collections::HashMap; + use std::sync::Mutex as StdMutex; + use tokio::sync::Mutex; + + #[derive(Default)] + struct InMemoryTopicRepo { + inner: Mutex>, + } + + #[async_trait] + impl TopicRepo for InMemoryTopicRepo { + async fn create( + &self, + app_id: AppId, + name: &str, + external_subscribable: bool, + auth_mode: TopicAuthMode, + ) -> Result { + let mut g = self.inner.lock().await; + if g.contains_key(&(app_id, name.to_string())) { + return Err(TopicRepoError::AlreadyExists(name.to_string())); + } + let now = Utc::now(); + let t = Topic { + name: name.to_string(), + external_subscribable, + auth_mode, + created_at: now, + updated_at: now, + }; + g.insert((app_id, name.to_string()), t.clone()); + Ok(t) + } + async fn list(&self, app_id: AppId) -> Result, TopicRepoError> { + let g = self.inner.lock().await; + let mut v: Vec = g + .iter() + .filter(|((a, _), _)| *a == app_id) + .map(|(_, t)| t.clone()) + .collect(); + v.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(v) + } + async fn get(&self, app_id: AppId, name: &str) -> Result, TopicRepoError> { + Ok(self + .inner + .lock() + .await + .get(&(app_id, name.to_string())) + .cloned()) + } + async fn update( + &self, + app_id: AppId, + name: &str, + external_subscribable: Option, + auth_mode: Option, + ) -> Result, TopicRepoError> { + let mut g = self.inner.lock().await; + let Some(t) = g.get_mut(&(app_id, name.to_string())) else { + return Ok(None); + }; + if let Some(e) = external_subscribable { + t.external_subscribable = e; + } + if let Some(m) = auth_mode { + t.auth_mode = m; + } + t.updated_at = Utc::now(); + Ok(Some(t.clone())) + } + async fn delete(&self, app_id: AppId, name: &str) -> Result { + Ok(self + .inner + .lock() + .await + .remove(&(app_id, name.to_string())) + .is_some()) + } + } + + struct InMemoryAppRepo(AppId); + #[async_trait] + impl AppRepository for InMemoryAppRepo { + async fn list(&self) -> Result, ScriptRepositoryError> { + unimplemented!() + } + async fn list_for_user(&self, _: AdminUserId) -> Result, ScriptRepositoryError> { + unimplemented!() + } + async fn get_by_id(&self, id: AppId) -> Result, ScriptRepositoryError> { + if id != self.0 { + return Ok(None); + } + let now = Utc::now(); + Ok(Some(App { + id, + slug: "test".into(), + name: "test".into(), + description: None, + created_at: now, + updated_at: now, + })) + } + async fn get_by_slug(&self, _: &str) -> Result, ScriptRepositoryError> { + unimplemented!() + } + async fn get_by_slug_or_history( + &self, + _: &str, + ) -> Result, ScriptRepositoryError> { + unimplemented!() + } + async fn slug_in_history(&self, _: &str) -> Result, ScriptRepositoryError> { + unimplemented!() + } + async fn create( + &self, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result { + unimplemented!() + } + async fn create_with_takeover( + &self, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result { + unimplemented!() + } + async fn update( + &self, + _: AppId, + _: Option<&str>, + _: Option>, + ) -> Result { + unimplemented!() + } + async fn rename_slug( + &self, + _: AppId, + _: &str, + _: bool, + ) -> Result { + unimplemented!() + } + async fn delete(&self, _: AppId) -> Result<(), ScriptRepositoryError> { + unimplemented!() + } + async fn delete_cascade(&self, _: AppId) -> Result<(), ScriptRepositoryError> { + unimplemented!() + } + async fn count_scripts_in_app(&self, _: AppId) -> Result { + unimplemented!() + } + } + + /// Grants `AppAdmin` only for `granted_app`; denies elsewhere — used + /// for the cross-app isolation test. + struct PerAppAuthzRepo { + granted_app: AppId, + } + #[async_trait] + impl AuthzRepo for PerAppAuthzRepo { + async fn membership( + &self, + _: UserId, + app_id: AppId, + ) -> Result, AuthzError> { + Ok((app_id == self.granted_app).then_some(AppRole::AppAdmin)) + } + } + + struct DenyAuthzRepo; + #[async_trait] + impl AuthzRepo for DenyAuthzRepo { + async fn membership(&self, _: UserId, _: AppId) -> Result, AuthzError> { + Ok(None) + } + } + + #[derive(Default)] + struct RecordingBroadcaster { + dropped: StdMutex>, + } + #[async_trait] + impl RealtimeBroadcaster for RecordingBroadcaster { + async fn subscribe( + &self, + _: AppId, + _: &str, + ) -> Result, BroadcasterError> { + unimplemented!() + } + async fn publish(&self, _: AppId, _: &str, _: RealtimeEvent) {} + async fn drop_topic(&self, app_id: AppId, topic: &str) { + self.dropped + .lock() + .unwrap() + .push((app_id, topic.to_string())); + } + } + + fn member() -> Principal { + Principal { + user_id: AdminUserId::new(), + instance_role: InstanceRole::Member, + scopes: None, + app_binding: None, + } + } + + fn state(app_id: AppId, authz: Arc) -> (TopicsState, Arc) { + let bc = Arc::new(RecordingBroadcaster::default()); + let state = TopicsState { + topics: Arc::new(InMemoryTopicRepo::default()), + apps: Arc::new(InMemoryAppRepo(app_id)), + authz, + broadcaster: bc.clone(), + }; + (state, bc) + } + + #[tokio::test] + async fn register_defaults_external_subscribable_false() { + let app = AppId::new(); + let (s, _) = state(app, Arc::new(PerAppAuthzRepo { granted_app: app })); + let (status, Json(topic)) = create_topic( + State(s), + Extension(member()), + Path(app), + Json(CreateTopicRequest { + name: "chat".into(), + external_subscribable: false, + auth_mode: TopicAuthMode::Public, + }), + ) + .await + .unwrap(); + assert_eq!(status, StatusCode::CREATED); + assert!(!topic.external_subscribable); + assert_eq!(topic.name, "chat"); + } + + #[tokio::test] + async fn flip_requires_app_admin_role() { + let app = AppId::new(); + // Topic exists; the caller has no role → PATCH is forbidden. + let (s, _) = state(app, Arc::new(DenyAuthzRepo)); + s.topics + .create(app, "chat", false, TopicAuthMode::Public) + .await + .unwrap(); + let err = update_topic( + State(s), + Extension(member()), + Path((app, "chat".to_string())), + Json(UpdateTopicRequest { + external_subscribable: Some(true), + auth_mode: None, + }), + ) + .await + .unwrap_err(); + assert!(matches!(err, TopicsApiError::Forbidden)); + } + + #[tokio::test] + async fn flip_is_its_own_endpoint_and_toggles_external() { + // The PATCH handler is a distinct surface from create; flipping + // external_subscribable false→true is a single discrete call. + let app = AppId::new(); + let (s, _) = state(app, Arc::new(PerAppAuthzRepo { granted_app: app })); + s.topics + .create(app, "chat", false, TopicAuthMode::Public) + .await + .unwrap(); + let Json(updated) = update_topic( + State(s), + Extension(member()), + Path((app, "chat".to_string())), + Json(UpdateTopicRequest { + external_subscribable: Some(true), + auth_mode: Some(TopicAuthMode::Token), + }), + ) + .await + .unwrap(); + assert!(updated.external_subscribable); + assert_eq!(updated.auth_mode, TopicAuthMode::Token); + } + + #[tokio::test] + async fn delete_disconnects_subscribers() { + let app = AppId::new(); + let (s, bc) = state(app, Arc::new(PerAppAuthzRepo { granted_app: app })); + s.topics + .create(app, "chat", true, TopicAuthMode::Public) + .await + .unwrap(); + let status = delete_topic( + State(s), + Extension(member()), + Path((app, "chat".to_string())), + ) + .await + .unwrap(); + assert_eq!(status, StatusCode::NO_CONTENT); + assert_eq!( + bc.dropped.lock().unwrap().as_slice(), + &[(app, "chat".to_string())] + ); + } + + #[tokio::test] + async fn cross_app_admin_cannot_manage_other_app() { + let app_a = AppId::new(); + let app_b = AppId::new(); + // Caller is admin of app A only; both apps exist via separate state. + let authz = Arc::new(PerAppAuthzRepo { granted_app: app_a }); + // App-B-scoped state, but the caller only has A's grant. + let (s, _) = state(app_b, authz); + let err = create_topic( + State(s), + Extension(member()), + Path(app_b), + Json(CreateTopicRequest { + name: "chat".into(), + external_subscribable: true, + auth_mode: TopicAuthMode::Public, + }), + ) + .await + .unwrap_err(); + assert!(matches!(err, TopicsApiError::Forbidden)); + } + + #[tokio::test] + async fn pattern_name_rejected() { + let app = AppId::new(); + let (s, _) = state(app, Arc::new(PerAppAuthzRepo { granted_app: app })); + let err = create_topic( + State(s), + Extension(member()), + Path(app), + Json(CreateTopicRequest { + name: "user.*".into(), + external_subscribable: true, + auth_mode: TopicAuthMode::Public, + }), + ) + .await + .unwrap_err(); + assert!(matches!(err, TopicsApiError::Invalid(_))); + } +} diff --git a/crates/manager-core/tests/expected_schema.txt b/crates/manager-core/tests/expected_schema.txt index 7eaab9c..91b2591 100644 --- a/crates/manager-core/tests/expected_schema.txt +++ b/crates/manager-core/tests/expected_schema.txt @@ -58,6 +58,12 @@ table: app_members role: text NOT NULL created_at: timestamp with time zone NOT NULL default=now() +table: app_secrets + app_id: uuid NOT NULL + realtime_signing_key: bytea NOT NULL + created_at: timestamp with time zone NOT NULL default=now() + updated_at: timestamp with time zone NOT NULL default=now() + table: app_slug_history slug: text NOT NULL current_app_id: uuid NOT NULL @@ -211,6 +217,14 @@ table: scripts app_id: uuid NOT NULL kind: text NOT NULL default='endpoint'::text +table: topics + app_id: uuid NOT NULL + name: text NOT NULL + external_subscribable: boolean NOT NULL default=false + auth_mode: text NOT NULL default='public'::text + created_at: timestamp with time zone NOT NULL default=now() + updated_at: timestamp with time zone NOT NULL default=now() + table: triggers id: uuid NOT NULL default=gen_random_uuid() app_id: uuid NOT NULL @@ -256,6 +270,9 @@ indexes on app_members: app_members_pkey: public.app_members USING btree (app_id, user_id) app_members_user_id_idx: public.app_members USING btree (user_id) +indexes on app_secrets: + app_secrets_pkey: public.app_secrets USING btree (app_id) + indexes on app_slug_history: app_slug_history_pkey: public.app_slug_history USING btree (slug) @@ -328,6 +345,9 @@ indexes on scripts: scripts_name_uidx: public.scripts USING btree (app_id, lower(name)) scripts_pkey: public.scripts USING btree (id) +indexes on topics: + topics_pkey: public.topics USING btree (app_id, name) + indexes on triggers: idx_triggers_app_kind_enabled: public.triggers USING btree (app_id, kind) WHERE (enabled = true) idx_triggers_app_pubsub_enabled: public.triggers USING btree (app_id, kind) WHERE ((enabled = true) AND (kind = 'pubsub'::text)) @@ -366,6 +386,10 @@ constraints on app_members: [FOREIGN KEY] app_members_user_id_fkey: FOREIGN KEY (user_id) REFERENCES admin_users(id) ON DELETE CASCADE [PRIMARY KEY] app_members_pkey: PRIMARY KEY (app_id, user_id) +constraints on app_secrets: + [FOREIGN KEY] app_secrets_app_id_fkey: FOREIGN KEY (app_id) REFERENCES apps(id) ON DELETE CASCADE + [PRIMARY KEY] app_secrets_pkey: PRIMARY KEY (app_id) + constraints on app_slug_history: [FOREIGN KEY] app_slug_history_current_app_id_fkey: FOREIGN KEY (current_app_id) REFERENCES apps(id) ON DELETE CASCADE [PRIMARY KEY] app_slug_history_pkey: PRIMARY KEY (slug) @@ -448,6 +472,11 @@ constraints on scripts: [FOREIGN KEY] scripts_app_id_fk: FOREIGN KEY (app_id) REFERENCES apps(id) ON DELETE RESTRICT [PRIMARY KEY] scripts_pkey: PRIMARY KEY (id) +constraints on topics: + [CHECK] topics_auth_mode_check: CHECK ((auth_mode = ANY (ARRAY['public'::text, 'token'::text]))) + [FOREIGN KEY] topics_app_id_fkey: FOREIGN KEY (app_id) REFERENCES apps(id) ON DELETE CASCADE + [PRIMARY KEY] topics_pkey: PRIMARY KEY (app_id, name) + constraints on triggers: [CHECK] triggers_dispatch_mode_check: CHECK ((dispatch_mode = ANY (ARRAY['sync'::text, 'async'::text]))) [CHECK] triggers_kind_check: CHECK ((kind = ANY (ARRAY['kv'::text, 'dead_letter'::text, 'docs'::text, 'cron'::text, 'files'::text, 'pubsub'::text]))) @@ -478,3 +507,5 @@ constraints on triggers: 0018: files 0019: files triggers 0020: pubsub triggers + 0021: topics + 0022: app secrets diff --git a/crates/orchestrator-core/Cargo.toml b/crates/orchestrator-core/Cargo.toml index d76e821..a4e924f 100644 --- a/crates/orchestrator-core/Cargo.toml +++ b/crates/orchestrator-core/Cargo.toml @@ -23,8 +23,13 @@ chrono.workspace = true reqwest.workspace = true rhai.workspace = true tokio.workspace = true +tokio-stream.workspace = true urlencoding.workspace = true # v1.1.3 — top-level script AST cache lives in orchestrator-core's # LocalExecutorClient; key is ScriptId, value is `(updated_at, Arc)`. lru.workspace = true + +[dev-dependencies] +# `ServiceExt::oneshot` for driving the SSE router in unit tests. +tower.workspace = true diff --git a/crates/orchestrator-core/src/lib.rs b/crates/orchestrator-core/src/lib.rs index 3cefa3a..3e2ce49 100644 --- a/crates/orchestrator-core/src/lib.rs +++ b/crates/orchestrator-core/src/lib.rs @@ -12,6 +12,8 @@ pub mod api; pub mod client; pub mod gate; pub mod inbox; +pub mod realtime; +pub mod realtime_api; pub mod resolver; pub mod routing; @@ -19,4 +21,8 @@ pub use api::{data_plane_router, user_routes_router, DataPlaneState}; pub use client::{ExecutorClient, LocalExecutorClient, RemoteExecutorClient, ScriptIdentity}; pub use gate::{AcquireError, ExecutionGate}; pub use inbox::InboxRegistry; +pub use realtime::{spawn_realtime_gc, InProcessBroadcaster, DEFAULT_BROADCAST_CAPACITY}; +pub use realtime_api::{ + heartbeat_secs_from_env, realtime_router, RealtimeState, DEFAULT_HEARTBEAT_SECS, +}; pub use resolver::{ResolverError, ScriptResolver}; diff --git a/crates/orchestrator-core/src/realtime.rs b/crates/orchestrator-core/src/realtime.rs new file mode 100644 index 0000000..2b6f4f0 --- /dev/null +++ b/crates/orchestrator-core/src/realtime.rs @@ -0,0 +1,242 @@ +//! In-process `RealtimeBroadcaster` — the SSE fan-out registry (v1.1.6). +//! +//! Sibling of [`crate::inbox::InboxRegistry`], but multi-receiver and +//! repeated-event: a `Mutex>` +//! over `tokio::sync::broadcast` instead of a oneshot map. The publish +//! side ([`PubsubServiceImpl`]) and the SSE subscribe side both hold one +//! shared `Arc`. +//! +//! Delivery is best-effort: each channel has a bounded buffer +//! (`PICLOUD_REALTIME_BROADCAST_CAPACITY`, default 64); a slow consumer +//! that falls behind sees the oldest events dropped (standard +//! `broadcast` lag semantics — the receiver gets `RecvError::Lagged`). +//! SSE's transport-layer auto-reconnect is the recovery path; there's no +//! server-side replay in v1.1.6. +//! +//! Channels are created lazily on first subscribe. A periodic GC task +//! ([`spawn_realtime_gc`]) drops senders whose receiver count has fallen +//! to zero so one-shot subscribers don't grow the map unboundedly. +//! +//! Cluster mode (v1.3+) swaps this for a Postgres `LISTEN/NOTIFY`-backed +//! resolver behind the same `RealtimeBroadcaster` trait. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use picloud_shared::{AppId, BroadcasterError, RealtimeBroadcaster, RealtimeEvent}; +use tokio::sync::broadcast; + +/// Default per-channel broadcast buffer depth. +pub const DEFAULT_BROADCAST_CAPACITY: usize = 64; +const ENV_CAPACITY: &str = "PICLOUD_REALTIME_BROADCAST_CAPACITY"; + +/// Default GC sweep interval for empty channels. +pub const DEFAULT_GC_INTERVAL_SECS: u64 = 60; + +pub struct InProcessBroadcaster { + inner: Mutex>>, + capacity: usize, +} + +impl InProcessBroadcaster { + #[must_use] + pub fn new(capacity: usize) -> Self { + Self { + inner: Mutex::new(HashMap::new()), + capacity: capacity.max(1), + } + } + + /// Build from `PICLOUD_REALTIME_BROADCAST_CAPACITY` (default 64). + #[must_use] + pub fn from_env() -> Self { + let capacity = match std::env::var(ENV_CAPACITY) { + Err(_) => DEFAULT_BROADCAST_CAPACITY, + Ok(v) => match v.parse::() { + Ok(n) if n > 0 => n, + Ok(_) => { + tracing::warn!(env = ENV_CAPACITY, value = %v, "must be > 0; using default"); + DEFAULT_BROADCAST_CAPACITY + } + Err(e) => { + tracing::warn!(env = ENV_CAPACITY, value = %v, error = %e, "invalid; using default"); + DEFAULT_BROADCAST_CAPACITY + } + }, + }; + Self::new(capacity) + } + + /// Number of live channels in the map (test/observability helper). + #[must_use] + pub fn channel_count(&self) -> usize { + self.inner.lock().map(|g| g.len()).unwrap_or(0) + } + + /// Drop senders with zero receivers. Returns how many were removed. + /// Called periodically by [`spawn_realtime_gc`]. + pub fn gc(&self) -> usize { + let Ok(mut g) = self.inner.lock() else { + return 0; + }; + let before = g.len(); + g.retain(|_, tx| tx.receiver_count() > 0); + before - g.len() + } +} + +#[async_trait] +impl RealtimeBroadcaster for InProcessBroadcaster { + async fn subscribe( + &self, + app_id: AppId, + topic: &str, + ) -> Result, BroadcasterError> { + let mut g = self + .inner + .lock() + .map_err(|_| BroadcasterError::Unavailable("broadcaster map poisoned".into()))?; + let tx = g + .entry((app_id, topic.to_string())) + .or_insert_with(|| broadcast::channel(self.capacity).0); + Ok(tx.subscribe()) + } + + async fn publish(&self, app_id: AppId, topic: &str, event: RealtimeEvent) { + let Ok(g) = self.inner.lock() else { + return; + }; + // Only fan out to an existing channel: a topic with no live + // subscribers has no sender (publish never creates one). `send` + // returns Err iff every receiver has dropped — a benign no-op. + if let Some(tx) = g.get(&(app_id, topic.to_string())) { + let _ = tx.send(event); + } + } + + async fn drop_topic(&self, app_id: AppId, topic: &str) { + if let Ok(mut g) = self.inner.lock() { + // Removing the sender closes the channel; existing receivers + // observe `RecvError::Closed` and disconnect cleanly. + g.remove(&(app_id, topic.to_string())); + } + } +} + +/// Spawn the background GC sweep that drops empty channels every +/// `interval_secs` (default [`DEFAULT_GC_INTERVAL_SECS`]). Spawned at +/// startup alongside the other housekeeping tasks. +pub fn spawn_realtime_gc(broadcaster: Arc, interval_secs: u64) { + let period = Duration::from_secs(interval_secs.max(1)); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(period); + ticker.tick().await; // skip the immediate first fire + loop { + ticker.tick().await; + let removed = broadcaster.gc(); + if removed > 0 { + tracing::debug!(removed, "realtime broadcaster GC dropped empty channels"); + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use serde_json::json; + + fn event(topic: &str, n: i64) -> RealtimeEvent { + RealtimeEvent { + topic: topic.to_string(), + message: json!({ "n": n }), + published_at: Utc::now(), + } + } + + #[tokio::test] + async fn multiple_subscribers_each_receive_each_event() { + let b = InProcessBroadcaster::new(16); + let app = AppId::new(); + let mut rx1 = b.subscribe(app, "chat").await.unwrap(); + let mut rx2 = b.subscribe(app, "chat").await.unwrap(); + + b.publish(app, "chat", event("chat", 1)).await; + b.publish(app, "chat", event("chat", 2)).await; + + for rx in [&mut rx1, &mut rx2] { + assert_eq!(rx.recv().await.unwrap().message, json!({ "n": 1 })); + assert_eq!(rx.recv().await.unwrap().message, json!({ "n": 2 })); + } + } + + #[tokio::test] + async fn dropped_subscriber_does_not_leak_after_gc() { + let b = InProcessBroadcaster::new(16); + let app = AppId::new(); + let rx = b.subscribe(app, "t").await.unwrap(); + assert_eq!(b.channel_count(), 1); + drop(rx); + // GC reclaims the now-empty channel. + assert_eq!(b.gc(), 1); + assert_eq!(b.channel_count(), 0); + } + + #[tokio::test] + async fn drop_topic_disconnects_existing_subscribers() { + let b = InProcessBroadcaster::new(16); + let app = AppId::new(); + let mut rx = b.subscribe(app, "t").await.unwrap(); + b.drop_topic(app, "t").await; + // Sender gone → receiver observes a closed channel. + assert!(rx.recv().await.is_err()); + assert_eq!(b.channel_count(), 0); + } + + #[tokio::test] + async fn slow_consumer_loses_oldest_events() { + // Capacity 2: a consumer that never drains sees the oldest + // events dropped (broadcast Lagged semantics). + let b = InProcessBroadcaster::new(2); + let app = AppId::new(); + let mut rx = b.subscribe(app, "t").await.unwrap(); + for i in 0..5 { + b.publish(app, "t", event("t", i)).await; + } + // First recv reports the lag rather than event 0. + let first = rx.recv().await; + assert!( + matches!(first, Err(broadcast::error::RecvError::Lagged(_))), + "expected Lagged, got {first:?}" + ); + // Subsequent recvs return the most recent buffered events. + let next = rx.recv().await.unwrap(); + assert_eq!(next.message, json!({ "n": 3 })); + } + + #[tokio::test] + async fn cross_app_isolation() { + let b = InProcessBroadcaster::new(16); + let app_a = AppId::new(); + let app_b = AppId::new(); + let mut rx_a = b.subscribe(app_a, "shared").await.unwrap(); + let mut rx_b = b.subscribe(app_b, "shared").await.unwrap(); + + b.publish(app_a, "shared", event("shared", 1)).await; + // App B's subscriber must not see app A's publish. + assert_eq!(rx_a.recv().await.unwrap().message, json!({ "n": 1 })); + assert!(rx_b.try_recv().is_err()); + } + + #[tokio::test] + async fn publish_with_no_subscribers_is_noop() { + let b = InProcessBroadcaster::new(16); + let app = AppId::new(); + // No subscriber → no sender created → no panic, nothing fanned out. + b.publish(app, "ghost", event("ghost", 1)).await; + assert_eq!(b.channel_count(), 0); + } +} diff --git a/crates/orchestrator-core/src/realtime_api.rs b/crates/orchestrator-core/src/realtime_api.rs new file mode 100644 index 0000000..5d68839 --- /dev/null +++ b/crates/orchestrator-core/src/realtime_api.rs @@ -0,0 +1,408 @@ +//! SSE realtime endpoint — `GET /realtime/topics/{topic}` (v1.1.6). +//! +//! This is a data-plane surface, deliberately NOT under `/api/` +//! (realtime is its own versioning surface per the path scheme). It is +//! merged at the router root by the `picloud` binary alongside +//! `/healthz`, `/version`, and the user-route fallback. +//! +//! Handshake: +//! 1. Resolve `Host` → `app_id` (two-phase dispatch). No app → 404. +//! 2. Extract the token from `Authorization: Bearer ` OR `?token=` +//! (EventSource can't set custom headers, so the query form is the +//! browser-compatible path). +//! 3. Ask the injected [`RealtimeAuthority`]: missing/internal topic → +//! 404, bad/absent token on a token-gated topic → 401, otherwise OK. +//! 4. Acquire a `broadcast::Receiver` and stream events as SSE until +//! the client disconnects (dropping the receiver — the broadcaster +//! cleans up on its own). +//! +//! Heartbeats (`:` comment lines) keep idle proxies from closing the +//! connection; interval is `PICLOUD_REALTIME_HEARTBEAT_SEC` (default 30). + +use std::sync::Arc; +use std::time::Duration; + +use axum::extract::{Path, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::Router; +use picloud_shared::{RealtimeAuthority, RealtimeBroadcaster, SubscribeDenied}; +use serde::Deserialize; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::{Stream, StreamExt}; + +use crate::routing::AppDomainTable; + +/// Default heartbeat interval (seconds) for idle SSE connections. +pub const DEFAULT_HEARTBEAT_SECS: u64 = 30; +const ENV_HEARTBEAT: &str = "PICLOUD_REALTIME_HEARTBEAT_SEC"; + +#[derive(Clone)] +pub struct RealtimeState { + /// Host → app_id resolver (shared with the rest of the data plane). + pub app_domains: Arc, + pub broadcaster: Arc, + pub authority: Arc, + pub heartbeat: Duration, +} + +impl RealtimeState { + #[must_use] + pub fn new( + app_domains: Arc, + broadcaster: Arc, + authority: Arc, + ) -> Self { + Self { + app_domains, + broadcaster, + authority, + heartbeat: Duration::from_secs(heartbeat_secs_from_env()), + } + } +} + +/// Read `PICLOUD_REALTIME_HEARTBEAT_SEC` (default 30, must be > 0). +#[must_use] +pub fn heartbeat_secs_from_env() -> u64 { + match std::env::var(ENV_HEARTBEAT) { + Err(_) => DEFAULT_HEARTBEAT_SECS, + Ok(v) => match v.parse::() { + Ok(n) if n > 0 => n, + _ => { + tracing::warn!(env = ENV_HEARTBEAT, value = %v, "invalid; using default"); + DEFAULT_HEARTBEAT_SECS + } + }, + } +} + +/// Router for the realtime SSE surface. Merged at the router root. +#[must_use] +pub fn realtime_router(state: RealtimeState) -> Router { + Router::new() + .route("/realtime/topics/{topic}", get(sse_topic)) + .with_state(state) +} + +#[derive(Debug, Deserialize)] +struct TokenQuery { + token: Option, +} + +async fn sse_topic( + State(state): State, + Path(topic): Path, + Query(q): Query, + headers: HeaderMap, +) -> Response { + // 1. Host → app. + let host = headers + .get("host") + .and_then(|h| h.to_str().ok()) + .unwrap_or(""); + let Some(app_id) = state.app_domains.resolve_app(host) else { + return not_found("no app claims this host"); + }; + + // 2. Token: Authorization: Bearer takes precedence, else ?token=. + let token = bearer_token(&headers).or(q.token); + + // 3. Authorize. + match state + .authority + .authorize_subscribe(app_id, &topic, token.as_deref()) + .await + { + Ok(()) => {} + Err(SubscribeDenied::NotFound) => return not_found("topic not found"), + Err(SubscribeDenied::Unauthorized) => return unauthorized(), + Err(SubscribeDenied::Backend(e)) => { + tracing::error!(error = %e, "realtime authority backend error"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({ "error": "internal error" })), + ) + .into_response(); + } + } + + // 4. Subscribe + stream. + let rx = match state.broadcaster.subscribe(app_id, &topic).await { + Ok(rx) => rx, + Err(e) => { + tracing::error!(error = %e, "failed to acquire realtime subscription"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({ "error": "internal error" })), + ) + .into_response(); + } + }; + + let stream = event_stream(rx); + let sse = + Sse::new(stream).keep_alive(KeepAlive::new().interval(state.heartbeat).text("heartbeat")); + + // Sse sets Content-Type: text/event-stream + Cache-Control: no-cache. + // Add X-Accel-Buffering: no so an intermediate nginx doesn't buffer + // the stream (ignored by other proxies). Connection management is + // hyper's concern (and is hop-by-hop on HTTP/1.1, server-managed on + // HTTP/2), so we don't set Connection ourselves. + let mut resp = sse.into_response(); + resp.headers_mut().insert( + "X-Accel-Buffering", + axum::http::HeaderValue::from_static("no"), + ); + resp +} + +/// Map the broadcast receiver into a stream of SSE events. Lagged +/// notifications (slow consumer) are skipped; a closed channel +/// (`drop_topic`, or all senders gone) ends the stream and the SSE +/// connection closes cleanly. +fn event_stream( + rx: tokio::sync::broadcast::Receiver, +) -> impl Stream> { + BroadcastStream::new(rx).filter_map(|item| { + let ev = item.ok()?; // drop Lagged errors + let payload = serde_json::json!({ + "topic": ev.topic, + "message": ev.message, + "published_at": ev.published_at.to_rfc3339(), + }); + Some(Ok(Event::default().data(payload.to_string()))) + }) +} + +fn bearer_token(headers: &HeaderMap) -> Option { + let raw = headers + .get(axum::http::header::AUTHORIZATION)? + .to_str() + .ok()?; + raw.strip_prefix("Bearer ") + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()) +} + +fn not_found(msg: &str) -> Response { + ( + StatusCode::NOT_FOUND, + axum::Json(serde_json::json!({ "error": msg })), + ) + .into_response() +} + +fn unauthorized() -> Response { + // Generic — never leaks which check failed. + ( + StatusCode::UNAUTHORIZED, + axum::Json(serde_json::json!({ "error": "unauthorized" })), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::realtime::InProcessBroadcaster; + use crate::routing::AppDomainTable; + use async_trait::async_trait; + use axum::body::Body; + use axum::http::Request; + use picloud_shared::{AppId, RealtimeEvent}; + use tower::ServiceExt; // oneshot + + /// Authority stub returning a fixed verdict. + struct StubAuthority(Result<(), SubscribeDenied>); + #[async_trait] + impl RealtimeAuthority for StubAuthority { + async fn authorize_subscribe( + &self, + _: AppId, + _: &str, + _: Option<&str>, + ) -> Result<(), SubscribeDenied> { + self.0.clone() + } + } + + /// App-domain table that maps a fixed host to a fixed app. + fn domains(host: &str, app: AppId) -> Arc { + use crate::routing::{parse_app_domain, CompiledAppDomain}; + let d = parse_app_domain(host).unwrap(); + let table = AppDomainTable::new(); + table.replace(vec![CompiledAppDomain { + app_id: app, + pattern: d.pattern, + shape_key: d.shape_key, + }]); + Arc::new(table) + } + + fn state( + app: AppId, + host: &str, + verdict: Result<(), SubscribeDenied>, + broadcaster: Arc, + ) -> RealtimeState { + RealtimeState { + app_domains: domains(host, app), + broadcaster, + authority: Arc::new(StubAuthority(verdict)), + heartbeat: Duration::from_millis(100), + } + } + + async fn get_status(state: RealtimeState, host: &str, topic: &str) -> StatusCode { + let app = realtime_router(state); + let req = Request::builder() + .uri(format!("/realtime/topics/{topic}")) + .header("host", host) + .body(Body::empty()) + .unwrap(); + app.oneshot(req).await.unwrap().status() + } + + #[tokio::test] + async fn unknown_host_is_404() { + let app = AppId::new(); + let st = state( + app, + "app.example.com", + Ok(()), + Arc::new(InProcessBroadcaster::new(8)), + ); + // Request a different host → no app claims it. + assert_eq!( + get_status(st, "other.example.com", "chat").await, + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn not_found_topic_is_404() { + let app = AppId::new(); + let st = state( + app, + "app.example.com", + Err(SubscribeDenied::NotFound), + Arc::new(InProcessBroadcaster::new(8)), + ); + assert_eq!( + get_status(st, "app.example.com", "ghost").await, + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn unauthorized_token_is_401() { + let app = AppId::new(); + let st = state( + app, + "app.example.com", + Err(SubscribeDenied::Unauthorized), + Arc::new(InProcessBroadcaster::new(8)), + ); + assert_eq!( + get_status(st, "app.example.com", "chat").await, + StatusCode::UNAUTHORIZED + ); + } + + #[tokio::test] + async fn public_topic_returns_event_stream() { + let app = AppId::new(); + let st = state( + app, + "app.example.com", + Ok(()), + Arc::new(InProcessBroadcaster::new(8)), + ); + let appr = realtime_router(st); + let req = Request::builder() + .uri("/realtime/topics/chat") + .header("host", "app.example.com") + .body(Body::empty()) + .unwrap(); + let resp = appr.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let ct = resp + .headers() + .get(axum::http::header::CONTENT_TYPE) + .unwrap() + .to_str() + .unwrap(); + assert!(ct.starts_with("text/event-stream")); + assert_eq!(resp.headers().get("x-accel-buffering").unwrap(), "no"); + } + + #[tokio::test] + async fn subscribe_receives_published_event() { + let app = AppId::new(); + let broadcaster = Arc::new(InProcessBroadcaster::new(8)); + let st = state(app, "app.example.com", Ok(()), broadcaster.clone()); + let appr = realtime_router(st); + let req = Request::builder() + .uri("/realtime/topics/chat") + .header("host", "app.example.com") + .body(Body::empty()) + .unwrap(); + let resp = appr.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // The handler has subscribed; publish and read the first chunk. + // Give the streaming task a beat to register its receiver. + let mut body = resp.into_body().into_data_stream(); + tokio::time::sleep(Duration::from_millis(50)).await; + broadcaster + .publish( + app, + "chat", + RealtimeEvent { + topic: "chat".into(), + message: serde_json::json!({ "hi": 1 }), + published_at: chrono::Utc::now(), + }, + ) + .await; + + let chunk = tokio::time::timeout(Duration::from_secs(2), body.next()) + .await + .expect("a chunk within timeout") + .expect("stream item") + .expect("chunk ok"); + let text = String::from_utf8_lossy(&chunk); + assert!(text.contains("data:"), "got: {text}"); + assert!(text.contains("\"hi\":1"), "got: {text}"); + } + + #[tokio::test] + async fn heartbeat_fires_on_idle_connection() { + let app = AppId::new(); + let broadcaster = Arc::new(InProcessBroadcaster::new(8)); + // Hold a clone so the channel's sender outlives the router (which + // oneshot consumes) — otherwise the stream closes immediately. + let _keepalive = broadcaster.clone(); + let st = state(app, "app.example.com", Ok(()), broadcaster); + let appr = realtime_router(st); + let req = Request::builder() + .uri("/realtime/topics/chat") + .header("host", "app.example.com") + .body(Body::empty()) + .unwrap(); + let resp = appr.oneshot(req).await.unwrap(); + let mut body = resp.into_body().into_data_stream(); + // No publish — with a 100ms heartbeat, a keep-alive comment must + // arrive well within a second. + let chunk = tokio::time::timeout(Duration::from_secs(1), body.next()) + .await + .expect("heartbeat within timeout") + .expect("stream item") + .expect("chunk ok"); + let text = String::from_utf8_lossy(&chunk); + assert!(text.starts_with(':'), "expected SSE comment, got: {text}"); + } +} diff --git a/crates/picloud/src/lib.rs b/crates/picloud/src/lib.rs index 8e05ff4..1adb9a3 100644 --- a/crates/picloud/src/lib.rs +++ b/crates/picloud/src/lib.rs @@ -12,29 +12,33 @@ use picloud_executor_core::{Engine, Limits}; use picloud_manager_core::{ admin_router, admins_router, api_keys_router, app_members_router, apps_api, apps_router, attach_principal_if_present, auth_router, compile_routes, dead_letters_router, - files_admin_router, migrations, require_authenticated, route_admin_router, triggers_router, - AbandonedRepo, AdminPrincipalResolver, AdminSessionRepository, AdminState, AdminUserRepository, - AdminsState, ApiKeyRepository, ApiKeysState, AppDomainRepository, AppMembersRepository, - AppMembersState, AppRepository, AppsState, AuthState, AuthzRepo, DeadLetterRepo, - DeadLettersState, Dispatcher, DocsServiceImpl, FilesAdminState, FilesConfig, FilesServiceImpl, - FsFilesRepo, HttpConfig, HttpServiceImpl, KvServiceImpl, OutboxEventEmitter, OutboxRepo, - PostgresAbandonedRepo, PostgresAdminSessionRepository, PostgresAdminUserRepository, + files_admin_router, migrations, require_authenticated, route_admin_router, topics_router, + triggers_router, AbandonedRepo, AdminPrincipalResolver, AdminSessionRepository, AdminState, + AdminUserRepository, AdminsState, ApiKeyRepository, ApiKeysState, AppDomainRepository, + AppMembersRepository, AppMembersState, AppRepository, AppsState, AuthState, AuthzRepo, + DeadLetterRepo, DeadLettersState, Dispatcher, DocsServiceImpl, FilesAdminState, FilesConfig, + FilesServiceImpl, FsFilesRepo, HttpConfig, HttpServiceImpl, KvServiceImpl, OutboxEventEmitter, + OutboxRepo, PostgresAbandonedRepo, PostgresAdminSessionRepository, PostgresAdminUserRepository, PostgresApiKeyRepository, PostgresAppDomainRepository, PostgresAppMembersRepository, - PostgresAppRepository, PostgresDeadLetterRepo, PostgresDeadLetterService, PostgresDocsRepo, - PostgresExecutionLogRepository, PostgresExecutionLogSink, PostgresKvRepo, PostgresOutboxRepo, - PostgresPubsubRepo, PostgresRouteRepository, PostgresScriptRepository, PostgresTriggerRepo, - PrincipalResolver, PubsubServiceImpl, RepoResolver, RouteAdminState, RouteRepository, - SandboxCeiling, ScriptRepository, TriggerConfig, TriggerRepo, TriggersState, + PostgresAppRepository, PostgresAppSecretsRepo, PostgresDeadLetterRepo, + PostgresDeadLetterService, PostgresDocsRepo, PostgresExecutionLogRepository, + PostgresExecutionLogSink, PostgresKvRepo, PostgresOutboxRepo, PostgresPubsubRepo, + PostgresRouteRepository, PostgresScriptRepository, PostgresTopicRepo, PostgresTriggerRepo, + PrincipalResolver, PubsubServiceImpl, RealtimeAuthorityImpl, RepoResolver, RouteAdminState, + RouteRepository, SandboxCeiling, ScriptRepository, SubscriberTokenConfig, TopicRepo, + TopicsState, TriggerConfig, TriggerRepo, TriggersState, }; +use picloud_orchestrator_core::realtime::DEFAULT_GC_INTERVAL_SECS; use picloud_orchestrator_core::routing::{AppDomainTable, RouteTable}; use picloud_orchestrator_core::{ - data_plane_router, user_routes_router, DataPlaneState, ExecutionGate, InboxRegistry, - LocalExecutorClient, + data_plane_router, realtime_router, spawn_realtime_gc, user_routes_router, DataPlaneState, + ExecutionGate, InProcessBroadcaster, InboxRegistry, LocalExecutorClient, RealtimeState, }; use picloud_shared::{ DeadLetterService, DocsService, ExecutionLogSink, FilesService, HttpService, InboxResolver, - KvService, OutboxWriter, PubsubService, ScriptValidator, ServiceEventEmitter, Services, - API_VERSION, PRODUCT_VERSION, SDK_VERSION, WIRE_VERSION, + KvService, OutboxWriter, PubsubService, RealtimeAuthority, RealtimeBroadcaster, + ScriptValidator, ServiceEventEmitter, Services, API_VERSION, PRODUCT_VERSION, SDK_VERSION, + WIRE_VERSION, }; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; @@ -162,6 +166,8 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { // the bytes live on disk under `PICLOUD_FILES_ROOT` (default ./data). let files_config = FilesConfig::from_env(); let files_max_size = files_config.max_file_size_bytes; + // Kept for the v1.1.6 orphan sweeper (cleans stale `*.tmp.*` files). + let files_root = files_config.root.clone(); let files_repo = Arc::new(FsFilesRepo::new(pool.clone(), files_config)); let files: Arc = Arc::new(FilesServiceImpl::new( files_repo.clone(), @@ -169,12 +175,34 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { events.clone(), files_max_size, )); - // v1.1.5 durable pub/sub. Publishes fan out to matching pubsub - // triggers at publish time (one outbox row each), delivered by the - // same dispatcher as every other async trigger. + // v1.1.6 realtime: the in-process broadcaster is shared between the + // publish path (PubsubServiceImpl fans out to SSE subscribers after + // the durable outbox fan-out) and the SSE endpoint (subscribe side). + // The topic registry + app-secrets repo back the subscriber-token + // mint + SSE subscribe-authorization. + let broadcaster_concrete = Arc::new(InProcessBroadcaster::from_env()); + let broadcaster: Arc = broadcaster_concrete.clone(); + let topic_repo: Arc = Arc::new(PostgresTopicRepo::new(pool.clone())); + let app_secrets_repo = Arc::new(PostgresAppSecretsRepo::new(pool.clone())); + let realtime_authority: Arc = Arc::new(RealtimeAuthorityImpl::new( + topic_repo.clone(), + app_secrets_repo.clone(), + )); + + // v1.1.5 durable pub/sub, extended in v1.1.6 with the realtime + // broadcast + subscriber-token mint. Publishes fan out to matching + // pubsub triggers at publish time (one outbox row each, delivered by + // the same dispatcher as every other async trigger) AND, best-effort, + // to in-process SSE subscribers. let pubsub_repo = Arc::new(PostgresPubsubRepo::new(pool.clone())); - let pubsub: Arc = - Arc::new(PubsubServiceImpl::new(pubsub_repo, authz.clone())); + let pubsub: Arc = Arc::new( + PubsubServiceImpl::new(pubsub_repo, authz.clone()).with_realtime( + broadcaster.clone(), + topic_repo.clone(), + app_secrets_repo, + SubscriberTokenConfig::from_env(), + ), + ); let services = Services::new( kv, docs, @@ -284,6 +312,10 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { // enqueues due triggers into the outbox; the dispatcher above // delivers them like any other async trigger. picloud_manager_core::spawn_cron_scheduler(pool, trigger_config.cron_tick_interval_ms); + // v1.1.6: GC empty realtime broadcast channels (one-shot subscribers) + // and sweep orphaned `*.tmp.*` blobs left by crashed file writes. + spawn_realtime_gc(broadcaster_concrete, DEFAULT_GC_INTERVAL_SECS); + picloud_manager_core::spawn_files_orphan_sweep(files_root); let triggers_state = TriggersState { triggers: trigger_repo, apps: apps_repo.clone(), @@ -302,11 +334,17 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { apps: apps_repo.clone(), authz: authz.clone(), }; + let topics_state = TopicsState { + topics: topic_repo, + apps: apps_repo.clone(), + authz: authz.clone(), + broadcaster: broadcaster.clone(), + }; let apps_state = AppsState { apps: apps_repo, domains: domains_repo, routes: route_repo, - domain_table: app_domain_table, + domain_table: app_domain_table.clone(), authz: authz.clone(), }; @@ -345,6 +383,7 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { .merge(api_keys_router(api_keys_state)) .merge(triggers_router(triggers_state)) .merge(files_admin_router(files_admin_state)) + .merge(topics_router(topics_state)) .merge(dead_letters_router(dead_letters_state)) .layer(from_fn_with_state( auth_state.clone(), @@ -375,10 +414,21 @@ pub async fn build_app(pool: PgPool, auth: AuthDeps) -> anyhow::Result { .nest("/admin", guarded_admin) .merge(data_plane_routed); + // v1.1.6 SSE realtime surface, merged at the root (deliberately NOT + // under /api/ — realtime is its own versioning surface). Public auth + // is per-topic; no principal middleware (token verification is the + // gate, handled inside the authority). + let realtime = realtime_router(RealtimeState::new( + app_domain_table, + broadcaster, + realtime_authority, + )); + Ok(Router::new() .route("/healthz", get(healthz)) .route("/version", get(version)) .nest(&format!("/api/v{API_VERSION}"), api_v1) + .merge(realtime) .merge(user_routes) .layer(TraceLayer::new_for_http())) } diff --git a/crates/picloud/tests/dispatcher_e2e.rs b/crates/picloud/tests/dispatcher_e2e.rs new file mode 100644 index 0000000..daedf42 --- /dev/null +++ b/crates/picloud/tests/dispatcher_e2e.rs @@ -0,0 +1,353 @@ +//! End-to-end dispatcher tests — one per trigger kind (v1.1.5 follow-up, +//! landed in v1.1.6). Each test wires the full all-in-one app via +//! `build_app` (which spawns the real dispatcher + cron scheduler + +//! executor), creates an app + a logging handler script + a trigger, +//! causes the originating event, and polls for the handler's side effect. +//! +//! ## Gating +//! +//! These need a Postgres reachable via `DATABASE_URL`. They follow the +//! `schema_snapshot` pattern (NOT `#[ignore]`): when `DATABASE_URL` is +//! unset the test prints a notice and returns early, so plain +//! `cargo test` stays green locally while CI (which sets `DATABASE_URL`) +//! runs them. +//! +//! ## How "the handler fired" is observed +//! +//! The dispatcher does not write `execution_log` rows for trigger +//! handlers, so each handler instead records its `ctx.event` into a KV +//! marker (`collection = "e2e_markers"`, which no trigger watches — no +//! recursion). The test polls `kv_entries` for that marker and asserts +//! the event shape. See HANDBACK §deviations for why this lives in +//! `picloud/tests/` rather than `manager-core/tests/` (build_app lives in +//! the `picloud` crate) and for the `dead_letter` reinterpretation. + +#![allow(clippy::needless_pass_by_value)] + +use std::time::Duration; + +use axum_test::TestServer; +use serde_json::{json, Value}; +use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; +use uuid::Uuid; + +/// Connect + migrate, or return `None` (printing a skip notice) when +/// `DATABASE_URL` is unset — mirrors `schema_snapshot.rs`. +async fn pool_or_skip() -> Option { + let Ok(url) = std::env::var("DATABASE_URL") else { + eprintln!("dispatcher_e2e: DATABASE_URL unset — skipping"); + return None; + }; + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .expect("connect to DATABASE_URL"); + sqlx::migrate!("../manager-core/migrations") + .run(&pool) + .await + .expect("apply migrations"); + Some(pool) +} + +/// Build the app over the shared pool with a uniquely-named owner admin, +/// log in, and create a fresh app. `suffix` must be unique per test (the +/// pool is shared, so names must not collide). +async fn server_for(pool: PgPool, suffix: &str) -> (TestServer, String) { + use picloud_manager_core::auth::hash_password; + use picloud_shared::InstanceRole; + + let unique = format!("{suffix}-{}", Uuid::new_v4().simple()); + let auth = picloud::AuthDeps::from_pool(pool.clone()); + let username = format!("e2e-{unique}"); + let hash = hash_password("pw").expect("hash"); + auth.users + .create(&username, &hash, InstanceRole::Owner, None) + .await + .expect("seed admin"); + + let app = picloud::build_app(pool, auth).await.expect("build_app"); + let mut server = TestServer::new(app).expect("TestServer"); + let resp = server + .post("/api/v1/admin/auth/login") + .json(&json!({ "username": username, "password": "pw" })) + .await; + resp.assert_status_ok(); + let token = resp.json::()["token"] + .as_str() + .expect("login token") + .to_string(); + server.add_header("authorization", format!("Bearer {token}")); + + // A fresh app keeps each test's KV / events isolated from siblings. + let slug = format!("e2e-{unique}"); + let created: Value = server + .post("/api/v1/admin/apps") + .json(&json!({ "slug": slug, "name": slug })) + .await + .json(); + let app_id = created["id"].as_str().expect("app id").to_string(); + (server, app_id) +} + +async fn create_script(server: &TestServer, app_id: &str, name: &str, source: &str) -> String { + let created: Value = server + .post("/api/v1/admin/scripts") + .json(&json!({ "app_id": app_id, "name": name, "source": source })) + .await + .json(); + created["id"].as_str().expect("script id").to_string() +} + +/// A handler that records its `ctx.event` into a KV marker the test can +/// observe. The marker collection is watched by no trigger. +const MARKER_HANDLER: &str = r#" + let e = ctx.event; + kv::collection("e2e_markers").set("marker", e); + #{ ok: true } +"#; + +/// Poll the marker KV key until present (or ~10s timeout). +async fn poll_marker(pool: &PgPool, app_id: &str) -> Option { + poll_marker_n(pool, app_id, 100).await +} + +/// Poll the marker KV key for `iters` × 100ms. +async fn poll_marker_n(pool: &PgPool, app_id: &str, iters: u32) -> Option { + let app_uuid = Uuid::parse_str(app_id).expect("app uuid"); + for _ in 0..iters { + let row: Option<(Value,)> = sqlx::query_as( + "SELECT value FROM kv_entries \ + WHERE app_id = $1 AND collection = 'e2e_markers' AND key = 'marker'", + ) + .bind(app_uuid) + .fetch_optional(pool) + .await + .expect("query marker"); + if let Some((value,)) = row { + return Some(value); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + None +} + +async fn execute(server: &TestServer, script_id: &str) { + server + .post(&format!("/api/v1/execute/{script_id}")) + .json(&json!({})) + .await + .assert_status_ok(); +} + +#[tokio::test] +async fn dispatcher_delivers_kv_to_handler() { + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "kv").await; + + let handler = create_script(&server, &app_id, "kv-handler", MARKER_HANDLER).await; + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/kv")) + .json(&json!({ "script_id": handler, "collection_glob": "src" })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + let source = create_script( + &server, + &app_id, + "kv-source", + r#"kv::collection("src").set("k", 42); #{ ok: true }"#, + ) + .await; + execute(&server, &source).await; + + let event = poll_marker(&pool, &app_id).await.expect("kv handler fired"); + assert_eq!(event["source"], "kv"); + assert_eq!(event["op"], "insert"); + assert_eq!(event["kv"]["collection"], "src"); + assert_eq!(event["kv"]["key"], "k"); +} + +#[tokio::test] +async fn dispatcher_delivers_docs_to_handler() { + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "docs").await; + + let handler = create_script(&server, &app_id, "docs-handler", MARKER_HANDLER).await; + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/docs")) + .json(&json!({ "script_id": handler, "collection_glob": "src" })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + let source = create_script( + &server, + &app_id, + "docs-source", + r#"docs::collection("src").create(#{ x: 1 }); #{ ok: true }"#, + ) + .await; + execute(&server, &source).await; + + let event = poll_marker(&pool, &app_id) + .await + .expect("docs handler fired"); + assert_eq!(event["source"], "docs"); + assert_eq!(event["op"], "create"); + assert_eq!(event["docs"]["collection"], "src"); +} + +#[tokio::test] +async fn dispatcher_delivers_cron_to_handler() { + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "cron").await; + + let handler = create_script(&server, &app_id, "cron-handler", MARKER_HANDLER).await; + // Fire every second (6-field cron, seconds-resolution). + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/cron")) + .json(&json!({ "script_id": handler, "schedule": "* * * * * *", "timezone": "UTC" })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + // No source — the scheduler enqueues the due tick on its own. The + // scheduler skips its first tick and then ticks every + // PICLOUD_CRON_TICK_INTERVAL_MS (default 30s), so poll past that + // (set the env var lower to speed CI up if desired). + let event = poll_marker_n(&pool, &app_id, 450) + .await + .expect("cron handler fired"); + assert_eq!(event["source"], "cron"); + assert_eq!(event["op"], "tick"); + assert_eq!(event["cron"]["timezone"], "UTC"); +} + +#[tokio::test] +async fn dispatcher_delivers_files_to_handler() { + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "files").await; + + let handler = create_script(&server, &app_id, "files-handler", MARKER_HANDLER).await; + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/files")) + .json(&json!({ "script_id": handler, "collection_glob": "src" })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + let source = create_script( + &server, + &app_id, + "files-source", + r#" + let data = base64::decode("aGk="); + files::collection("src").create(#{ name: "f.txt", content_type: "text/plain", data: data }); + #{ ok: true } + "#, + ) + .await; + execute(&server, &source).await; + + let event = poll_marker(&pool, &app_id) + .await + .expect("files handler fired"); + assert_eq!(event["source"], "files"); + assert_eq!(event["op"], "create"); + assert_eq!(event["files"]["collection"], "src"); + assert_eq!(event["files"]["name"], "f.txt"); +} + +#[tokio::test] +async fn dispatcher_delivers_pubsub_to_handler() { + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "pubsub").await; + + let handler = create_script(&server, &app_id, "pubsub-handler", MARKER_HANDLER).await; + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/pubsub")) + .json(&json!({ "script_id": handler, "topic_pattern": "e2e.topic" })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + let source = create_script( + &server, + &app_id, + "pubsub-source", + r#"pubsub::publish_durable("e2e.topic", #{ hello: 1 }); #{ ok: true }"#, + ) + .await; + execute(&server, &source).await; + + let event = poll_marker(&pool, &app_id) + .await + .expect("pubsub handler fired"); + assert_eq!(event["source"], "pubsub"); + assert_eq!(event["op"], "publish"); + assert_eq!(event["pubsub"]["topic"], "e2e.topic"); + assert_eq!(event["pubsub"]["message"]["hello"], 1); +} + +#[tokio::test] +async fn dispatcher_delivers_dead_letter_to_handler() { + // NOTE: the dead-letter creation path (`dispatcher::handle_failure` → + // `DeadLetterRepo::insert`) writes the `dead_letters` row but does not + // appear to enqueue deliveries for `dead_letter`-kind triggers + // (`TriggerRepo::list_matching_dead_letter` has no production caller — + // see HANDBACK latent-findings). So this test asserts the wired + // behavior: a failing handler that exhausts its (single) attempt + // produces a dead-letter row. If/when DL→handler fan-out lands, this + // can be upgraded to assert the handler marker like the others. + let Some(pool) = pool_or_skip().await else { + return; + }; + let (server, app_id) = server_for(pool.clone(), "dl").await; + + // A handler that always throws, with a single attempt so it + // dead-letters immediately (no retry backoff). + let failing = create_script(&server, &app_id, "dl-failing", r#"throw "boom";"#).await; + server + .post(&format!("/api/v1/admin/apps/{app_id}/triggers/kv")) + .json(&json!({ + "script_id": failing, + "collection_glob": "dlsrc", + "retry_max_attempts": 1, + "retry_base_ms": 0 + })) + .await + .assert_status(axum::http::StatusCode::CREATED); + + let source = create_script( + &server, + &app_id, + "dl-source", + r#"kv::collection("dlsrc").set("k", 1); #{ ok: true }"#, + ) + .await; + execute(&server, &source).await; + + // Poll the dead_letters table for this app. + let app_uuid = Uuid::parse_str(&app_id).unwrap(); + let mut count: i64 = 0; + for _ in 0..100 { + count = sqlx::query_scalar("SELECT COUNT(*) FROM dead_letters WHERE app_id = $1") + .bind(app_uuid) + .fetch_one(&pool) + .await + .expect("count dead_letters"); + if count > 0 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!(count > 0, "a dead-letter row should have been produced"); +} diff --git a/crates/shared/Cargo.toml b/crates/shared/Cargo.toml index 25e83a8..47e2b70 100644 --- a/crates/shared/Cargo.toml +++ b/crates/shared/Cargo.toml @@ -15,3 +15,12 @@ serde_json.workspace = true thiserror.workspace = true uuid.workspace = true chrono.workspace = true +# Realtime broadcaster trait returns a broadcast::Receiver; subscriber +# tokens are HMAC-SHA256 over a base64url payload (v1.1.6). +tokio = { workspace = true, features = ["sync"] } +hmac.workspace = true +sha2.workspace = true +base64.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "time", "sync"] } diff --git a/crates/shared/src/files.rs b/crates/shared/src/files.rs index b3842d3..8810a7d 100644 --- a/crates/shared/src/files.rs +++ b/crates/shared/src/files.rs @@ -177,8 +177,11 @@ pub enum FilesError { impl NewFile { /// Validate required fields + length caps at the SDK boundary. - /// `data` must be non-empty (v1.1.5 treats an empty blob as a - /// missing `data` field — see HANDBACK §7). + /// + /// Empty `data` is **accepted** as a valid stored state (v1.1.6 + /// relaxed the v1.1.5 rejection — empty files are a legitimate use + /// case: sentinels, placeholders, zero-byte uploads. See HANDBACK + /// §7). `name` and `content_type` are still required. /// /// # Errors /// @@ -191,9 +194,6 @@ impl NewFile { if self.content_type.trim().is_empty() { return Err(FilesError::MissingField("content_type")); } - if self.data.is_empty() { - return Err(FilesError::MissingField("data")); - } if self.name.len() > MAX_FILE_NAME_BYTES { return Err(FilesError::NameTooLong(self.name.len())); } @@ -218,9 +218,9 @@ impl FileUpdate { /// Returns the field-specific [`FilesError`] for the first failing /// check. pub fn validate(&self, max_size: usize) -> Result<(), FilesError> { - if self.data.is_empty() { - return Err(FilesError::MissingField("data")); - } + // Empty replacement bytes are accepted (v1.1.6 relaxation — + // consistent with NewFile::validate; updating a file to zero + // bytes is as legitimate as creating one). if let Some(name) = &self.name { if name.trim().is_empty() { return Err(FilesError::MissingField("name")); diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 64fab48..e30a46f 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -21,11 +21,14 @@ pub mod log_sink; pub mod modules; pub mod outbox_writer; pub mod pubsub; +pub mod realtime; +pub mod realtime_authority; pub mod route; pub mod sandbox; pub mod script; pub mod sdk_cx; pub mod services; +pub mod subscriber_token; pub mod trigger_event; pub mod validator; pub mod version; @@ -54,6 +57,8 @@ pub use outbox_writer::{HttpDispatchPayload, NewHttpOutbox, OutboxWriter, Outbox pub use pubsub::{ topic_matches, validate_topic_pattern, NoopPubsubService, PubsubError, PubsubService, }; +pub use realtime::{BroadcasterError, NoopRealtimeBroadcaster, RealtimeBroadcaster, RealtimeEvent}; +pub use realtime_authority::{DenyAllRealtimeAuthority, RealtimeAuthority, SubscribeDenied}; pub use route::{DispatchMode, HostKind, PathKind, Route}; pub use sandbox::ScriptSandbox; pub use script::{Script, ScriptKind}; diff --git a/crates/shared/src/pubsub.rs b/crates/shared/src/pubsub.rs index 2f78d60..68ff3b6 100644 --- a/crates/shared/src/pubsub.rs +++ b/crates/shared/src/pubsub.rs @@ -30,6 +30,32 @@ pub trait PubsubService: Send + Sync { topic: &str, message: serde_json::Value, ) -> Result<(), PubsubError>; + + /// Mint an HMAC-signed realtime subscriber token (v1.1.6). Backs the + /// `pubsub::subscriber_token(topics, ttl)` Rhai SDK call. The minted + /// token authorizes an external SSE client to subscribe to the given + /// `topics` for `ttl_seconds` (clamped to the configured bounds; the + /// configured default applies when `ttl_seconds` is `None`). + /// + /// Every topic must already be registered as externally subscribable + /// in `cx.app_id`; `cx.principal` must be `Some` (anonymous + /// public-HTTP scripts can't mint). See [`PubsubError::SubscriberToken`] + /// for the rejection messages. + /// + /// The default impl errors `Unavailable` so test fakes and the + /// `NoopPubsubService` keep compiling; the real minting lives in + /// manager-core's `PubsubServiceImpl`. + async fn mint_subscriber_token( + &self, + cx: &SdkCallCx, + topics: Vec, + ttl_seconds: Option, + ) -> Result { + let _ = (cx, topics, ttl_seconds); + Err(PubsubError::Unavailable( + "subscriber tokens are not wired in".into(), + )) + } } #[derive(Debug, Error)] @@ -47,6 +73,13 @@ pub enum PubsubError { #[error("pubsub rejected: {0}")] Rejected(String), + /// A `pubsub::subscriber_token` mint was rejected (empty topics, + /// unregistered topic, ttl out of range, anonymous caller). The + /// string is the full user-facing message; the SDK surfaces it + /// verbatim so scripts see the documented wording. + #[error("{0}")] + SubscriberToken(String), + /// Anything else — Postgres unavailable, etc. #[error("pubsub backend error: {0}")] Unavailable(String), diff --git a/crates/shared/src/realtime.rs b/crates/shared/src/realtime.rs new file mode 100644 index 0000000..16e8dc2 --- /dev/null +++ b/crates/shared/src/realtime.rs @@ -0,0 +1,86 @@ +//! `RealtimeBroadcaster` — the in-process fan-out seam for SSE realtime +//! delivery (v1.1.6). +//! +//! Structurally a sibling of [`crate::inbox::InboxResolver`]: the trait +//! lives here in `picloud-shared` because the publish side +//! (`PubsubServiceImpl` in manager-core) and the subscribe side (the SSE +//! handler in orchestrator-core) live in different crates and both need +//! one shared instance. The in-process impl lives in orchestrator-core +//! (`Mutex>`); cluster mode +//! (v1.3+) swaps it for a Postgres `LISTEN/NOTIFY`-backed resolver behind +//! the same trait without touching either caller. +//! +//! Delivery is **best-effort, at-most-once**: this is the realtime path, +//! NOT the durable one. Durable trigger fan-out (retry / dead-letter) +//! goes through the outbox and is the publish caller's separate concern. +//! A slow SSE consumer loses the oldest events (bounded broadcast +//! buffer); SSE's own transport-layer auto-reconnect is the recovery +//! mechanism (no server-side replay in v1.1.6). + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use thiserror::Error; +use tokio::sync::broadcast; + +use crate::AppId; + +/// A single realtime event delivered to in-process SSE subscribers. The +/// SSE handler serializes this to `data: {...}\n\n` on the wire. +#[derive(Debug, Clone)] +pub struct RealtimeEvent { + pub topic: String, + pub message: serde_json::Value, + pub published_at: DateTime, +} + +#[derive(Debug, Error)] +pub enum BroadcasterError { + /// Reserved for backends that can fail to register a subscriber + /// (e.g. the cluster-mode `LISTEN/NOTIFY` resolver). The in-process + /// impl never returns this. + #[error("realtime broadcaster unavailable: {0}")] + Unavailable(String), +} + +#[async_trait] +pub trait RealtimeBroadcaster: Send + Sync { + /// Subscribe to events on `(app_id, topic)`. Returns a receiver that + /// yields events until dropped. Channels are created lazily on first + /// subscribe. + async fn subscribe( + &self, + app_id: AppId, + topic: &str, + ) -> Result, BroadcasterError>; + + /// Publish an event to in-process subscribers. NOT durable — the + /// outbox-backed durable fan-out is the publish caller's separate + /// concern. A publish with no live subscribers is a silent no-op. + async fn publish(&self, app_id: AppId, topic: &str, event: RealtimeEvent); + + /// Drop every subscriber for a topic (called on topic DELETE). Live + /// receivers observe a closed channel and disconnect cleanly. + async fn drop_topic(&self, app_id: AppId, topic: &str); +} + +/// Bootstrap / test impl: subscribe yields a receiver on a throwaway +/// channel, publish is a no-op. Lets a `Services`-style bundle build +/// without the real registry wired in. +#[derive(Debug, Default, Clone, Copy)] +pub struct NoopRealtimeBroadcaster; + +#[async_trait] +impl RealtimeBroadcaster for NoopRealtimeBroadcaster { + async fn subscribe( + &self, + _app_id: AppId, + _topic: &str, + ) -> Result, BroadcasterError> { + let (_tx, rx) = broadcast::channel(1); + Ok(rx) + } + + async fn publish(&self, _app_id: AppId, _topic: &str, _event: RealtimeEvent) {} + + async fn drop_topic(&self, _app_id: AppId, _topic: &str) {} +} diff --git a/crates/shared/src/realtime_authority.rs b/crates/shared/src/realtime_authority.rs new file mode 100644 index 0000000..fe21186 --- /dev/null +++ b/crates/shared/src/realtime_authority.rs @@ -0,0 +1,70 @@ +//! `RealtimeAuthority` — the SSE subscribe authorization seam (v1.1.6). +//! +//! The SSE endpoint (`GET /realtime/topics/{topic}`) is a data-plane +//! surface in orchestrator-core, but deciding whether a subscribe is +//! allowed needs a `topics` table read plus (for token-gated topics) an +//! HMAC verify against the app's signing key — both of which require DB +//! access and the signing-key material that must NOT leak into the +//! data-plane crate. This trait keeps all of that inside the manager-core +//! impl: orchestrator-core only ever sees the three-way verdict below. +//! +//! `NotFound` is deliberately returned for *both* "no such topic" and +//! "topic exists but isn't externally subscribable" so the endpoint +//! can't be used to probe which internal topics exist (design notes §5). + +use async_trait::async_trait; + +use crate::AppId; + +/// Why a subscribe attempt was refused. The SSE handler maps these to +/// HTTP status codes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SubscribeDenied { + /// No externally-subscribable topic by that name in this app → 404. + /// Used for genuinely-missing topics AND internal-only ones, so the + /// endpoint doesn't leak which internal topics exist. + NotFound, + /// The topic is token-gated and the presented token was missing, + /// malformed, badly signed, expired, or not scoped to this topic → + /// 401 (generic; never says which check failed). + Unauthorized, + /// Backend failure (DB unavailable, etc.) → 500. + Backend(String), +} + +#[async_trait] +pub trait RealtimeAuthority: Send + Sync { + /// Decide whether an external client may subscribe to + /// `(app_id, topic)`. `token` is the bearer/query token if the + /// client presented one (`None` otherwise). + /// + /// Returns `Ok(())` when the subscribe is permitted (public topic, + /// or token-gated topic with a valid token scoped to it). + /// + /// # Errors + /// + /// [`SubscribeDenied`] — see the variants for the status mapping. + async fn authorize_subscribe( + &self, + app_id: AppId, + topic: &str, + token: Option<&str>, + ) -> Result<(), SubscribeDenied>; +} + +/// Bootstrap impl: denies everything as `NotFound`. Replaced in +/// `build_app` with the manager-core DB-backed authority. +#[derive(Debug, Default, Clone, Copy)] +pub struct DenyAllRealtimeAuthority; + +#[async_trait] +impl RealtimeAuthority for DenyAllRealtimeAuthority { + async fn authorize_subscribe( + &self, + _app_id: AppId, + _topic: &str, + _token: Option<&str>, + ) -> Result<(), SubscribeDenied> { + Err(SubscribeDenied::NotFound) + } +} diff --git a/crates/shared/src/subscriber_token.rs b/crates/shared/src/subscriber_token.rs new file mode 100644 index 0000000..dbf5ef3 --- /dev/null +++ b/crates/shared/src/subscriber_token.rs @@ -0,0 +1,200 @@ +//! HMAC-signed realtime subscriber tokens (v1.1.6, design notes §5). +//! +//! A token is a compact, URL-safe, two-part string: +//! +//! ```text +//! . +//! ``` +//! +//! where `payload` is the JSON [`TokenClaims`] and `signature` is +//! `HMAC-SHA256(app_signing_key, base64url(payload))`. Tokens are minted +//! by scripts via `pubsub::subscriber_token` (the minter lives in +//! manager-core's `PubsubServiceImpl`) and verified by the SSE subscribe +//! path (the verifier lives in manager-core's `RealtimeAuthority` impl). +//! Both sides depend on this module so the byte-for-byte contract has a +//! single home. +//! +//! There is no per-token revocation in v1.1.6 by design: HMAC bearers +//! can't be individually revoked. Rotating an app's signing key +//! invalidates every token for that app wholesale; short TTLs are the +//! safety mechanism. + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine as _; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use thiserror::Error; + +use crate::AppId; + +type HmacSha256 = Hmac; + +/// The signed payload. `exp` / `iat` are Unix seconds. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenClaims { + pub app_id: AppId, + pub topics: Vec, + pub exp: i64, + pub iat: i64, +} + +impl TokenClaims { + /// Does this token grant access to `topic`? + #[must_use] + pub fn allows_topic(&self, topic: &str) -> bool { + self.topics.iter().any(|t| t == topic) + } + + /// Is the token expired relative to `now_unix` (Unix seconds)? + #[must_use] + pub fn is_expired(&self, now_unix: i64) -> bool { + now_unix >= self.exp + } +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TokenError { + #[error("token is malformed")] + Malformed, + #[error("token signature is invalid")] + BadSignature, + #[error("token has expired")] + Expired, +} + +/// Sign `claims` with `key`, producing the `payload.signature` string. +#[must_use] +pub fn sign(key: &[u8], claims: &TokenClaims) -> String { + // `serde_json` on a fixed-field struct never fails to serialize. + let payload_json = serde_json::to_vec(claims).expect("TokenClaims serialize"); + let payload_b64 = URL_SAFE_NO_PAD.encode(&payload_json); + let sig = mac_sign(key, payload_b64.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig); + format!("{payload_b64}.{sig_b64}") +} + +/// Verify `token` against `key` and check expiry against `now_unix` +/// (Unix seconds). Returns the decoded [`TokenClaims`] on success. +/// +/// Topic-scope checking (is the requested topic in the token's list?) +/// is the caller's responsibility via [`TokenClaims::allows_topic`] — +/// this function proves authenticity + liveness only. +/// +/// # Errors +/// +/// [`TokenError::Malformed`] if the shape / base64 / JSON is wrong, +/// [`TokenError::BadSignature`] if the HMAC doesn't match, or +/// [`TokenError::Expired`] if `now_unix >= exp`. +pub fn verify(key: &[u8], token: &str, now_unix: i64) -> Result { + let (payload_b64, sig_b64) = token.split_once('.').ok_or(TokenError::Malformed)?; + + let provided_sig = URL_SAFE_NO_PAD + .decode(sig_b64) + .map_err(|_| TokenError::Malformed)?; + + // Constant-time verify of the MAC over the exact payload bytes. + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length"); + mac.update(payload_b64.as_bytes()); + mac.verify_slice(&provided_sig) + .map_err(|_| TokenError::BadSignature)?; + + // Signature good → decode the claims and check expiry. + let payload_json = URL_SAFE_NO_PAD + .decode(payload_b64) + .map_err(|_| TokenError::Malformed)?; + let claims: TokenClaims = + serde_json::from_slice(&payload_json).map_err(|_| TokenError::Malformed)?; + + if claims.is_expired(now_unix) { + return Err(TokenError::Expired); + } + Ok(claims) +} + +fn mac_sign(key: &[u8], data: &[u8]) -> Vec { + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn claims(app: AppId, topics: &[&str], exp: i64) -> TokenClaims { + TokenClaims { + app_id: app, + topics: topics.iter().map(|s| (*s).to_string()).collect(), + iat: 1000, + exp, + } + } + + #[test] + fn round_trip_verifies() { + let key = b"super-secret-key-bytes-0123456789"; + let app = AppId::new(); + let c = claims(app, &["chat.room.1", "user.notify"], 5000); + let token = sign(key, &c); + let got = verify(key, &token, 2000).expect("valid token verifies"); + assert_eq!(got, c); + assert!(got.allows_topic("chat.room.1")); + assert!(!got.allows_topic("chat.room.2")); + } + + #[test] + fn tampered_payload_fails() { + let key = b"super-secret-key-bytes-0123456789"; + let app = AppId::new(); + let token = sign(key, &claims(app, &["t"], 5000)); + // Flip a character in the payload half. + let (payload, sig) = token.split_once('.').unwrap(); + let mut bytes = payload.as_bytes().to_vec(); + bytes[0] ^= 0x01; + let tampered = format!("{}.{sig}", String::from_utf8_lossy(&bytes)); + assert_eq!(verify(key, &tampered, 2000), Err(TokenError::BadSignature)); + } + + #[test] + fn tampered_signature_fails() { + let key = b"super-secret-key-bytes-0123456789"; + let app = AppId::new(); + let token = sign(key, &claims(app, &["t"], 5000)); + let (payload, _sig) = token.split_once('.').unwrap(); + // A valid-base64 but wrong signature. + let bogus = URL_SAFE_NO_PAD.encode([0u8; 32]); + let tampered = format!("{payload}.{bogus}"); + assert_eq!(verify(key, &tampered, 2000), Err(TokenError::BadSignature)); + } + + #[test] + fn different_key_fails() { + let app = AppId::new(); + let token = sign( + b"key-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + &claims(app, &["t"], 5000), + ); + assert_eq!( + verify(b"key-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", &token, 2000), + Err(TokenError::BadSignature) + ); + } + + #[test] + fn expired_token_fails_at_expiry_check() { + let key = b"super-secret-key-bytes-0123456789"; + let app = AppId::new(); + let token = sign(key, &claims(app, &["t"], 5000)); + // now == exp → expired (>= boundary). + assert_eq!(verify(key, &token, 5000), Err(TokenError::Expired)); + assert_eq!(verify(key, &token, 9999), Err(TokenError::Expired)); + } + + #[test] + fn malformed_token_fails() { + let key = b"super-secret-key-bytes-0123456789"; + assert_eq!(verify(key, "no-dot-here", 0), Err(TokenError::Malformed)); + assert_eq!(verify(key, "a.b.c", 0), Err(TokenError::Malformed)); + } +} diff --git a/crates/shared/src/version.rs b/crates/shared/src/version.rs index d86bd1c..e783885 100644 --- a/crates/shared/src/version.rs +++ b/crates/shared/src/version.rs @@ -49,7 +49,16 @@ pub const PRODUCT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// publish-time fan-out and `ctx.event.pubsub` for pubsub-trigger /// handlers. The `Services` bundle gains `files: Arc` /// and `pubsub: Arc`. -pub const SDK_VERSION: &str = "1.6"; +/// +/// 1.7 additions (v1.1.6): `pubsub::subscriber_token(topics, ttl)` — +/// mints an HMAC-signed realtime subscriber token for externally- +/// subscribable topics (requires an authenticated principal). This is +/// the only new script-visible surface; the rest of the release is +/// server-side (the SSE `/realtime/topics/{topic}` endpoint; the +/// `RealtimeBroadcaster` / `RealtimeEvent` / `RealtimeAuthority` traits; +/// the `topics` registry + admin endpoints; the `@picloud/client` +/// TypeScript package). +pub const SDK_VERSION: &str = "1.7"; /// HTTP API major version. Appears in URL paths as `/api/v{N}/...`. /// Bump (new integer + new URL prefix) when the request/response diff --git a/dashboard/package.json b/dashboard/package.json index ef89a4d..da1d364 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "picloud-dashboard", - "version": "0.11.0", + "version": "0.12.0", "private": true, "type": "module", "scripts": { diff --git a/dashboard/src/lib/api.ts b/dashboard/src/lib/api.ts index 2540c4f..91fadb3 100644 --- a/dashboard/src/lib/api.ts +++ b/dashboard/src/lib/api.ts @@ -270,6 +270,28 @@ export interface CreatePubsubTriggerInput { retry_base_ms?: number; } +// v1.1.6 — externally-subscribable realtime topics. +export type TopicAuthMode = 'public' | 'token'; + +export interface Topic { + name: string; + external_subscribable: boolean; + auth_mode: TopicAuthMode; + created_at: string; + updated_at: string; +} + +export interface CreateTopicInput { + name: string; + external_subscribable: boolean; + auth_mode: TopicAuthMode; +} + +export interface UpdateTopicInput { + external_subscribable?: boolean; + auth_mode?: TopicAuthMode; +} + export interface ExecutionResult { status: number; headers: Record; @@ -653,6 +675,28 @@ export const api = { ) }, + topics: { + list: (idOrSlug: string) => + adminRequest<{ topics: Topic[] }>( + `/api/v1/admin/apps/${encodeURIComponent(idOrSlug)}/topics` + ), + create: (idOrSlug: string, input: CreateTopicInput) => + adminRequest(`/api/v1/admin/apps/${encodeURIComponent(idOrSlug)}/topics`, { + method: 'POST', + body: JSON.stringify(input) + }), + update: (idOrSlug: string, name: string, input: UpdateTopicInput) => + adminRequest( + `/api/v1/admin/apps/${encodeURIComponent(idOrSlug)}/topics/${encodeURIComponent(name)}`, + { method: 'PATCH', body: JSON.stringify(input) } + ), + remove: (idOrSlug: string, name: string) => + adminRequest( + `/api/v1/admin/apps/${encodeURIComponent(idOrSlug)}/topics/${encodeURIComponent(name)}`, + { method: 'DELETE' } + ) + }, + files: { list: (idOrSlug: string, collection: string, opts: { cursor?: string; limit?: number } = {}) => { const params = new URLSearchParams(); diff --git a/dashboard/src/routes/apps/[slug]/+page.svelte b/dashboard/src/routes/apps/[slug]/+page.svelte index 3c83274..709f0fa 100644 --- a/dashboard/src/routes/apps/[slug]/+page.svelte +++ b/dashboard/src/routes/apps/[slug]/+page.svelte @@ -11,7 +11,9 @@ type AppMemberDto, type AppRole, type Script, - type Trigger + type Trigger, + type Topic, + type TopicAuthMode } from '$lib/api'; import CodeEditor from '$lib/CodeEditor.svelte'; import ConfirmModal from '$lib/ConfirmModal.svelte'; @@ -25,7 +27,7 @@ const SAMPLE_SOURCE = '#{\n statusCode: 200,\n body: #{ ok: true, echo: ctx.request.body }\n}'; - type Tab = 'scripts' | 'domains' | 'members' | 'settings' | 'triggers'; + type Tab = 'scripts' | 'domains' | 'members' | 'settings' | 'triggers' | 'topics'; // Common IANA timezones offered in the cron form dropdown. Not // exhaustive — the backend validates any IANA name via chrono-tz. @@ -194,6 +196,100 @@ } } + // Topics tab (v1.1.6 — externally-subscribable realtime topics). Admin-gated. + let topics = $state([]); + let createTopicName = $state(''); + let createTopicExternal = $state(false); + let createTopicAuthMode = $state('public'); + let creatingTopic = $state(false); + let createTopicError = $state(null); + // Edit modal. + let topicToEdit = $state(null); + let editTopicExternal = $state(false); + let editTopicAuthMode = $state('public'); + let savingTopic = $state(false); + let editTopicError = $state(null); + // Flipping internal → external is the security-sensitive change. + const editFlipToExternal = $derived( + !!topicToEdit && !topicToEdit.external_subscribable && editTopicExternal + ); + // Delete confirm. + let topicToRemove = $state(null); + let removingTopic = $state(false); + + async function loadTopics(idOrSlug: string) { + try { + const r = await api.topics.list(idOrSlug); + topics = r.topics; + } catch { + topics = []; + } + } + + async function submitCreateTopic(e: SubmitEvent) { + e.preventDefault(); + if (!app) return; + creatingTopic = true; + createTopicError = null; + try { + await api.topics.create(app.id, { + name: createTopicName.trim(), + external_subscribable: createTopicExternal, + auth_mode: createTopicAuthMode + }); + createTopicName = ''; + createTopicExternal = false; + createTopicAuthMode = 'public'; + await loadTopics(app.id); + } catch (err) { + createTopicError = + err instanceof ApiError ? err.message : err instanceof Error ? err.message : String(err); + } finally { + creatingTopic = false; + } + } + + function openEditTopic(t: Topic) { + topicToEdit = t; + editTopicExternal = t.external_subscribable; + editTopicAuthMode = t.auth_mode; + editTopicError = null; + } + + async function confirmEditTopic() { + if (!app || !topicToEdit) return; + savingTopic = true; + editTopicError = null; + try { + await api.topics.update(app.id, topicToEdit.name, { + external_subscribable: editTopicExternal, + auth_mode: editTopicAuthMode + }); + topicToEdit = null; + await loadTopics(app.id); + } catch (err) { + editTopicError = + err instanceof ApiError ? err.message : err instanceof Error ? err.message : String(err); + } finally { + savingTopic = false; + } + } + + async function confirmRemoveTopic() { + if (!app || !topicToRemove) return; + removingTopic = true; + try { + await api.topics.remove(app.id, topicToRemove.name); + topicToRemove = null; + await loadTopics(app.id); + } catch (err) { + createTopicError = + err instanceof ApiError ? err.message : err instanceof Error ? err.message : String(err); + } finally { + removingTopic = false; + } + } + // Members tab let eligibleUsers = $state([]); let eligibleLoadError = $state(null); @@ -234,7 +330,12 @@ loadDeadLetterCount(app.id) ]; if (canAdmin) { - loaders.push(loadMembers(app.id), loadEligibleUsers(), loadTriggers(app.id)); + loaders.push( + loadMembers(app.id), + loadEligibleUsers(), + loadTriggers(app.id), + loadTopics(app.id) + ); } await Promise.all(loaders); } catch (e) { @@ -503,7 +604,10 @@ $effect(() => { if ( !canAdmin && - (activeTab === 'settings' || activeTab === 'members' || activeTab === 'triggers') + (activeTab === 'settings' || + activeTab === 'members' || + activeTab === 'triggers' || + activeTab === 'topics') ) { activeTab = 'scripts'; } @@ -551,6 +655,11 @@ class:active={activeTab === 'triggers'} onclick={() => (activeTab = 'triggers')}>Triggers ({triggers.length}) + + + + + {#if topics.length === 0} +

No registered topics in this app yet.

+ {:else} +
    + {#each topics as t (t.name)} +
  • +
    + {t.name} + {#if t.external_subscribable} + + external + + {t.auth_mode} + {:else} + + internal + + {/if} + · {shortDate(t.created_at)} +
    +
    + + +
    +
    • + {/each} +
+ {/if} + {:else if activeTab === 'settings' && canAdmin}

Settings

@@ -1113,6 +1305,65 @@

{/if} + + {#if topicToEdit} + (topicToEdit = null)} + > + + {#if editTopicExternal} +
+ Auth mode + + +
+ {/if} + {#if editFlipToExternal} +
+ Marking {topicToEdit.name} externally-subscribable means + anyone with the URL can subscribe to this topic (if auth_mode is + public) or anyone with a valid token can subscribe (if + auth_mode is token). Are you sure? +
+ {/if} + {#if editTopicError} +

{editTopicError}

+ {/if} + + {/if} + + {#if topicToRemove} + (topicToRemove = null)} + > +

+ Unregistering {topicToRemove.name} disconnects any live SSE + subscribers immediately. Scripts can still publish_durable to + it (internal triggers keep working) — it just won't be externally + subscribable. +

+ + {/if} {/if}