From 3b650a2b146ffc5fb5826806a4c968708f435ca9 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sat, 20 Jun 2026 21:52:21 +0200 Subject: [PATCH] feat: declarative project-tool foundation (pull/plan/apply/prune) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a server-side, atomic, declarative reconcile loop for a single app — the foundation of the project-tool design. Developers describe an app's scripts, routes, triggers, and secret-names in `picloud.toml`, then `pic pull / plan / apply [--prune]` to converge live state to the manifest. Server (manager-core): - apply_service: a pure diff engine (compute_diff) shared by plan and apply, plus an ApplyService that composes the existing per-repo writes into ONE Postgres transaction. Identity keys mirror the DB UNIQUE constraints (script=lower(name); route=(method,host_kind,host, path_kind,path); trigger=per-kind semantic tuple; secret=name). Apply takes a per-app advisory lock, recomputes the diff in-tx, applies scripts -> routes -> triggers, prunes dependents-first, commits, then refreshes the route table once post-commit. - apply_api: POST /apps/{id}/plan (AppRead) and /apps/{id}/apply. Apply requires the per-kind write caps the bundle exercises (all three when --prune), plus AppSecretsRead when it binds an email trigger. - tx-accepting repo siblings (insert/update/delete *_tx) so the existing create/update/delete delegate to one SQL definition each. - email triggers reference an inbound secret by NAME; the value is resolved, decrypted (AAD-bound), and re-sealed server-side at apply — it never travels in the manifest. CLI (picloud-cli): - manifest.rs (picloud.toml model), client plan/apply, and the pull/plan/ apply commands. pull rejects filesystem-unsafe script names up front. Safety properties enforced and tested: - idempotent: a freshly-pulled manifest re-applies as all-NoOp. - atomic: a mid-bundle failure rolls back with nothing written. - routes delete-before-insert so a freed binding is reusable in one apply. - queue one-consumer invariant held inside the shared tx. - email triggers are never pruned, and a script that still owns an email/dead-letter trigger can't be pruned (the FK cascade would destroy the sealed secret) — refused with a pointer to `pic triggers rm`. - plan and apply agree on unset email-secret references. No migration: the existing schema's UNIQUE constraints serve as identity keys. Groups, env-scoping, and the `enabled` toggle are later milestones. Tested: manager-core lib (360) + CLI bins (27) + 8 project-tool journeys (pull/plan/apply/prune/email+queue), all green; clippy -D warnings clean. Co-Authored-By: Claude Opus 4.8 --- crates/manager-core/src/apply_api.rs | 160 ++ crates/manager-core/src/apply_service.rs | 1880 ++++++++++++++++++++++ crates/manager-core/src/lib.rs | 4 + crates/manager-core/src/repo.rs | 200 +-- crates/manager-core/src/route_admin.rs | 2 +- crates/manager-core/src/route_repo.rs | 84 +- crates/manager-core/src/trigger_repo.rs | 214 +++ crates/picloud-cli/src/client.rs | 74 + crates/picloud-cli/src/cmds/apply.rs | 52 + crates/picloud-cli/src/cmds/mod.rs | 3 + crates/picloud-cli/src/cmds/plan.rs | 123 ++ crates/picloud-cli/src/cmds/pull.rs | 285 ++++ crates/picloud-cli/src/main.rs | 44 + crates/picloud-cli/src/manifest.rs | 362 +++++ crates/picloud-cli/tests/apply.rs | 153 ++ crates/picloud-cli/tests/cli.rs | 5 + crates/picloud-cli/tests/email_queue.rs | 222 +++ crates/picloud-cli/tests/plan.rs | 81 + crates/picloud-cli/tests/prune.rs | 111 ++ crates/picloud-cli/tests/pull.rs | 91 ++ crates/picloud/src/lib.rs | 34 +- 21 files changed, 4055 insertions(+), 129 deletions(-) create mode 100644 crates/manager-core/src/apply_api.rs create mode 100644 crates/manager-core/src/apply_service.rs create mode 100644 crates/picloud-cli/src/cmds/apply.rs create mode 100644 crates/picloud-cli/src/cmds/plan.rs create mode 100644 crates/picloud-cli/src/cmds/pull.rs create mode 100644 crates/picloud-cli/src/manifest.rs create mode 100644 crates/picloud-cli/tests/apply.rs create mode 100644 crates/picloud-cli/tests/email_queue.rs create mode 100644 crates/picloud-cli/tests/plan.rs create mode 100644 crates/picloud-cli/tests/prune.rs create mode 100644 crates/picloud-cli/tests/pull.rs diff --git a/crates/manager-core/src/apply_api.rs b/crates/manager-core/src/apply_api.rs new file mode 100644 index 0000000..bf8f071 --- /dev/null +++ b/crates/manager-core/src/apply_api.rs @@ -0,0 +1,160 @@ +//! Admin HTTP surface for the declarative reconcile engine. +//! +//! `POST /api/v1/admin/apps/{id}/plan` — diff a desired-state bundle +//! against the app's live state and return the plan. Read-only; requires +//! `AppRead`. The `apply` route (write path) lands in the next milestone. + +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::post, + Extension, Json, Router, +}; +use picloud_shared::{AppId, Principal}; +use serde::Deserialize; +use serde_json::json; + +use crate::app_repo::AppRepository; +use crate::apply_service::{ApplyError, ApplyReport, ApplyService, Bundle, BundleTrigger, Plan}; +use crate::authz::{require, AuthzDenied, Capability}; + +/// Build the apply/plan router. Mounted under `/api/v1/admin`. +pub fn apply_router(service: ApplyService) -> Router { + Router::new() + .route("/apps/{id}/plan", post(plan_handler)) + .route("/apps/{id}/apply", post(apply_handler)) + .with_state(service) +} + +#[derive(Deserialize)] +pub struct ApplyRequest { + pub bundle: Bundle, + #[serde(default)] + pub prune: bool, +} + +async fn apply_handler( + State(svc): State, + Extension(principal): Extension, + Path(id_or_slug): Path, + Json(req): Json, +) -> Result, ApplyError> { + let app_id = resolve_app_id(svc.apps.as_ref(), &id_or_slug).await?; + // Read is always needed; write caps are required for the resource kinds + // the bundle touches — and for ALL kinds when `prune` is set, since + // pruning deletes resources whose bundle section is empty (and a script + // delete cascades its routes/triggers). + require(svc.authz.as_ref(), &principal, Capability::AppRead(app_id)) + .await + .map_err(map_authz)?; + if req.prune || !req.bundle.scripts.is_empty() { + require( + svc.authz.as_ref(), + &principal, + Capability::AppWriteScript(app_id), + ) + .await + .map_err(map_authz)?; + } + if req.prune || !req.bundle.routes.is_empty() { + require( + svc.authz.as_ref(), + &principal, + Capability::AppWriteRoute(app_id), + ) + .await + .map_err(map_authz)?; + } + if req.prune || !req.bundle.triggers.is_empty() { + require( + svc.authz.as_ref(), + &principal, + Capability::AppManageTriggers(app_id), + ) + .await + .map_err(map_authz)?; + } + // Email triggers resolve and decrypt a stored secret by name server-side, + // which the secrets API guards with `AppSecretsRead`. Require it here too + // so apply can't bind a secret a principal couldn't otherwise read — the + // caps aren't strictly nested on the API-key scope path. + if req.bundle.triggers.iter().any(BundleTrigger::is_email) { + require( + svc.authz.as_ref(), + &principal, + Capability::AppSecretsRead(app_id), + ) + .await + .map_err(map_authz)?; + } + let report = svc + .apply(app_id, &req.bundle, req.prune, principal.user_id) + .await?; + Ok(Json(report)) +} + +async fn plan_handler( + State(svc): State, + Extension(principal): Extension, + Path(id_or_slug): Path, + Json(bundle): Json, +) -> Result, ApplyError> { + let app_id = resolve_app_id(svc.apps.as_ref(), &id_or_slug).await?; + // NOTE: the returned `Plan` discloses live secret NAMES (not values). That + // is safe today only because `AppRead` and `AppSecretsRead` are co-granted + // at every tier (same `script:read` scope, both in the viewer role). If a + // future authz split puts `AppSecretsRead` on its own tier, this handler + // must additionally require it — otherwise it leaks names a principal + // couldn't enumerate via the secrets API. + require(svc.authz.as_ref(), &principal, Capability::AppRead(app_id)) + .await + .map_err(map_authz)?; + let plan = svc.plan(app_id, &bundle).await?; + Ok(Json(plan)) +} + +/// Resolve a slug-or-id path param to an `AppId`, mapping miss → 404. +/// Mirrors the `triggers_api` helper of the same shape. +async fn resolve_app_id(apps: &dyn AppRepository, ident: &str) -> Result { + crate::app_repo::resolve_app(apps, ident) + .await + .map_err(|e| ApplyError::Backend(e.to_string()))? + .map(|l| l.app.id) + .ok_or_else(|| ApplyError::AppNotFound(ident.to_string())) +} + +fn map_authz(denied: AuthzDenied) -> ApplyError { + match denied { + AuthzDenied::Denied => ApplyError::Forbidden, + AuthzDenied::Repo(e) => ApplyError::AuthzRepo(e.to_string()), + } +} + +impl IntoResponse for ApplyError { + fn into_response(self) -> Response { + let (status, body) = match &self { + Self::AppNotFound(_) => (StatusCode::NOT_FOUND, json!({ "error": self.to_string() })), + Self::Invalid(_) => ( + StatusCode::UNPROCESSABLE_ENTITY, + json!({ "error": self.to_string() }), + ), + Self::Forbidden => (StatusCode::FORBIDDEN, json!({ "error": self.to_string() })), + Self::AuthzRepo(e) => { + tracing::error!(error = %e, "apply authz repo error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + json!({ "error": "internal error" }), + ) + } + Self::Backend(e) => { + tracing::error!(error = %e, "apply backend error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + json!({ "error": "internal error" }), + ) + } + }; + (status, Json(body)).into_response() + } +} diff --git a/crates/manager-core/src/apply_service.rs b/crates/manager-core/src/apply_service.rs new file mode 100644 index 0000000..1d346f3 --- /dev/null +++ b/crates/manager-core/src/apply_service.rs @@ -0,0 +1,1880 @@ +//! Declarative reconcile engine for a single app. +//! +//! Takes a desired-state [`Bundle`] (an app's scripts, routes, triggers, +//! and the names of its secrets) and either diffs it against the live +//! database ([`ApplyService::plan`], read-only) or reconciles it in one +//! transaction ([`ApplyService::apply`]). +//! +//! Identity keys match the DB UNIQUE constraints so the diff is stable: +//! scripts by `lower(name)`, routes by the `(method, host, path)` tuple, +//! triggers by their per-kind semantic tuple, secrets by name. +//! +//! **Trigger-identity limitation:** a trigger's identity *is* its semantic +//! definition, so the diff only ever Creates or Deletes (a change is +//! delete-old + create-new). For queue the identity is `queue|{queue_name}` +//! (script-independent — it encodes the one-consumer-per-queue invariant) +//! and for email it is `email|{script}`. Consequence: rebinding a queue to +//! a different script, changing its visibility timeout, or rotating an email +//! trigger's secret/reference diffs as a **NoOp and is not applied**. To +//! change those, recreate the trigger (drop it from the manifest, apply +//! `--prune`, then re-add it). +//! +//! Other known limitations vs. the interactive API: a script **rename** +//! diffs as delete+create, so under `--prune` it loses the old script's id, +//! version counter, and execution logs; an `endpoint`→`module` kind flip is +//! not guarded against *live* (un-pruned) routes/triggers still bound to the +//! script; and cross-pattern route conflict-vs-live is not checked (only +//! identical tuples collide). See `validate_bundle`. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::Arc; + +use picloud_orchestrator_core::routing::{pattern, RouteTable}; +use picloud_shared::{ + AdminUserId, AppId, DispatchMode, DocsEventOp, FilesEventOp, HostKind, KvEventOp, MasterKey, + PathKind, Route, Script, ScriptId, ScriptKind, ScriptSandbox, ScriptValidator, TriggerId, +}; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; + +use crate::app_domain_repo::AppDomainRepository; +use crate::app_repo::AppRepository; +use crate::authz::AuthzRepo; +use crate::repo::{ + delete_script_tx, insert_script_tx, update_script_tx, NewScript, ScriptPatch, ScriptRepository, + ScriptRepositoryError, +}; +use crate::route_repo::{delete_route_tx, insert_route_tx, NewRoute, RouteRepository}; +use crate::sandbox::SandboxCeiling; +use crate::secrets_repo::SecretsRepo; +use crate::trigger_config::TriggerConfig; +use crate::trigger_repo::{ + delete_trigger_tx, insert_email_trigger_tx, insert_trigger_tx, Trigger, TriggerDetails, + TriggerDispatchMode, TriggerRepo, TriggerRepoError, +}; + +/// Reserved path prefixes that user routes may not claim (mirrors the +/// route-create rejection — see CLAUDE.md "Reserved path prefixes"). +const RESERVED_PATH_PREFIXES: &[&str] = &["/api/", "/admin/"]; +const RESERVED_PATH_EXACT: &[&str] = &["/healthz", "/version"]; + +// ---------------------------------------------------------------------------- +// Wire types — desired state (request) and the computed plan (response) +// ---------------------------------------------------------------------------- + +/// Desired state for one app, sent by `pic plan` / `pic apply`. +#[derive(Debug, Clone, Deserialize)] +pub struct Bundle { + #[serde(default)] + pub scripts: Vec, + #[serde(default)] + pub routes: Vec, + #[serde(default)] + pub triggers: Vec, + /// Declared secret *names* only; values are pushed out-of-band. + #[serde(default)] + pub secrets: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct BundleScript { + pub name: String, + pub source: String, + #[serde(default)] + pub kind: ScriptKind, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub timeout_seconds: Option, + #[serde(default)] + pub memory_limit_mb: Option, + #[serde(default)] + pub sandbox: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct BundleRoute { + /// Name of the script this route binds to. + pub script: String, + #[serde(default)] + pub method: Option, + pub host_kind: HostKind, + #[serde(default)] + pub host: String, + #[serde(default)] + pub host_param_name: Option, + pub path_kind: PathKind, + pub path: String, + #[serde(default)] + pub dispatch_mode: DispatchMode, +} + +/// Desired trigger, tagged by kind on the wire (`{ "kind": "cron", … }`). +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BundleTrigger { + Kv { + script: String, + collection_glob: String, + #[serde(default)] + ops: Vec, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Docs { + script: String, + collection_glob: String, + #[serde(default)] + ops: Vec, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Files { + script: String, + collection_glob: String, + #[serde(default)] + ops: Vec, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Cron { + script: String, + schedule: String, + #[serde(default = "default_timezone")] + timezone: String, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Pubsub { + script: String, + topic_pattern: String, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Email { + script: String, + /// Name of the secret (set via `pic secret set`) holding the + /// inbound HMAC value; resolved + sealed at apply time. Never the + /// value itself. + inbound_secret_ref: String, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, + Queue { + script: String, + queue_name: String, + #[serde(default)] + visibility_timeout_secs: Option, + #[serde(default)] + dispatch_mode: Option, + #[serde(default)] + retry_max_attempts: Option, + }, +} + +fn default_timezone() -> String { + "UTC".to_string() +} + +impl BundleTrigger { + /// The script name this trigger fires. + #[must_use] + pub fn script(&self) -> &str { + match self { + Self::Kv { script, .. } + | Self::Docs { script, .. } + | Self::Files { script, .. } + | Self::Cron { script, .. } + | Self::Pubsub { script, .. } + | Self::Email { script, .. } + | Self::Queue { script, .. } => script, + } + } + + /// Whether this trigger is an email trigger, which resolves and decrypts + /// a stored secret by reference at apply time (gating an extra capability). + #[must_use] + pub fn is_email(&self) -> bool { + matches!(self, Self::Email { .. }) + } + + /// Stable semantic identity (the per-kind tuple), used for the diff. + #[must_use] + pub fn identity(&self) -> String { + match self { + Self::Kv { + script, + collection_glob, + ops, + .. + } => format!( + "kv|{script}|{collection_glob}|{}", + sorted_csv(ops.iter().copied().map(KvEventOp::as_str)) + ), + Self::Docs { + script, + collection_glob, + ops, + .. + } => format!( + "docs|{script}|{collection_glob}|{}", + sorted_csv(ops.iter().copied().map(DocsEventOp::as_str)) + ), + Self::Files { + script, + collection_glob, + ops, + .. + } => format!( + "files|{script}|{collection_glob}|{}", + sorted_csv(ops.iter().copied().map(FilesEventOp::as_str)) + ), + Self::Cron { + script, + schedule, + timezone, + .. + } => format!("cron|{script}|{schedule}|{timezone}"), + Self::Pubsub { + script, + topic_pattern, + .. + } => format!("pubsub|{script}|{topic_pattern}"), + Self::Email { script, .. } => format!("email|{script}"), + Self::Queue { queue_name, .. } => format!("queue|{queue_name}"), + } + } + + #[must_use] + pub fn kind_str(&self) -> &'static str { + match self { + Self::Kv { .. } => "kv", + Self::Docs { .. } => "docs", + Self::Files { .. } => "files", + Self::Cron { .. } => "cron", + Self::Pubsub { .. } => "pubsub", + Self::Email { .. } => "email", + Self::Queue { .. } => "queue", + } + } +} + +/// One change in the plan. `key` is the human-renderable identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResourceChange { + pub op: Op, + pub key: String, + /// Optional one-line note (e.g. why an update, or "value must be set"). + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Op { + Create, + Update, + NoOp, + Delete, +} + +/// The computed diff, grouped by resource kind. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Plan { + pub scripts: Vec, + pub routes: Vec, + pub triggers: Vec, + pub secrets: Vec, +} + +impl Plan { + /// True when every resource is a no-op (nothing to apply). + #[must_use] + pub fn is_noop(&self) -> bool { + self.scripts + .iter() + .chain(&self.routes) + .chain(&self.triggers) + .chain(&self.secrets) + .all(|c| c.op == Op::NoOp) + } +} + +// ---------------------------------------------------------------------------- +// Current state snapshot +// ---------------------------------------------------------------------------- + +/// The app's live state, loaded once for a diff. +#[derive(Debug, Default)] +pub struct CurrentState { + pub scripts: Vec