//! §4.5 M5: materialization of STATEFUL group trigger templates. //! //! A group-owned cron / queue / email trigger is a TEMPLATE that is never //! dispatched directly (the scheduler / queue consumer / email webhook all //! filter `t.app_id IS NOT NULL`). Instead this reconciler expands each template //! into an app-owned copy — `materialized_from = template.id` — for every //! descendant app, and removes copies whose (app, template) pair no longer //! inherits. The copy owns its per-app state (cron `last_fired_at`, the queue //! advisory lock), so the unchanged dispatch paths work against it as if it were //! hand-authored. An EMAIL copy is a verbatim byte-copy of the group template's //! sealed inbound secret — the group resolved + sealed it against its own store //! once at apply (shared-group-secret model), and email secrets are v0/no-AAD so //! the ciphertext is portable across rows; no master key is needed here. //! //! Full-live like `rebuild_route_table`: called at the same chokepoints (apply, //! app create/delete, group reparent). A precise create/delete diff (not //! delete-all-then-recreate) preserves `last_fired_at` on unchanged cron copies; //! a DISABLED cron template keeps its copies (their `enabled` is synced off) //! rather than deleting them, so a later re-enable resumes from the preserved //! `last_fired_at` instead of firing from a fresh reference. //! //! Returns per-app WARNINGS (e.g. a queue whose consumer slot is already taken) //! — the caller surfaces them; materialization of the other pairs still //! proceeds. use std::collections::HashSet; use picloud_shared::AppId; use sqlx::PgPool; use uuid::Uuid; /// The stateful kinds that materialize (M5.2 cron; M5.4 queue; M5.5 email). const MATERIALIZED_KINDS: &[&str] = &["cron", "queue", "email"]; /// Fixed advisory-lock key serializing all materialization reconcilers (§4.5 /// M5). Two reconcilers running concurrently (e.g. two parallel app-creates) /// would otherwise (a) read `should` and `existing` at different READ-COMMITTED /// snapshots — a copy created by one between the two reads could be spuriously /// DELETEd by the other (a lost cron copy that only self-heals on the next /// mutation), and (b) both pass the queue one-consumer check and double-fill a /// slot. An xact-scoped advisory lock makes each reconcile atomic w.r.t. the /// others; reconciles are infrequent + fast, so serializing is cheap. const REMATERIALIZE_LOCK_KEY: i64 = 0x0004_0005_0041_054d; // "M5" materialize marker /// (descendant app, source template, kind) that SHOULD have a materialized copy. #[derive(sqlx::FromRow)] struct ShouldRow { effective_app_id: Uuid, template_id: Uuid, kind: String, /// §11.6 D3: a `shared` queue template's copies drain the GROUP store as /// COMPETING consumers, so the one-consumer-per-(app, queue) slot check is /// skipped for them (each descendant intentionally gets a consumer). shared: bool, } /// Reconcile all materialized stateful-template copies. Idempotent; safe to call /// on every tree/apply mutation. Best-effort per pair — a pair that can't /// materialize (queue slot taken, missing email secret) yields a warning and is /// skipped, not an error. pub async fn rematerialize_stateful_templates(pool: &PgPool) -> Result, sqlx::Error> { let mut tx = pool.begin().await?; // Serialize reconcilers (see REMATERIALIZE_LOCK_KEY): the lock is held until // this transaction commits, so the `should`/`existing` reads below are // snapshot-consistent relative to any other reconcile and the queue slot // check can't race. sqlx::query("SELECT pg_advisory_xact_lock($1)") .bind(REMATERIALIZE_LOCK_KEY) .execute(&mut *tx) .await?; // Every (descendant app × ancestor-group stateful template) that should // exist — the all-apps `app_chain` CTE (each app × its ancestor chain) ⋈ // group-owned stateful templates. let kinds: Vec = MATERIALIZED_KINDS .iter() .map(|s| (*s).to_string()) .collect(); let should: Vec = sqlx::query_as( "WITH RECURSIVE app_chain AS ( \ SELECT a.id AS effective_app_id, a.group_id AS owner_group, \ a.group_id AS next_group, 0 AS depth \ FROM apps a WHERE a.group_id IS NOT NULL \ UNION ALL \ SELECT ac.effective_app_id, g.parent_id, g.parent_id, ac.depth + 1 \ FROM groups g JOIN app_chain ac ON g.id = ac.next_group \ WHERE ac.depth < 64 AND g.parent_id IS NOT NULL \ ) \ SELECT DISTINCT ac.effective_app_id, t.id AS template_id, t.kind, t.shared \ FROM app_chain ac \ JOIN triggers t ON t.group_id = ac.owner_group \ WHERE t.group_id IS NOT NULL AND t.kind = ANY($1) \ AND (t.enabled = TRUE OR t.kind = 'cron')", ) .bind(&kinds) .fetch_all(&mut *tx) .await?; let should_set: HashSet<(Uuid, Uuid)> = should .iter() .map(|r| (r.effective_app_id, r.template_id)) .collect(); // Existing managed copies. let existing: Vec<(Uuid, Uuid)> = sqlx::query_as( "SELECT app_id, materialized_from FROM triggers WHERE materialized_from IS NOT NULL", ) .fetch_all(&mut *tx) .await?; let existing_set: HashSet<(Uuid, Uuid)> = existing.iter().copied().collect(); // Delete stale copies (template gone or app no longer a descendant). for (app_id, template_id) in &existing_set { if !should_set.contains(&(*app_id, *template_id)) { sqlx::query("DELETE FROM triggers WHERE app_id = $1 AND materialized_from = $2") .bind(app_id) .bind(template_id) .execute(&mut *tx) .await?; } } // Create missing copies. let mut warnings = Vec::new(); for r in &should { if existing_set.contains(&(r.effective_app_id, r.template_id)) { continue; } if let Some(w) = materialize_one( &mut tx, r.effective_app_id, r.template_id, &r.kind, r.shared, ) .await? { warnings.push(w); } } // Sync each CRON copy's `enabled` to its template's, in place. A cron // template's copies are kept across a disable (the `should` query includes // disabled cron templates), so a disable→re-enable toggles this flag rather // than delete+recreate — preserving each copy's per-app `last_fired_at` (and // its `created_at` reference). Without this, a re-enabled cron would reset to // a fresh reference and lose/shift its schedule state. Cron-only: queue/email // keep delete-on-disable so the queue one-consumer slot is freed on disable. sqlx::query( "UPDATE triggers c SET enabled = t.enabled \ FROM triggers t \ WHERE c.materialized_from = t.id AND t.kind = 'cron' AND c.enabled <> t.enabled", ) .execute(&mut *tx) .await?; tx.commit().await?; Ok(warnings) } /// Create one app-owned copy of a group template + its per-kind detail. Returns /// `Some(warning)` when the pair is intentionally skipped (e.g. a queue slot the /// app already fills). `name` uses the column default (a fresh UUID) so it can't /// collide with the app's own trigger names. async fn materialize_one( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, app_id: Uuid, template_id: Uuid, kind: &str, shared: bool, ) -> Result, sqlx::Error> { // §M5.4: a per-app queue copy would violate the one-consumer-per-(app_id, // queue_name) invariant if the app already has a consumer on that queue — // skip with a warning. §11.6 D3: a SHARED queue copy drains the group store // as a competing consumer (a different store), so the slot check is skipped // — every descendant intentionally gets a consumer. if kind == "queue" && !shared { // Resolve the queue name so we can take the SAME per-(app, queue) // advisory lock the interactive `create_queue_trigger` uses. Without it, // this check-then-insert (serialized only against other reconcilers by // REMATERIALIZE_LOCK) could race an interactive consumer-create and // double-fill the slot — the two paths held different locks. let queue_name: Option<(String,)> = sqlx::query_as("SELECT queue_name FROM queue_trigger_details WHERE trigger_id = $1") .bind(template_id) .fetch_optional(&mut **tx) .await?; if let Some((queue_name,)) = queue_name { sqlx::query("SELECT pg_advisory_xact_lock($1)") .bind(crate::trigger_repo::advisory_lock_key( AppId::from(app_id), &queue_name, )) .execute(&mut **tx) .await?; let taken: Option<(Uuid,)> = sqlx::query_as( "SELECT t.id FROM triggers t \ JOIN queue_trigger_details d ON d.trigger_id = t.id \ WHERE t.app_id = $1 AND t.kind = 'queue' AND d.queue_name = $2", ) .bind(app_id) .bind(&queue_name) .fetch_optional(&mut **tx) .await?; if taken.is_some() { return Ok(Some(format!( "queue template not materialized for one app — it already has a \ consumer on that queue (app_id={app_id})" ))); } } } // Parent row: copy the template's settings, owned by the app, linked back. // `ON CONFLICT DO NOTHING` (against the (app_id, materialized_from) partial // unique index) makes this idempotent under concurrent reconciles — the // loser gets no RETURNING row and skips the detail insert. let new_id: Option<(Uuid,)> = sqlx::query_as( "INSERT INTO triggers ( \ app_id, script_id, kind, enabled, dispatch_mode, \ retry_max_attempts, retry_backoff, retry_base_ms, \ registered_by_principal, materialized_from) \ SELECT $1, script_id, kind, enabled, dispatch_mode, \ retry_max_attempts, retry_backoff, retry_base_ms, \ registered_by_principal, id \ FROM triggers WHERE id = $2 \ ON CONFLICT DO NOTHING RETURNING id", ) .bind(app_id) .bind(template_id) .fetch_optional(&mut **tx) .await?; let Some(new_id) = new_id else { // A concurrent reconcile already created this copy. return Ok(None); }; match kind { "cron" => { // A first-ever copy starts with last_fired_at NULL → its reference // is created_at (now), so it fires at the next scheduled slot, not // immediately. Re-enabling a previously-materialized template does // NOT hit this path — that copy is kept and its `enabled` synced, so // last_fired_at survives (see the enabled-sync UPDATE above). sqlx::query( "INSERT INTO cron_trigger_details (trigger_id, schedule, timezone) \ SELECT $1, schedule, timezone FROM cron_trigger_details WHERE trigger_id = $2", ) .bind(new_id.0) .bind(template_id) .execute(&mut **tx) .await?; } "queue" => { sqlx::query( "INSERT INTO queue_trigger_details \ (trigger_id, queue_name, visibility_timeout_secs) \ SELECT $1, queue_name, visibility_timeout_secs \ FROM queue_trigger_details WHERE trigger_id = $2", ) .bind(new_id.0) .bind(template_id) .execute(&mut **tx) .await?; } "email" => { // §M5.5: the group template sealed its inbound HMAC secret against // the GROUP's own store once at apply (shared-group-secret model). // The secret's AAD (§M5.5 / audit H-D1) is bound to the GROUP owner, // NOT this copy's row, so a verbatim byte-copy stays openable — the // inbound path recovers the group via `materialized_from`. Copy the // version verbatim too. Each copy has its own trigger_id (its own // inbound webhook URL). sqlx::query( "INSERT INTO email_trigger_details \ (trigger_id, inbound_secret_encrypted, inbound_secret_nonce, \ inbound_secret_version) \ SELECT $1, inbound_secret_encrypted, inbound_secret_nonce, \ inbound_secret_version \ FROM email_trigger_details WHERE trigger_id = $2", ) .bind(new_id.0) .bind(template_id) .execute(&mut **tx) .await?; } _ => {} } Ok(None) }