feat(stateful-templates): journey + docs + concurrency fix; email deferred (M5.6)

- 0063: partial unique index on (app_id, materialized_from) + ON CONFLICT DO
  NOTHING in the reconciler, so a materialized copy is idempotent under
  concurrent reconciles (two parallel app-creates no longer duplicate a copy).
- stateful_templates journey: a group cron template + a descendant app → a
  materialized cron copy in `pic triggers ls --app`; re-apply is a NoOp.
- docs (§4.5 + CLAUDE.md): cron+queue materialization implemented; group EMAIL
  templates deferred (per-descendant inbound-secret reseal needs the master key
  threaded through the hooks + a secret-ref schema) and cleanly rejected at
  apply for now, alongside a `materialized` ls column.

Completes the v1.2 near-term batch (M1-M5, cron+queue); group email is the one
scoped follow-up.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

crates/manager-core/migrations/0063_materialized_unique.sql

@@ -0,0 +1,12 @@
+-- §4.5 M5: exactly one materialized copy per (descendant app, source template).
+--
+-- The materialization reconciler (materialize::rematerialize_stateful_templates)
+-- runs on every tree/apply mutation, and two of them can run concurrently (e.g.
+-- two apps created in parallel each trigger a full reconcile). Both compute the
+-- same "missing" (app, template) pair and both try to insert the copy — without
+-- a constraint that races into a DUPLICATE. This partial unique index makes the
+-- second insert a no-op (the reconciler uses `ON CONFLICT DO NOTHING`), so a
+-- copy is idempotent under concurrency.
+
+CREATE UNIQUE INDEX triggers_materialized_uidx
+    ON triggers (app_id, materialized_from) WHERE materialized_from IS NOT NULL;

crates/manager-core/src/materialize.rs

@@ -141,7 +141,10 @@ async fn materialize_one(
     }
 
     // Parent row: copy the template's settings, owned by the app, linked back.
-    let new_id: (Uuid,) = sqlx::query_as(
+    // `ON CONFLICT DO NOTHING` (against the (app_id, materialized_from) partial
+    // unique index) makes this idempotent under concurrent reconciles — the
+    // loser gets no RETURNING row and skips the detail insert.
+    let new_id: Option<(Uuid,)> = sqlx::query_as(
         "INSERT INTO triggers ( \
             app_id, script_id, kind, enabled, dispatch_mode, \
             retry_max_attempts, retry_backoff, retry_base_ms, \
@@ -149,12 +152,17 @@ async fn materialize_one(
          SELECT $1, script_id, kind, enabled, dispatch_mode, \
                 retry_max_attempts, retry_backoff, retry_base_ms, \
                 registered_by_principal, id \
-         FROM triggers WHERE id = $2 RETURNING id",
+         FROM triggers WHERE id = $2 \
+         ON CONFLICT DO NOTHING RETURNING id",
     )
     .bind(app_id)
     .bind(template_id)
-    .fetch_one(&mut **tx)
+    .fetch_optional(&mut **tx)
     .await?;
+    let Some(new_id) = new_id else {
+        // A concurrent reconcile already created this copy.
+        return Ok(None);
+    };
 
     match kind {
         "cron" => {

crates/manager-core/tests/expected_schema.txt

@@ -635,6 +635,7 @@ indexes on triggers:
   idx_triggers_materialized_from: public.triggers USING btree (materialized_from) WHERE (materialized_from IS NOT NULL)
   triggers_app_name_uniq: public.triggers USING btree (app_id, name) WHERE (app_id IS NOT NULL)
   triggers_group_name_uniq: public.triggers USING btree (group_id, name) WHERE (group_id IS NOT NULL)
+  triggers_materialized_uidx: public.triggers USING btree (app_id, materialized_from) WHERE (materialized_from IS NOT NULL)
   triggers_pkey: public.triggers USING btree (id)
 
 indexes on vars:
@@ -946,3 +947,4 @@ constraints on vars:
   0060: group suppressions
   0061: shared triggers
   0062: materialized triggers
+  0063: materialized unique

crates/picloud-cli/tests/cli.rs

@@ -45,6 +45,7 @@ mod sealed;
 mod secrets;
 mod shared_triggers;
 mod staleness;
+mod stateful_templates;
 mod suppress;
 mod tree;
 mod triggers;

crates/picloud-cli/tests/stateful_templates.rs

@@ -0,0 +1,120 @@
+//! §4.5 M5 — stateful (cron/queue) group trigger templates, declarative end to
+//! end via `pic`. A group declares a cron template; applying it and creating a
+//! descendant app materializes a per-app cron copy (visible in `pic triggers ls
+//! --app`). The reconcile mechanism itself (per-app copy, idempotent,
+//! de-materialize on reparent, queue slot-conflict warning) is pinned
+//! deterministically by `manager-core/tests/stateful_templates.rs`; this journey
+//! exercises the CLI authoring plus the apply and app-create hooks.
+
+use std::fs;
+
+use tempfile::TempDir;
+
+use crate::common;
+use crate::common::cleanup::{AppGuard, GroupGuard, ScriptGuard};
+
+fn manifest_dir() -> TempDir {
+    let dir = TempDir::new().expect("tempdir");
+    fs::create_dir_all(dir.path().join("scripts")).expect("scripts dir");
+    dir
+}
+
+fn group_script_id(env: &common::TestEnv, group: &str, name: &str) -> String {
+    let ls = common::pic_as(env)
+        .args(["scripts", "ls", "--group", group])
+        .output()
+        .expect("scripts ls");
+    let table = String::from_utf8(ls.stdout).unwrap();
+    table
+        .lines()
+        .map(common::cells)
+        .find(|c| c.get(2) == Some(&name))
+        .and_then(|c| c.first().map(|s| (*s).to_string()))
+        .unwrap_or_else(|| panic!("group script `{name}` not found:\n{table}"))
+}
+
+/// Count `cron` rows in `pic triggers ls --app`.
+fn app_cron_count(env: &common::TestEnv, app: &str) -> usize {
+    let out = common::pic_as(env)
+        .args(["triggers", "ls", "--app", app])
+        .output()
+        .expect("triggers ls");
+    String::from_utf8(out.stdout)
+        .unwrap()
+        .lines()
+        .map(common::cells)
+        .filter(|c| c.contains(&"cron"))
+        .count()
+}
+
+#[ignore = "needs DATABASE_URL pointing at a running Postgres"]
+#[test]
+fn group_cron_template_materializes_for_descendant_apps() {
+    let Some(fx) = common::fixture_or_skip() else {
+        return;
+    };
+    let env = common::admin_env(fx);
+    let group = common::unique_slug("mat-grp");
+
+    let _g = GroupGuard::new(&env.url, &env.token, &group);
+    common::pic_as(&env)
+        .args(["groups", "create", &group])
+        .assert()
+        .success();
+
+    // Group handler + a cron TEMPLATE binding it.
+    let dir = manifest_dir();
+    fs::write(
+        dir.path().join("scripts/nightly.rhai"),
+        r#"log::info("nightly"); "ok""#,
+    )
+    .unwrap();
+    common::pic_as(&env)
+        .args(["scripts", "deploy"])
+        .arg(dir.path().join("scripts/nightly.rhai"))
+        .args(["--group", &group, "--name", "nightly"])
+        .assert()
+        .success();
+    let _gs = ScriptGuard::new(
+        &env.url,
+        &env.token,
+        &group_script_id(&env, &group, "nightly"),
+    );
+    let gmanifest = format!(
+        "[group]\nslug = \"{group}\"\nname = \"MatG\"\n\n\
+         [[triggers.cron]]\nscript = \"nightly\"\nschedule = \"0 0 * * * *\"\n"
+    );
+    let gpath = dir.path().join("group.toml");
+    fs::write(&gpath, &gmanifest).unwrap();
+    common::pic_as(&env)
+        .args(["apply", "--file"])
+        .arg(&gpath)
+        .assert()
+        .success();
+
+    // An app created under the group materializes a per-app cron copy (the
+    // app-create hook reconciles).
+    let app = common::unique_slug("mat-app");
+    let _a = AppGuard::new(&env.url, &env.token, &app);
+    common::pic_as(&env)
+        .args(["apps", "create", &app, "--group", &group])
+        .assert()
+        .success();
+    assert_eq!(
+        app_cron_count(&env, &app),
+        1,
+        "a descendant app must have a materialized cron trigger"
+    );
+
+    // Re-apply of the group is a NoOp for materialization (no duplicate copy).
+    common::pic_as(&env)
+        .args(["apply", "--file"])
+        .arg(&gpath)
+        .assert()
+        .success();
+    assert_eq!(
+        app_cron_count(&env, &app),
+        1,
+        "re-apply must not duplicate the materialized copy"
+    );
+}

docs/design/groups-and-project-tool.md

@@ -390,10 +390,23 @@ Two distinct constraints:
 > matches its own triggers **plus** ancestor-group templates in one query, the handler running under the
 > firing app's `app_id` (the Phase-4 boundary). **That walk is the isolation boundary**: a
 > sibling-subtree app never sees the template (pinned by `tests/group_trigger_templates.rs`). Authoring
-> is `[[triggers.kv]]` (etc.) on a `[group]`; read-only `pic triggers ls --group`. **Deferred:** the
-> **stateful** kinds (cron `last_fired_at`, queue advisory-lock, email sealed secret) need per-app rows
-> → materialization, rejected on a `[group]`; per-app opt-out / cross-owner dedup (a descendant
-> re-declaring an identical trigger double-fires — "overlapping triggers coexist").
+> is `[[triggers.kv]]` (etc.) on a `[group]`; read-only `pic triggers ls --group`.
+>
+> **Stateful templates via MATERIALIZATION (✅, M5 — cron + queue).** The stateful kinds can't resolve
+> live (each needs a per-app row: cron `last_fired_at`, the queue one-consumer advisory lock), so a group
+> `[[triggers.cron]]` / `[[triggers.queue]]` template is **materialized** into an app-owned copy per
+> descendant — `materialized_from = template.id` (`0062`) — by `materialize::rematerialize_stateful_templates`.
+> That reconciler (all-apps `app_chain` CTE ⋈ group-owned stateful templates, a precise create/delete diff
+> that preserves `last_fired_at`) runs full-live at the route-rebuild chokepoints: apply (single + tree),
+> app create/delete, group reparent. The dispatch paths are **unchanged** — the scheduler + queue consumer
+> gained `AND t.app_id IS NOT NULL`, so a group TEMPLATE is never dispatched directly; only its per-app
+> copies are. A queue copy is **skipped-with-warning** when the app already fills that queue's consumer
+> slot (the one-consumer invariant). Pinned by `tests/stateful_templates.rs` + the `stateful_templates`
+> journey. **Deferred:** **email** templates (per-descendant inbound-secret resolution + resealing needs
+> the master key threaded through the materialization hooks + a nullable-ciphertext / secret-ref schema —
+> a bounded follow-up; a `[group]` email template is cleanly rejected at apply for now); a per-app
+> `materialized` column in `pic triggers ls --app`; cross-owner dedup (a descendant re-declaring an
+> identical trigger double-fires — "overlapping triggers coexist").
 >
 > **Shipped — group ROUTE templates (live, inherited).** The same live model, adapted to routes. A
 > `[group]` declares a `[[routes]]` template binding a group-owned endpoint; `routes` gained a