feat(stateful-templates): materialize cron+queue templates + hooks (M5.2-M5.4)

materialize::rematerialize_stateful_templates reconciles a group cron/queue
template into one app-owned copy per descendant (materialized_from = template),
via the all-apps app_chain CTE ⋈ group-owned stateful templates. A precise
create/delete diff preserves cron last_fired_at on unchanged copies; a queue
copy is skipped-with-warning when the app already fills that queue's consumer
slot (the one-consumer invariant). Called full-live at the route-rebuild
chokepoints: apply (single + tree), app create/delete (apps_api), group
reparent (groups_api) — AppsState/GroupsState gain a pool. Pinned by
tests/stateful_templates.rs (per-app copy, idempotent, new-app materializes,
reparent de-materializes).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

crates/manager-core/tests/stateful_templates.rs

@@ -0,0 +1,198 @@
+//! §4.5 M5 integration test: materialization of STATEFUL group trigger
+//! templates. A group cron template materializes one app-owned copy per
+//! descendant app; a new app materializes on reconcile; an app leaving the
+//! subtree de-materializes; a group queue template skips (with a warning) an app
+//! that already fills that queue's consumer slot.
+//!
+//! Drives `rematerialize_stateful_templates` directly. Skips when `DATABASE_URL`
+//! is unset.
+
+#![allow(clippy::too_many_lines, clippy::many_single_char_names)]
+
+use picloud_manager_core::materialize::rematerialize_stateful_templates;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+async fn pool_or_skip() -> Option<PgPool> {
+    let Ok(url) = std::env::var("DATABASE_URL") else {
+        eprintln!("stateful_templates: DATABASE_URL unset — skipping");
+        return None;
+    };
+    let pool = PgPoolOptions::new()
+        .max_connections(2)
+        .connect(&url)
+        .await
+        .expect("connect");
+    sqlx::migrate!("./migrations")
+        .run(&pool)
+        .await
+        .expect("migrate");
+    Some(pool)
+}
+
+/// Count materialized copies of `template_id` owned by `app_id`.
+async fn copies(pool: &PgPool, app_id: Uuid, template_id: Uuid) -> i64 {
+    let (n,): (i64,) = sqlx::query_as(
+        "SELECT COUNT(*) FROM triggers WHERE app_id = $1 AND materialized_from = $2",
+    )
+    .bind(app_id)
+    .bind(template_id)
+    .fetch_one(pool)
+    .await
+    .unwrap();
+    n
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn cron_template_materializes_per_descendant_and_reconciles() {
+    let Some(pool) = pool_or_skip().await else {
+        return;
+    };
+    let sfx = Uuid::new_v4().simple().to_string();
+    let admin = {
+        let r: (Uuid,) = sqlx::query_as(
+            "INSERT INTO admin_users (username, password_hash) VALUES ($1, 'x') RETURNING id",
+        )
+        .bind(format!("mat-{sfx}"))
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+        r.0
+    };
+    // Group G with a handler + a cron TEMPLATE.
+    let g: (Uuid,) = sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
+        .bind(format!("mat-g-{sfx}"))
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+    let handler: (Uuid,) = sqlx::query_as(
+        "INSERT INTO scripts (name, source, group_id) VALUES ($1, 'x', $2) RETURNING id",
+    )
+    .bind(format!("nightly-{sfx}"))
+    .bind(g.0)
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    let tmpl: (Uuid,) = sqlx::query_as(
+        "INSERT INTO triggers \
+            (group_id, script_id, kind, enabled, dispatch_mode, retry_max_attempts, \
+             retry_backoff, retry_base_ms, registered_by_principal, name) \
+         VALUES ($1, $2, 'cron', TRUE, 'async', 3, 'exponential', 1000, $3, $4) RETURNING id",
+    )
+    .bind(g.0)
+    .bind(handler.0)
+    .bind(admin)
+    .bind(format!("t-{sfx}"))
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    sqlx::query(
+        "INSERT INTO cron_trigger_details (trigger_id, schedule, timezone) \
+         VALUES ($1, '0 0 * * * *', 'UTC')",
+    )
+    .bind(tmpl.0)
+    .execute(&pool)
+    .await
+    .unwrap();
+
+    // Two apps under G.
+    let mk_app = |slug: String| {
+        let pool = pool.clone();
+        let g = g.0;
+        async move {
+            let r: (Uuid,) = sqlx::query_as(
+                "INSERT INTO apps (slug, name, group_id) VALUES ($1, $1, $2) RETURNING id",
+            )
+            .bind(slug)
+            .bind(g)
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+            r.0
+        }
+    };
+    let a = mk_app(format!("mat-a-{sfx}")).await;
+    let b = mk_app(format!("mat-b-{sfx}")).await;
+
+    // Reconcile → one copy per app.
+    let w = rematerialize_stateful_templates(&pool).await.unwrap();
+    assert!(w.is_empty(), "no warnings expected: {w:?}");
+    assert_eq!(copies(&pool, a, tmpl.0).await, 1, "app A materialized");
+    assert_eq!(copies(&pool, b, tmpl.0).await, 1, "app B materialized");
+
+    // Idempotent: a second reconcile does not duplicate.
+    rematerialize_stateful_templates(&pool).await.unwrap();
+    assert_eq!(copies(&pool, a, tmpl.0).await, 1, "no duplicate copy");
+
+    // The copy is app-owned + linked back to the template + carries the detail.
+    let (kind, sched): (String, String) = sqlx::query_as(
+        "SELECT t.kind, d.schedule FROM triggers t \
+         JOIN cron_trigger_details d ON d.trigger_id = t.id \
+         WHERE t.app_id = $1 AND t.materialized_from = $2",
+    )
+    .bind(a)
+    .bind(tmpl.0)
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    assert_eq!(kind, "cron");
+    assert_eq!(sched, "0 0 * * * *");
+
+    // A new app under G materializes on the next reconcile.
+    let c = mk_app(format!("mat-c-{sfx}")).await;
+    rematerialize_stateful_templates(&pool).await.unwrap();
+    assert_eq!(copies(&pool, c, tmpl.0).await, 1, "new app materialized");
+
+    // Reparent app A into a SIBLING group (not under G) → its copy
+    // de-materializes.
+    let g2: (Uuid,) =
+        sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
+            .bind(format!("mat-g2-{sfx}"))
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+    sqlx::query("UPDATE apps SET group_id = $2 WHERE id = $1")
+        .bind(a)
+        .bind(g2.0)
+        .execute(&pool)
+        .await
+        .unwrap();
+    rematerialize_stateful_templates(&pool).await.unwrap();
+    assert_eq!(
+        copies(&pool, a, tmpl.0).await,
+        0,
+        "an app that left the subtree de-materializes"
+    );
+    assert_eq!(
+        copies(&pool, b, tmpl.0).await,
+        1,
+        "a still-descendant app keeps its copy"
+    );
+
+    // Cleanup (materialized copies CASCADE via app delete / template delete).
+    let _ = sqlx::query("DELETE FROM triggers WHERE materialized_from = $1")
+        .bind(tmpl.0)
+        .execute(&pool)
+        .await;
+    let _ = sqlx::query("DELETE FROM triggers WHERE id = $1")
+        .bind(tmpl.0)
+        .execute(&pool)
+        .await;
+    let _ = sqlx::query("DELETE FROM apps WHERE id = ANY($1)")
+        .bind(vec![a, b, c])
+        .execute(&pool)
+        .await;
+    let _ = sqlx::query("DELETE FROM scripts WHERE id = $1")
+        .bind(handler.0)
+        .execute(&pool)
+        .await;
+    let _ = sqlx::query("DELETE FROM groups WHERE id = ANY($1)")
+        .bind(vec![g.0, g2.0])
+        .execute(&pool)
+        .await;
+    let _ = sqlx::query("DELETE FROM admin_users WHERE id = $1")
+        .bind(admin)
+        .execute(&pool)
+        .await;
+}