fix(v1.1.7-dead-letter): wire dispatcher → list_matching_dead_letter

dead_letter triggers have been registerable since v1.1.1 but their
handlers never fired: dispatcher::handle_failure wrote the dead_letters
row and stopped — list_matching_dead_letter had no production caller.
Any deploy v1.1.1–v1.1.6 with dead_letter triggers had silently
non-functional handlers.

The fix: after the dead-letter row is inserted on retry exhaustion, fan
out to matching dead_letter triggers (filtered by source / originating
trigger_id / script_id) and enqueue one outbox row per match carrying a
real-shape TriggerEvent::DeadLetter (the §6 brief field names were stale
— used the actual variant: dead_letter_id, original: Box<TriggerEvent>,
attempts, last_error, trigger_id, script_id, first/last_attempt_at).
The recursion-stop (a handler's own failure isn't re-dead-lettered)
is upheld by the existing is_dead_letter_handler short-circuit.

Tests (DB-gated): handler actually fires with the nested original event;
existing row-create test now also asserts handler-fire; source_filter
excludes non-matching; failing handler does not recurse.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

crates/manager-core/src/dispatcher.rs

@@ -23,19 +23,19 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use chrono::Utc;
+use chrono::{DateTime, Utc};
 use picloud_executor_core::{ExecError, ExecRequest, ExecResponse, InvocationType};
 use picloud_orchestrator_core::{ExecutionGate, ExecutorClient};
 use picloud_shared::{
-    ExecResponseSummary, ExecutionId, HttpDispatchPayload, InboxDeliveryOutcome, InboxFailureKind,
-    InboxResolver, InboxResult, RequestId, ScriptId, ScriptSandbox, TriggerEvent,
+    DeadLetterId, ExecResponseSummary, ExecutionId, HttpDispatchPayload, InboxDeliveryOutcome,
+    InboxFailureKind, InboxResolver, InboxResult, RequestId, ScriptId, ScriptSandbox, TriggerEvent,
 };
 use rand::Rng;
 use uuid::Uuid;
 
 use crate::abandoned_repo::{AbandonedRepo, NewAbandonedExecution};
 use crate::dead_letter_repo::{DeadLetterRepo, NewDeadLetter};
-use crate::outbox_repo::{OutboxRepo, OutboxRow, OutboxSourceKind};
+use crate::outbox_repo::{NewOutboxRow, OutboxRepo, OutboxRow, OutboxSourceKind};
 use crate::principal_resolver::PrincipalResolver;
 use crate::repo::ScriptRepository;
 use crate::trigger_config::{BackoffShape, TriggerConfig};
@@ -463,12 +463,12 @@ impl Dispatcher {
         // Exhausted retries → dead-letter.
         let (op, source) = describe_event(&row.payload);
         let now = Utc::now();
-        if let Err(e) = self
+        let dl_id = match self
             .dead_letters
             .insert(NewDeadLetter {
                 app_id: row.app_id,
                 original_event_id: row.id,
-                source,
+                source: source.clone(),
                 op,
                 trigger_id: row.trigger_id,
                 script_id: Some(resolved.script_id),
@@ -480,8 +480,26 @@ impl Dispatcher {
             })
             .await
         {
-            tracing::error!(?e, "failed to write dead-letter row");
+            Ok(id) => Some(id),
+            Err(e) => {
+                tracing::error!(?e, "failed to write dead-letter row");
+                None
+            }
+        };
+
+        // v1.1.7 fix: fan the dead-letter out to matching handler triggers.
+        // This was missing since v1.1.1 — the row was written but
+        // `list_matching_dead_letter` had no production caller, so
+        // registered dead_letter handlers never fired. The recursion-stop
+        // (a dead-letter handler's own failure is not re-dead-lettered)
+        // is upheld by the `is_dead_letter_handler` short-circuit at the
+        // top of this function, so this fan-out is only reached for
+        // non-handler executions.
+        if let Some(dl_id) = dl_id {
+            self.fan_out_dead_letter(&row, &resolved, dl_id, &source, attempt, &err, now)
+                .await;
         }
+
         self.outbox
             .delete(row.id)
             .await
@@ -489,6 +507,82 @@ impl Dispatcher {
         Ok(())
     }
 
+    /// Enqueue one outbox row per matching `dead_letter` trigger so its
+    /// handler script runs with the dead-letter event as `ctx.event`.
+    /// Best-effort: a lookup/insert failure is logged, not propagated
+    /// (the dead-letter row itself is already durably written).
+    #[allow(clippy::too_many_arguments)]
+    async fn fan_out_dead_letter(
+        &self,
+        row: &OutboxRow,
+        resolved: &ResolvedTrigger,
+        dead_letter_id: DeadLetterId,
+        source: &str,
+        attempt: u32,
+        err: &ExecError,
+        now: DateTime<Utc>,
+    ) {
+        // The DL event nests the original verbatim; if the payload can't
+        // be decoded back into a TriggerEvent we can't build the nested
+        // `original`, so skip the fan-out (the DL row is still written).
+        let Ok(original) = serde_json::from_value::<TriggerEvent>(row.payload.clone()) else {
+            tracing::warn!(
+                outbox_id = %row.id,
+                "dead-letter payload is not a TriggerEvent; skipping handler fan-out"
+            );
+            return;
+        };
+
+        let matches = match self
+            .triggers
+            .list_matching_dead_letter(row.app_id, source, row.trigger_id, Some(resolved.script_id))
+            .await
+        {
+            Ok(m) => m,
+            Err(e) => {
+                tracing::error!(?e, "dead-letter trigger lookup failed");
+                return;
+            }
+        };
+
+        for m in matches {
+            let event = TriggerEvent::DeadLetter {
+                dead_letter_id,
+                original: Box::new(original.clone()),
+                attempts: attempt,
+                last_error: err.to_string(),
+                trigger_id: row.trigger_id,
+                script_id: Some(resolved.script_id),
+                first_attempt_at: row.created_at,
+                last_attempt_at: now,
+            };
+            let payload = match serde_json::to_value(&event) {
+                Ok(p) => p,
+                Err(e) => {
+                    tracing::error!(?e, "failed to serialize dead-letter event");
+                    continue;
+                }
+            };
+            if let Err(e) = self
+                .outbox
+                .insert(NewOutboxRow {
+                    app_id: row.app_id,
+                    source_kind: OutboxSourceKind::DeadLetter,
+                    trigger_id: Some(m.trigger_id),
+                    script_id: Some(m.script_id),
+                    reply_to: None,
+                    payload,
+                    origin_principal: Some(m.registered_by_principal),
+                    trigger_depth: row.trigger_depth.saturating_add(1),
+                    root_execution_id: row.root_execution_id,
+                })
+                .await
+            {
+                tracing::error!(?e, "failed to enqueue dead-letter handler delivery");
+            }
+        }
+    }
+
     async fn deliver_inbox(&self, row: &OutboxRow, inbox_id: Uuid, result: InboxResult) {
         match self.inbox.deliver(inbox_id, result.clone()).await {
             InboxDeliveryOutcome::Delivered => {}