PiCloud/crates/manager-core/src/dispatcher.rs

//! The triggers-framework dispatcher.
//!
//! Single tokio task that polls the outbox, claims due rows
//! (`FOR UPDATE SKIP LOCKED`), and routes each to the executor.
//! Shares the `ExecutionGate` with sync HTTP — they compete for the
//! same permit budget, matching design notes §2.
//!
//! Outcome handling per design notes §3 and §4:
//!   - reply_to.is_some() (sync HTTP): never retry. Deliver to inbox
//!     (or write `abandoned_executions` if the receiver dropped).
//!   - is_dead_letter_handler == true: never retry, never DL. Failure
//!     just annotates the original DL row with `resolution =
//!     'handler_failed'` and bumps a metric.
//!   - Otherwise on failure: if `attempt_count + 1 < max_attempts`,
//!     reschedule with backoff + jitter. Else, write a `dead_letters`
//!     row and delete from outbox.
//!
//! Depth-limit: `trigger_depth > max_trigger_depth` skips execution
//! entirely (log + metric) and deletes the row — does NOT dead-letter
//! (design notes §4: depth-exceeded means "you built a loop", and
//! dead-lettering would just re-fire the same loop).

use std::sync::Arc;
use std::time::Duration;

use chrono::{DateTime, Utc};
use picloud_executor_core::{ExecError, ExecRequest, ExecResponse, InvocationType};
use picloud_orchestrator_core::{ExecutionGate, ExecutorClient};
use picloud_shared::{
    DeadLetterId, ExecResponseSummary, ExecutionId, HttpDispatchPayload, InboxDeliveryOutcome,
    InboxFailureKind, InboxResolver, InboxResult, RequestId, ScriptId, ScriptSandbox, TriggerEvent,
};
use rand::Rng;
use uuid::Uuid;

use crate::abandoned_repo::{AbandonedRepo, NewAbandonedExecution};
use crate::dead_letter_repo::{DeadLetterRepo, NewDeadLetter};
use crate::outbox_repo::{NewOutboxRow, OutboxRepo, OutboxRow, OutboxSourceKind};
use crate::principal_resolver::PrincipalResolver;
use crate::queue_repo::{ClaimedMessage, QueueRepo};
use crate::repo::ScriptRepository;
use crate::trigger_config::{BackoffShape, TriggerConfig};
use crate::trigger_repo::{ActiveQueueConsumer, TriggerKind, TriggerRepo};

/// Bundle the dispatcher reads from. Each handle is `Arc<dyn …>` so
/// tests can substitute in-memory backings.
pub struct Dispatcher {
    pub outbox: Arc<dyn OutboxRepo>,
    pub triggers: Arc<dyn TriggerRepo>,
    pub scripts: Arc<dyn ScriptRepository>,
    pub dead_letters: Arc<dyn DeadLetterRepo>,
    pub abandoned: Arc<dyn AbandonedRepo>,
    pub principals: Arc<dyn PrincipalResolver>,
    pub executor: Arc<dyn ExecutorClient>,
    pub gate: Arc<ExecutionGate>,
    pub inbox: Arc<dyn InboxResolver>,
    /// v1.1.9. Reads `queue_messages` for the queue arm + the reclaim
    /// task. None in tests / harnesses that don't exercise queues.
    pub queue: Arc<dyn QueueRepo>,
    pub config: TriggerConfig,
    /// Stable id for this dispatcher instance — written into
    /// `outbox.claimed_by` for forensics. In MVP this is the host's
    /// pid; cluster mode (v1.3+) uses node identity.
    pub instance_id: String,
}

/// How many outbox rows the dispatcher tries to claim per tick.
/// Bounded to keep the working set small even if there's a flood.
const CLAIM_BATCH: i64 = 8;

/// Polling cadence. Short enough that fan-out feels instant; long
/// enough that an idle dispatcher doesn't burn cycles.
const TICK_INTERVAL: Duration = Duration::from_millis(100);

/// Hard cap on the wall-clock budget passed to the executor for an
/// async-dispatched script. Sync HTTP gets a per-script timeout via
/// the orchestrator path; async rows don't have one, so we apply a
/// platform-wide ceiling here. Matches `LocalExecutorClient`'s own
/// 5-minute cap.
const ASYNC_EXEC_TIMEOUT: Duration = Duration::from_secs(300);

impl Dispatcher {
    /// Spawn the dispatcher loop as a detached `tokio::task`. Also
    /// spawns the v1.1.9 queue visibility-timeout reclaim task. Both
    /// run for the process lifetime; returned `JoinHandle`s are
    /// dropped on purpose.
    pub fn spawn(self) {
        // Reclaim task: independent cadence (default 30s) so it doesn't
        // contend with the per-100ms dispatcher tick.
        let reclaim_queue = self.queue.clone();
        let reclaim_interval =
            Duration::from_millis(u64::from(self.config.queue_reclaim_interval_ms));
        tokio::spawn(async move {
            let mut ticker = tokio::time::interval(reclaim_interval);
            ticker.tick().await;
            loop {
                ticker.tick().await;
                match reclaim_queue.reclaim_visibility_timeouts().await {
                    Ok(0) => {}
                    Ok(n) => tracing::info!(reclaimed = n, "queue visibility-timeout reclaim"),
                    Err(e) => tracing::warn!(?e, "queue reclaim task errored"),
                }
            }
        });

        tokio::spawn(async move {
            self.run().await;
        });
    }

    async fn run(self) {
        let mut ticker = tokio::time::interval(TICK_INTERVAL);
        // Skip the immediate first fire so we don't race startup.
        ticker.tick().await;
        loop {
            ticker.tick().await;
            if let Err(err) = self.tick().await {
                tracing::warn!(?err, "dispatcher tick errored");
            }
        }
    }

    async fn tick(&self) -> Result<(), DispatcherError> {
        // 1) Outbox arm — KV/docs/pubsub/cron/email/files/dead-letter/http/invoke.
        let rows = self
            .outbox
            .claim_due(&self.instance_id, CLAIM_BATCH)
            .await
            .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
        for row in rows {
            // Process serially within a tick — the outer ticker is the
            // pacing mechanism. Concurrent dispatchers are a cluster-
            // mode concern; v1.1.1 MVP has one.
            if let Err(err) = self.dispatch_one(row).await {
                tracing::warn!(?err, "dispatch one errored");
            }
        }

        // 2) Queue arm (v1.1.9). One claim per active consumer per tick;
        // bounded by the number of registered consumers so a flood in
        // one queue doesn't starve the rest.
        if let Err(err) = self.tick_queue_arm().await {
            tracing::warn!(?err, "queue arm tick errored");
        }
        Ok(())
    }

    /// v1.1.9. Per tick: list every `queue:receive` consumer and
    /// attempt one claim per `(app_id, queue_name)`. Each successful
    /// claim runs through the same executor path as outbox-backed
    /// trigger dispatches, so retry / dead-letter wiring is shared.
    async fn tick_queue_arm(&self) -> Result<(), DispatcherError> {
        let consumers = self
            .triggers
            .list_active_queue_consumers()
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
        for consumer in consumers {
            if let Err(err) = self.dispatch_one_queue(&consumer).await {
                tracing::warn!(?err, queue = %consumer.queue_name, "queue dispatch errored");
            }
        }
        Ok(())
    }

    async fn dispatch_one_queue(
        &self,
        consumer: &ActiveQueueConsumer,
    ) -> Result<(), DispatcherError> {
        // Atomic claim — None → nothing pending right now for this queue.
        let Some(claimed) = self
            .queue
            .claim(consumer.app_id, &consumer.queue_name)
            .await
            .map_err(|e| DispatcherError::Outbox(e.to_string()))?
        else {
            return Ok(());
        };

        // Gate admission. If saturated, release the claim by nack-
        // immediate (which lets the next tick re-claim). Mirrors the
        // outbox arm.
        let Ok(permit) = self.gate.try_acquire() else {
            let _ = self
                .queue
                .nack(claimed.id, claimed.claim_token, chrono::Duration::milliseconds(100))
                .await;
            return Ok(());
        };

        // Build the TriggerEvent and ExecRequest. Mirrors build_exec_request
        // but uses the consumer + claimed-message data instead of an
        // outbox row.
        let event = TriggerEvent::Queue {
            queue_name: claimed.queue_name.clone(),
            message: claimed.payload.clone(),
            enqueued_at: claimed.enqueued_at,
            attempt: claimed.attempt,
            message_id: claimed.id.to_string(),
        };

        let script = match self.scripts.get(consumer.script_id).await {
            Ok(Some(s)) => s,
            Ok(None) => {
                tracing::warn!(script_id = %consumer.script_id, "queue trigger script missing; dead-lettering");
                let _ = self
                    .queue
                    .dead_letter(
                        claimed.id,
                        claimed.claim_token,
                        claimed.app_id,
                        &claimed.queue_name,
                        Some(consumer.trigger_id),
                        Some(consumer.script_id),
                        claimed.attempt,
                        claimed.enqueued_at,
                        "queue trigger script not found",
                    )
                    .await;
                drop(permit);
                return Ok(());
            }
            Err(e) => return Err(DispatcherError::ResolveTrigger(e.to_string())),
        };

        let principal = self
            .principals
            .resolve(consumer.registered_by_principal)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;

        let execution_id = ExecutionId::new();
        let exec_req = ExecRequest {
            execution_id,
            request_id: RequestId::new(),
            script_id: consumer.script_id,
            script_name: script.name.clone(),
            invocation_type: InvocationType::Function,
            path: "/trigger/queue".into(),
            headers: std::collections::BTreeMap::new(),
            body: serde_json::Value::Null,
            params: std::collections::BTreeMap::new(),
            query: std::collections::BTreeMap::new(),
            rest: String::new(),
            sandbox_overrides: script.sandbox,
            app_id: claimed.app_id,
            principal: Some(principal),
            trigger_depth: 0,
            root_execution_id: execution_id,
            is_dead_letter_handler: false,
            event: Some(event),
        };
        let identity = picloud_orchestrator_core::ScriptIdentity {
            script_id: consumer.script_id,
            updated_at: script.updated_at,
        };
        let outcome = self
            .executor
            .execute_with_identity(identity, &script.source, exec_req, ASYNC_EXEC_TIMEOUT)
            .await;
        drop(permit);

        // Best-effort touch on last_fired_at (a failure here doesn't
        // change ack/nack behavior).
        if let Err(e) = self
            .triggers
            .touch_queue_trigger_last_fired_at(consumer.trigger_id, Utc::now())
            .await
        {
            tracing::debug!(?e, "touch last_fired_at failed");
        }

        match outcome {
            Ok(_) => {
                // Auto-ack on success.
                if let Err(e) = self.queue.ack(claimed.id, claimed.claim_token).await {
                    tracing::warn!(?e, "queue ack failed");
                }
            }
            Err(err) => {
                self.handle_queue_failure(consumer, &claimed, err).await;
            }
        }
        Ok(())
    }

    async fn handle_queue_failure(
        &self,
        consumer: &ActiveQueueConsumer,
        claimed: &ClaimedMessage,
        err: ExecError,
    ) {
        if claimed.attempt < claimed.max_attempts {
            // Retry: clear claim + set deliver_after.
            let delay_ms = compute_backoff(
                claimed.attempt,
                consumer.retry_backoff,
                consumer.retry_base_ms,
                self.config.retry_jitter_pct,
            );
            let delay = chrono::Duration::milliseconds(i64::from(delay_ms));
            if let Err(e) = self.queue.nack(claimed.id, claimed.claim_token, delay).await {
                tracing::warn!(?e, "queue nack failed");
            }
            return;
        }
        // Exhausted. Dead-letter inline (the helper deletes the queue row +
        // writes a dead_letters row in one transaction). The existing
        // fan_out_dead_letter path on the dispatcher's outbox arm fires
        // registered dead_letter handlers off the new row.
        if let Err(e) = self
            .queue
            .dead_letter(
                claimed.id,
                claimed.claim_token,
                claimed.app_id,
                &claimed.queue_name,
                Some(consumer.trigger_id),
                Some(consumer.script_id),
                claimed.attempt,
                claimed.enqueued_at,
                &err.to_string(),
            )
            .await
        {
            tracing::error!(?e, "queue dead-letter write failed");
        }
        // TODO(metrics): bump picloud_queue_dead_letters_total{app_id, queue_name}.
    }

    async fn dispatch_one(&self, row: OutboxRow) -> Result<(), DispatcherError> {
        // Depth-limit check — design notes §4: loops aren't DL'd.
        if row.trigger_depth > self.config.max_trigger_depth {
            tracing::warn!(
                outbox_id = %row.id,
                app_id = %row.app_id,
                trigger_depth = row.trigger_depth,
                "trigger depth exceeded; dropping row"
            );
            // TODO(metrics): bump `picloud_trigger_depth_exceeded{app_id,trigger_id}`.
            self.outbox
                .delete(row.id)
                .await
                .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
            return Ok(());
        }

        // Gate admission — non-blocking. If the gate is saturated,
        // release the claim by rescheduling so another tick can pick
        // it up. The row stays "due" essentially immediately.
        let Ok(permit) = self.gate.try_acquire() else {
            let next = Utc::now() + chrono::Duration::milliseconds(100);
            self.outbox
                .reschedule(row.id, row.attempt_count, next)
                .await
                .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
            return Ok(());
        };

        // Resolve the trigger config (KV / DL) or pull the HTTP
        // payload directly off the outbox row.
        let (resolved, exec_req) = match row.source_kind {
            OutboxSourceKind::Http => match self.build_http_request(&row).await {
                Ok(pair) => pair,
                Err(err) => {
                    tracing::warn!(outbox_id = %row.id, ?err, "http exec build failed; dropping");
                    self.outbox
                        .delete(row.id)
                        .await
                        .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
                    drop(permit);
                    return Ok(());
                }
            },
            OutboxSourceKind::Invoke => match self.build_invoke_request(&row).await {
                Ok(pair) => pair,
                Err(err) => {
                    tracing::warn!(outbox_id = %row.id, ?err, "invoke exec build failed; dropping");
                    self.outbox
                        .delete(row.id)
                        .await
                        .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
                    drop(permit);
                    return Ok(());
                }
            },
            OutboxSourceKind::Kv
            | OutboxSourceKind::Docs
            | OutboxSourceKind::DeadLetter
            | OutboxSourceKind::Cron
            | OutboxSourceKind::Files
            | OutboxSourceKind::Pubsub
            | OutboxSourceKind::Email => {
                let resolved = self.resolve_trigger(&row).await?;
                let req = match self.build_exec_request(&row, &resolved).await {
                    Ok(req) => req,
                    Err(err) => {
                        tracing::warn!(outbox_id = %row.id, ?err, "exec request build failed; dropping row");
                        self.outbox
                            .delete(row.id)
                            .await
                            .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
                        drop(permit);
                        return Ok(());
                    }
                };
                (resolved, req)
            }
        };

        // The gate permit auto-releases when this scope ends or when
        // the executor finishes. We hand control to the executor and
        // wait synchronously here — sync HTTP and dispatcher share the
        // semaphore so this is intentional.
        let source = resolved.script_source.clone();
        let identity = picloud_orchestrator_core::ScriptIdentity {
            script_id: resolved.script_id,
            updated_at: resolved.script_updated_at,
        };
        let outcome = self
            .executor
            .execute_with_identity(identity, &source, exec_req, ASYNC_EXEC_TIMEOUT)
            .await;
        drop(permit);

        match outcome {
            Ok(resp) => self.handle_success(&row, &resolved, resp).await,
            Err(err) => self.handle_failure(&row, &resolved, err).await,
        }
    }

    async fn resolve_trigger(&self, row: &OutboxRow) -> Result<ResolvedTrigger, DispatcherError> {
        // For KV and DL kinds, the outbox carries `trigger_id`. Use it
        // to look up the trigger row, then resolve the script.
        let Some(trigger_id) = row.trigger_id else {
            return Err(DispatcherError::ResolveTrigger(
                "outbox row missing trigger_id".into(),
            ));
        };
        let trigger = self
            .triggers
            .get(trigger_id)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
            .ok_or_else(|| {
                DispatcherError::ResolveTrigger(format!("trigger {trigger_id} not found"))
            })?;

        let script = self
            .scripts
            .get(trigger.script_id)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
            .ok_or_else(|| {
                DispatcherError::ResolveTrigger(format!("script {} not found", trigger.script_id))
            })?;

        Ok(ResolvedTrigger {
            trigger_kind: trigger.kind,
            is_dead_letter_handler: matches!(trigger.kind, TriggerKind::DeadLetter),
            script_id: script.id,
            script_source: script.source,
            script_name: script.name,
            script_updated_at: script.updated_at,
            sandbox_overrides: script.sandbox,
            registered_by_principal: trigger.registered_by_principal,
            retry_max_attempts: trigger.retry_max_attempts,
            retry_backoff: trigger.retry_backoff,
            retry_base_ms: trigger.retry_base_ms,
        })
    }

    async fn build_exec_request(
        &self,
        row: &OutboxRow,
        resolved: &ResolvedTrigger,
    ) -> Result<ExecRequest, DispatcherError> {
        let trigger_event: TriggerEvent = serde_json::from_value(row.payload.clone())
            .map_err(|e| DispatcherError::ResolveTrigger(format!("decode payload: {e}")))?;

        let principal = self
            .principals
            .resolve(resolved.registered_by_principal)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;

        let execution_id = ExecutionId::new();
        Ok(ExecRequest {
            execution_id,
            request_id: RequestId::new(),
            script_id: resolved.script_id,
            script_name: resolved.script_name.clone(),
            invocation_type: InvocationType::Function,
            path: format!("/trigger/{}", trigger_event.source()),
            headers: std::collections::BTreeMap::new(),
            body: serde_json::Value::Null,
            params: std::collections::BTreeMap::new(),
            query: std::collections::BTreeMap::new(),
            rest: String::new(),
            sandbox_overrides: resolved.sandbox_overrides,
            app_id: row.app_id,
            principal: Some(principal),
            trigger_depth: row.trigger_depth,
            root_execution_id: row.root_execution_id.unwrap_or(execution_id),
            is_dead_letter_handler: resolved.is_dead_letter_handler,
            event: Some(trigger_event),
        })
    }

    /// Build an `(ResolvedTrigger, ExecRequest)` for an HTTP outbox
    /// row. HTTP rows don't have a backing `triggers` row (the
    /// `trigger_id` references `routes.id` instead). We pull the
    /// script id off the outbox row, the request shape off the
    /// payload, and synthesize a `ResolvedTrigger` with retry
    /// settings irrelevant for HTTP (sync HTTP is never retried;
    /// async HTTP uses default policy from `TriggerConfig`).
    async fn build_http_request(
        &self,
        row: &OutboxRow,
    ) -> Result<(ResolvedTrigger, ExecRequest), DispatcherError> {
        let Some(script_id) = row.script_id else {
            return Err(DispatcherError::ResolveTrigger(
                "HTTP outbox row missing script_id".into(),
            ));
        };
        let script = self
            .scripts
            .get(script_id)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
            .ok_or_else(|| {
                DispatcherError::ResolveTrigger(format!("script {script_id} not found"))
            })?;

        let payload: HttpDispatchPayload = serde_json::from_value(row.payload.clone())
            .map_err(|e| DispatcherError::ResolveTrigger(format!("decode http payload: {e}")))?;

        let execution_id = ExecutionId::new();
        let req = ExecRequest {
            execution_id,
            request_id: RequestId::new(),
            script_id,
            script_name: payload.script_name.clone(),
            invocation_type: InvocationType::Http,
            path: payload.path.clone(),
            headers: payload.headers,
            body: payload.body,
            params: payload.params,
            query: payload.query,
            rest: payload.rest,
            sandbox_overrides: script.sandbox,
            app_id: row.app_id,
            // HTTP outbox rows don't run as the trigger registrant —
            // they run with no principal (public ingress) or the
            // attached one (origin_principal forensic field is not
            // promoted to execution principal in this MVP).
            principal: None,
            trigger_depth: row.trigger_depth,
            root_execution_id: row.root_execution_id.unwrap_or(execution_id),
            is_dead_letter_handler: false,
            event: None,
        };

        let resolved = ResolvedTrigger {
            trigger_kind: TriggerKind::Kv, // placeholder; HTTP doesn't have a kind
            is_dead_letter_handler: false,
            script_id,
            script_source: script.source,
            script_name: payload.script_name,
            script_updated_at: script.updated_at,
            sandbox_overrides: script.sandbox,
            // HTTP outbox rows don't carry a registered_by_principal
            // — use a sentinel zero UUID since this field isn't used
            // downstream for HTTP (no retries, no inbox principal).
            registered_by_principal: picloud_shared::AdminUserId::from(uuid::Uuid::nil()),
            // Async HTTP uses the platform default retry policy from
            // TriggerConfig. Sync HTTP (reply_to.is_some) never retries
            // regardless.
            retry_max_attempts: self.config.retry_max_attempts,
            retry_backoff: self.config.retry_backoff,
            retry_base_ms: self.config.retry_base_ms,
        };
        Ok((resolved, req))
    }

    /// v1.1.9. Build an `(ResolvedTrigger, ExecRequest)` from an
    /// invoke_async outbox row. The payload was written by
    /// `InvokeService::enqueue_async`. No retry policy — invoke_async
    /// runs once; on failure the row is deleted and a dead_letters row
    /// is written (so the misfire is observable).
    async fn build_invoke_request(
        &self,
        row: &OutboxRow,
    ) -> Result<(ResolvedTrigger, ExecRequest), DispatcherError> {
        // Extract the args + script_id from the payload (InvokeService
        // populates these). Defensive: dead_letter the row if any
        // required field is missing.
        let payload = &row.payload;
        let Some(script_id_str) = payload.get("script_id").and_then(|v| v.as_str()) else {
            return Err(DispatcherError::ResolveTrigger(
                "invoke row missing script_id".into(),
            ));
        };
        let script_id = picloud_shared::ScriptId::from(
            uuid::Uuid::parse_str(script_id_str)
                .map_err(|e| DispatcherError::ResolveTrigger(format!("script_id: {e}")))?,
        );
        let script = self
            .scripts
            .get(script_id)
            .await
            .map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
            .ok_or_else(|| {
                DispatcherError::ResolveTrigger(format!("script {script_id} not found"))
            })?;
        // Same-app guard — the script could have been re-assigned or
        // the row could have been hand-edited. Reject mismatches.
        if script.app_id != row.app_id {
            return Err(DispatcherError::ResolveTrigger(
                "invoke target belongs to a different app".into(),
            ));
        }

        let args = payload.get("args").cloned().unwrap_or(serde_json::Value::Null);
        let trigger_depth = payload
            .get("trigger_depth")
            .and_then(serde_json::Value::as_u64)
            .map(|n| u32::try_from(n).unwrap_or(u32::MAX))
            .unwrap_or(row.trigger_depth);

        let execution_id = ExecutionId::new();
        let req = ExecRequest {
            execution_id,
            request_id: RequestId::new(),
            script_id: script.id,
            script_name: script.name.clone(),
            invocation_type: InvocationType::Function,
            path: "/invoke_async".into(),
            headers: std::collections::BTreeMap::new(),
            body: args,
            params: std::collections::BTreeMap::new(),
            query: std::collections::BTreeMap::new(),
            rest: String::new(),
            sandbox_overrides: script.sandbox,
            app_id: row.app_id,
            // invoke_async runs with no principal — same as HTTP outbox.
            principal: None,
            trigger_depth,
            root_execution_id: row.root_execution_id.unwrap_or(execution_id),
            is_dead_letter_handler: false,
            event: None,
        };

        // Build a synthetic ResolvedTrigger so the dispatch_one outer
        // loop can carry on through the same code path. Retry knobs
        // bound to 1 attempt: invoke_async runs ONCE (callers wrap in
        // retry::with for retries).
        let resolved = ResolvedTrigger {
            trigger_kind: TriggerKind::Cron, // placeholder; not used downstream
            is_dead_letter_handler: false,
            script_id: script.id,
            script_source: script.source,
            script_name: script.name,
            script_updated_at: script.updated_at,
            sandbox_overrides: script.sandbox,
            registered_by_principal: picloud_shared::AdminUserId::from(uuid::Uuid::nil()),
            retry_max_attempts: 1,
            retry_backoff: BackoffShape::Constant,
            retry_base_ms: 0,
        };
        Ok((resolved, req))
    }

    async fn handle_success(
        &self,
        row: &OutboxRow,
        _resolved: &ResolvedTrigger,
        resp: ExecResponse,
    ) -> Result<(), DispatcherError> {
        if let Some(inbox_id) = row.reply_to {
            self.deliver_inbox(row, inbox_id, InboxResult::Success(summarize(&resp)))
                .await;
        }
        self.outbox
            .delete(row.id)
            .await
            .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
        Ok(())
    }

    async fn handle_failure(
        &self,
        row: &OutboxRow,
        resolved: &ResolvedTrigger,
        err: ExecError,
    ) -> Result<(), DispatcherError> {
        // Sync HTTP: always single-attempt. Always deliver outcome
        // (success-or-failure) to the inbox. Never retry, never DL.
        if let Some(inbox_id) = row.reply_to {
            let (kind, message) = classify_exec_error(&err);
            self.deliver_inbox(
                row,
                inbox_id,
                InboxResult::Failure {
                    kind,
                    message: message.clone(),
                },
            )
            .await;
            self.outbox
                .delete(row.id)
                .await
                .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
            return Ok(());
        }

        // Dead-letter handler: never retry, never DL. Failure
        // annotates the original DL row + bumps a metric.
        if resolved.is_dead_letter_handler {
            tracing::error!(
                outbox_id = %row.id,
                app_id = %row.app_id,
                ?err,
                "dead-letter handler failed; not retrying"
            );
            // TODO(metrics): bump `picloud_dead_letter_handler_failures{app_id}`.
            // Annotate the original DL row (id is `row.payload.dead_letter.id`
            // when the payload is a DeadLetter TriggerEvent). Best-effort:
            // if the payload doesn't decode, just log and move on.
            if let Ok(TriggerEvent::DeadLetter { dead_letter_id, .. }) =
                serde_json::from_value::<TriggerEvent>(row.payload.clone())
            {
                if let Err(e) = self
                    .dead_letters
                    .resolve(dead_letter_id, "handler_failed")
                    .await
                {
                    tracing::warn!(?e, "could not annotate DL row as handler_failed");
                }
            }
            self.outbox
                .delete(row.id)
                .await
                .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
            return Ok(());
        }

        // Async event: retry per policy, then dead-letter.
        let attempt = row.attempt_count + 1;
        if attempt < resolved.retry_max_attempts {
            let delay = compute_backoff(
                attempt,
                resolved.retry_backoff,
                resolved.retry_base_ms,
                self.config.retry_jitter_pct,
            );
            let next = Utc::now() + chrono::Duration::milliseconds(i64::from(delay));
            tracing::info!(
                outbox_id = %row.id,
                attempt,
                max_attempts = resolved.retry_max_attempts,
                retry_in_ms = delay,
                "rescheduling outbox row"
            );
            self.outbox
                .reschedule(row.id, attempt, next)
                .await
                .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
            return Ok(());
        }

        // Exhausted retries → dead-letter.
        let (op, source) = describe_event(&row.payload);
        let now = Utc::now();
        let dl_id = match self
            .dead_letters
            .insert(NewDeadLetter {
                app_id: row.app_id,
                original_event_id: row.id,
                source: source.clone(),
                op,
                trigger_id: row.trigger_id,
                script_id: Some(resolved.script_id),
                payload: row.payload.clone(),
                attempt_count: attempt,
                first_attempt_at: row.created_at,
                last_attempt_at: now,
                last_error: err.to_string(),
            })
            .await
        {
            Ok(id) => Some(id),
            Err(e) => {
                tracing::error!(?e, "failed to write dead-letter row");
                None
            }
        };

        // v1.1.7 fix: fan the dead-letter out to matching handler triggers.
        // This was missing since v1.1.1 — the row was written but
        // `list_matching_dead_letter` had no production caller, so
        // registered dead_letter handlers never fired. The recursion-stop
        // (a dead-letter handler's own failure is not re-dead-lettered)
        // is upheld by the `is_dead_letter_handler` short-circuit at the
        // top of this function, so this fan-out is only reached for
        // non-handler executions.
        if let Some(dl_id) = dl_id {
            self.fan_out_dead_letter(row, resolved, dl_id, &source, attempt, &err, now)
                .await;
        }

        self.outbox
            .delete(row.id)
            .await
            .map_err(|e| DispatcherError::Outbox(e.to_string()))?;
        Ok(())
    }

    /// Enqueue one outbox row per matching `dead_letter` trigger so its
    /// handler script runs with the dead-letter event as `ctx.event`.
    /// Best-effort: a lookup/insert failure is logged, not propagated
    /// (the dead-letter row itself is already durably written).
    #[allow(clippy::too_many_arguments)]
    async fn fan_out_dead_letter(
        &self,
        row: &OutboxRow,
        resolved: &ResolvedTrigger,
        dead_letter_id: DeadLetterId,
        source: &str,
        attempt: u32,
        err: &ExecError,
        now: DateTime<Utc>,
    ) {
        // The DL event nests the original verbatim; if the payload can't
        // be decoded back into a TriggerEvent we can't build the nested
        // `original`, so skip the fan-out (the DL row is still written).
        let Ok(original) = serde_json::from_value::<TriggerEvent>(row.payload.clone()) else {
            tracing::warn!(
                outbox_id = %row.id,
                "dead-letter payload is not a TriggerEvent; skipping handler fan-out"
            );
            return;
        };

        let matches = match self
            .triggers
            .list_matching_dead_letter(row.app_id, source, row.trigger_id, Some(resolved.script_id))
            .await
        {
            Ok(m) => m,
            Err(e) => {
                tracing::error!(?e, "dead-letter trigger lookup failed");
                return;
            }
        };

        for m in matches {
            let event = TriggerEvent::DeadLetter {
                dead_letter_id,
                original: Box::new(original.clone()),
                attempts: attempt,
                last_error: err.to_string(),
                trigger_id: row.trigger_id,
                script_id: Some(resolved.script_id),
                first_attempt_at: row.created_at,
                last_attempt_at: now,
            };
            let payload = match serde_json::to_value(&event) {
                Ok(p) => p,
                Err(e) => {
                    tracing::error!(?e, "failed to serialize dead-letter event");
                    continue;
                }
            };
            if let Err(e) = self
                .outbox
                .insert(NewOutboxRow {
                    app_id: row.app_id,
                    source_kind: OutboxSourceKind::DeadLetter,
                    trigger_id: Some(m.trigger_id),
                    script_id: Some(m.script_id),
                    reply_to: None,
                    payload,
                    origin_principal: Some(m.registered_by_principal),
                    trigger_depth: row.trigger_depth.saturating_add(1),
                    root_execution_id: row.root_execution_id,
                })
                .await
            {
                tracing::error!(?e, "failed to enqueue dead-letter handler delivery");
            }
        }
    }

    async fn deliver_inbox(&self, row: &OutboxRow, inbox_id: Uuid, result: InboxResult) {
        match self.inbox.deliver(inbox_id, result.clone()).await {
            InboxDeliveryOutcome::Delivered => {}
            InboxDeliveryOutcome::Abandoned => {
                // Receiver was dropped — record forensic row + bump
                // metric.
                let (status_code, summary) = match &result {
                    InboxResult::Success(s) => (s.status_code, None),
                    InboxResult::Failure { kind, message } => {
                        (failure_kind_to_status(*kind), Some(message.clone()))
                    }
                };
                if let Err(e) = self
                    .abandoned
                    .insert(NewAbandonedExecution {
                        app_id: row.app_id,
                        outbox_id: row.id,
                        script_id: row.script_id,
                        inbox_id,
                        status_code,
                        result_summary: summary,
                    })
                    .await
                {
                    tracing::warn!(?e, "abandoned_executions insert failed");
                }
                // TODO(metrics): bump `picloud_abandoned_executions_total{app_id}`.
            }
        }
    }
}

#[derive(Debug)]
pub struct ResolvedTrigger {
    pub trigger_kind: TriggerKind,
    pub is_dead_letter_handler: bool,
    pub script_id: ScriptId,
    pub script_source: String,
    pub script_name: String,
    /// v1.1.3: freshness comparator for the orchestrator's top-level
    /// script cache. The dispatcher hands `(script_id, updated_at)`
    /// in alongside the source so cached ASTs can be reused across
    /// triggered invocations.
    pub script_updated_at: chrono::DateTime<chrono::Utc>,
    pub sandbox_overrides: ScriptSandbox,
    pub registered_by_principal: picloud_shared::AdminUserId,
    pub retry_max_attempts: u32,
    pub retry_backoff: BackoffShape,
    pub retry_base_ms: u32,
}

#[derive(Debug, thiserror::Error)]
pub enum DispatcherError {
    #[error("outbox: {0}")]
    Outbox(String),
    #[error("resolve trigger: {0}")]
    ResolveTrigger(String),
}

fn summarize(resp: &ExecResponse) -> ExecResponseSummary {
    ExecResponseSummary {
        status_code: resp.status_code,
        headers: resp.headers.clone(),
        body: resp.body.clone(),
    }
}

/// Map `ExecError` onto the design-notes §3 status-code table.
fn classify_exec_error(err: &ExecError) -> (InboxFailureKind, String) {
    match err {
        ExecError::Parse(s) | ExecError::InvalidResponse(s) => {
            (InboxFailureKind::Validation, s.clone())
        }
        ExecError::Timeout(_) => (InboxFailureKind::Timeout, err.to_string()),
        ExecError::OperationBudgetExceeded => (InboxFailureKind::OperationBudget, err.to_string()),
        ExecError::Overloaded { .. } => (InboxFailureKind::Overloaded, err.to_string()),
        ExecError::Runtime(s) => (InboxFailureKind::Runtime, s.clone()),
    }
}

fn failure_kind_to_status(k: InboxFailureKind) -> u16 {
    match k {
        InboxFailureKind::Validation => 422,
        InboxFailureKind::Runtime => 502,
        InboxFailureKind::Overloaded => 503,
        InboxFailureKind::Timeout => 504,
        InboxFailureKind::OperationBudget => 507,
        InboxFailureKind::Platform => 500,
    }
}

/// `(op, source)` extracted from the outbox payload. Used to seed the
/// `dead_letters` row when retries exhaust.
fn describe_event(payload: &serde_json::Value) -> (String, String) {
    let source = payload
        .get("source")
        .and_then(|v| v.as_str())
        .unwrap_or("")
        .to_string();
    let op = payload
        .get("op")
        .and_then(|v| v.as_str())
        .unwrap_or("")
        .to_string();
    (op, source)
}

/// Compute backoff (ms) for the given attempt + policy + jitter.
/// Attempt is 1-indexed (first retry = attempt 1).
#[must_use]
pub fn compute_backoff(attempt: u32, backoff: BackoffShape, base_ms: u32, jitter_pct: u32) -> u32 {
    let base_ms = u64::from(base_ms);
    let attempt = u64::from(attempt.saturating_sub(1));
    let raw = match backoff {
        BackoffShape::Constant => base_ms,
        BackoffShape::Linear => base_ms * (attempt + 1),
        // 1x base, 2x base, 4x base, … (saturating).
        BackoffShape::Exponential => base_ms.saturating_mul(1u64 << attempt.min(20)),
    };
    let raw = u32::try_from(raw.min(u64::from(u32::MAX))).unwrap_or(u32::MAX);
    apply_jitter(raw, jitter_pct)
}

fn apply_jitter(raw: u32, pct: u32) -> u32 {
    if pct == 0 {
        return raw;
    }
    let pct = pct.min(100);
    // ±span% — bounded by raw itself so we can't underflow when
    // raw + offset goes below zero.
    let span = u64::from(raw) * u64::from(pct) / 100;
    if span == 0 {
        return raw;
    }
    let span_i64 = i64::try_from(span).unwrap_or(i64::MAX);
    let mut rng = rand::thread_rng();
    let offset = rng.gen_range(-span_i64..=span_i64);
    let signed = i64::from(raw).saturating_add(offset).max(0);
    u32::try_from(signed.min(i64::from(u32::MAX))).unwrap_or(u32::MAX)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn exponential_backoff_doubles_per_attempt() {
        // No jitter (pct=0) for a deterministic check.
        assert_eq!(compute_backoff(1, BackoffShape::Exponential, 1000, 0), 1000);
        assert_eq!(compute_backoff(2, BackoffShape::Exponential, 1000, 0), 2000);
        assert_eq!(compute_backoff(3, BackoffShape::Exponential, 1000, 0), 4000);
        assert_eq!(compute_backoff(4, BackoffShape::Exponential, 1000, 0), 8000);
    }

    #[test]
    fn linear_backoff_scales_with_attempt() {
        assert_eq!(compute_backoff(1, BackoffShape::Linear, 100, 0), 100);
        assert_eq!(compute_backoff(2, BackoffShape::Linear, 100, 0), 200);
        assert_eq!(compute_backoff(5, BackoffShape::Linear, 100, 0), 500);
    }

    #[test]
    fn constant_backoff_returns_base() {
        for attempt in 1..=5 {
            assert_eq!(
                compute_backoff(attempt, BackoffShape::Constant, 750, 0),
                750
            );
        }
    }

    #[test]
    fn jitter_within_pct_of_base() {
        for _ in 0..100 {
            let v = compute_backoff(1, BackoffShape::Constant, 1000, 20);
            // ±20% of 1000 = 800..=1200.
            assert!((800..=1200).contains(&v), "jitter out of range: {v}");
        }
    }

    #[test]
    fn classify_exec_error_covers_every_variant() {
        let parse = classify_exec_error(&ExecError::Parse("nope".into()));
        assert!(matches!(parse.0, InboxFailureKind::Validation));
        let invalid = classify_exec_error(&ExecError::InvalidResponse("bad".into()));
        assert!(matches!(invalid.0, InboxFailureKind::Validation));
        let timeout = classify_exec_error(&ExecError::Timeout(30));
        assert!(matches!(timeout.0, InboxFailureKind::Timeout));
        let budget = classify_exec_error(&ExecError::OperationBudgetExceeded);
        assert!(matches!(budget.0, InboxFailureKind::OperationBudget));
        let runtime = classify_exec_error(&ExecError::Runtime("threw".into()));
        assert!(matches!(runtime.0, InboxFailureKind::Runtime));
        let overload = classify_exec_error(&ExecError::Overloaded {
            retry_after_secs: 1,
        });
        assert!(matches!(overload.0, InboxFailureKind::Overloaded));
    }

    #[test]
    fn failure_kind_status_codes_match_design_notes() {
        assert_eq!(failure_kind_to_status(InboxFailureKind::Validation), 422);
        assert_eq!(failure_kind_to_status(InboxFailureKind::Runtime), 502);
        assert_eq!(failure_kind_to_status(InboxFailureKind::Overloaded), 503);
        assert_eq!(failure_kind_to_status(InboxFailureKind::Timeout), 504);
        assert_eq!(
            failure_kind_to_status(InboxFailureKind::OperationBudget),
            507
        );
        assert_eq!(failure_kind_to_status(InboxFailureKind::Platform), 500);
    }
}