bd64a25c97
build_invoke_request already asserts `script.app_id == row.app_id` (dispatcher.rs:752) before constructing an ExecRequest. The queue dispatcher (`dispatch_one_queue`) and the HTTP outbox arm (`build_http_request`) did not. validate_trigger_target blocks cross- app registration at write time, so the gap was latent; but a hand- edited row, a partial backup restore, or a script re-pointed across apps post-hoc could otherwise execute one app's script under another app's SdkCallCx, breaking the cross-app isolation boundary the SDK relies on. Adds the runtime guard at both sites: * dispatch_one_queue — on mismatch, dead-letter the message so the misfire is observable rather than silently dropped, and short- circuit. * build_http_request — on mismatch, return ResolveTrigger error; the outer dispatch arm logs + drops the row (same path as decode failures). Audit ref: security_audit/02_authz_isolation.md (H-F1). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1403 lines
55 KiB
Rust
1403 lines
55 KiB
Rust
//! The triggers-framework dispatcher.
|
||
//!
|
||
//! Single tokio task that polls the outbox, claims due rows
|
||
//! (`FOR UPDATE SKIP LOCKED`), and routes each to the executor.
|
||
//! Shares the `ExecutionGate` with sync HTTP — they compete for the
|
||
//! same permit budget, matching design notes §2.
|
||
//!
|
||
//! Outcome handling per design notes §3 and §4:
|
||
//! - reply_to.is_some() (sync HTTP): never retry. Deliver to inbox
|
||
//! (or write `abandoned_executions` if the receiver dropped).
|
||
//! - is_dead_letter_handler == true: never retry, never DL. Failure
|
||
//! just annotates the original DL row with `resolution =
|
||
//! 'handler_failed'` and bumps a metric.
|
||
//! - Otherwise on failure: if `attempt_count + 1 < max_attempts`,
|
||
//! reschedule with backoff + jitter. Else, write a `dead_letters`
|
||
//! row and delete from outbox.
|
||
//!
|
||
//! Depth-limit: `trigger_depth > max_trigger_depth` skips execution
|
||
//! entirely (log + metric) and deletes the row — does NOT dead-letter
|
||
//! (design notes §4: depth-exceeded means "you built a loop", and
|
||
//! dead-lettering would just re-fire the same loop).
|
||
|
||
use std::sync::Arc;
|
||
use std::time::Duration;
|
||
|
||
use chrono::{DateTime, Utc};
|
||
use picloud_executor_core::{ExecError, ExecRequest, ExecResponse, InvocationType};
|
||
use picloud_orchestrator_core::{ExecutionGate, ExecutorClient};
|
||
use picloud_shared::{
|
||
DeadLetterId, ExecResponseSummary, ExecutionId, HttpDispatchPayload, InboxDeliveryOutcome,
|
||
InboxFailureKind, InboxResolver, InboxResult, RequestId, ScriptId, ScriptSandbox, TriggerEvent,
|
||
};
|
||
use rand::Rng;
|
||
use uuid::Uuid;
|
||
|
||
use crate::abandoned_repo::{AbandonedRepo, NewAbandonedExecution};
|
||
use crate::dead_letter_repo::{DeadLetterRepo, NewDeadLetter};
|
||
use crate::outbox_repo::{NewOutboxRow, OutboxRepo, OutboxRow, OutboxSourceKind};
|
||
use crate::principal_resolver::PrincipalResolver;
|
||
use crate::queue_repo::{ClaimedMessage, QueueRepo};
|
||
use crate::repo::ScriptRepository;
|
||
use crate::trigger_config::{BackoffShape, TriggerConfig};
|
||
use crate::trigger_repo::{ActiveQueueConsumer, TriggerKind, TriggerRepo};
|
||
|
||
/// Bundle the dispatcher reads from. Each handle is `Arc<dyn …>` so
|
||
/// tests can substitute in-memory backings.
|
||
pub struct Dispatcher {
|
||
pub outbox: Arc<dyn OutboxRepo>,
|
||
pub triggers: Arc<dyn TriggerRepo>,
|
||
pub scripts: Arc<dyn ScriptRepository>,
|
||
pub dead_letters: Arc<dyn DeadLetterRepo>,
|
||
pub abandoned: Arc<dyn AbandonedRepo>,
|
||
pub principals: Arc<dyn PrincipalResolver>,
|
||
pub executor: Arc<dyn ExecutorClient>,
|
||
pub gate: Arc<ExecutionGate>,
|
||
pub inbox: Arc<dyn InboxResolver>,
|
||
/// v1.1.9. Reads `queue_messages` for the queue arm + the reclaim
|
||
/// task. None in tests / harnesses that don't exercise queues.
|
||
pub queue: Arc<dyn QueueRepo>,
|
||
pub config: TriggerConfig,
|
||
/// Stable id for this dispatcher instance — written into
|
||
/// `outbox.claimed_by` for forensics. In MVP this is the host's
|
||
/// pid; cluster mode (v1.3+) uses node identity.
|
||
pub instance_id: String,
|
||
}
|
||
|
||
/// How many outbox rows the dispatcher tries to claim per tick.
|
||
/// Bounded to keep the working set small even if there's a flood.
|
||
const CLAIM_BATCH: i64 = 8;
|
||
|
||
/// Inputs to `Dispatcher::fan_out_dead_letter`. Pulled out as a struct
|
||
/// so the outbox arm (`handle_failure`) and the queue arm
|
||
/// (`handle_queue_failure`) can both call it — the queue arm doesn't
|
||
/// have an `OutboxRow` to pass.
|
||
struct DeadLetterFanOutCtx {
|
||
app_id: picloud_shared::AppId,
|
||
/// The originating event being failed, verbatim. The DL event
|
||
/// nests this under `original` so handler scripts see what
|
||
/// triggered the chain.
|
||
original: TriggerEvent,
|
||
/// Trigger source the DL row was filed under (`"kv"`, `"queue"`,
|
||
/// `"http"`, …). Drives the `source_filter` match on registered
|
||
/// `dead_letter` triggers.
|
||
source: String,
|
||
dead_letter_id: DeadLetterId,
|
||
attempts: u32,
|
||
last_error: String,
|
||
/// Original trigger id (for trigger_id_filter matching).
|
||
trigger_id: Option<picloud_shared::TriggerId>,
|
||
/// Original script id (for script_id_filter matching).
|
||
script_id: Option<ScriptId>,
|
||
first_attempt_at: DateTime<Utc>,
|
||
last_attempt_at: DateTime<Utc>,
|
||
trigger_depth: u32,
|
||
root_execution_id: Option<ExecutionId>,
|
||
}
|
||
|
||
/// Polling cadence. Short enough that fan-out feels instant; long
|
||
/// enough that an idle dispatcher doesn't burn cycles.
|
||
/// F-Q-009: env-overridable via `PICLOUD_DISPATCHER_TICK_INTERVAL_MS`.
|
||
const DEFAULT_TICK_INTERVAL: Duration = Duration::from_millis(100);
|
||
|
||
/// Hard cap on the wall-clock budget passed to the executor for an
|
||
/// async-dispatched script. Sync HTTP gets a per-script timeout via
|
||
/// the orchestrator path; async rows don't have one, so we apply a
|
||
/// platform-wide ceiling here. Matches `LocalExecutorClient`'s own
|
||
/// 5-minute cap.
|
||
/// F-Q-009: env-overridable via `PICLOUD_DISPATCHER_ASYNC_EXEC_TIMEOUT_SEC`.
|
||
///
|
||
/// `pub` so `triggers_api::validate_queue_visibility_timeout` can use
|
||
/// the same default for its warn-threshold — otherwise a change here
|
||
/// would silently leave the visibility-timeout warner out of sync.
|
||
pub const DEFAULT_ASYNC_EXEC_TIMEOUT_SECS: u32 = 300;
|
||
const DEFAULT_ASYNC_EXEC_TIMEOUT: Duration =
|
||
Duration::from_secs(DEFAULT_ASYNC_EXEC_TIMEOUT_SECS as u64);
|
||
|
||
/// Read `PICLOUD_DISPATCHER_TICK_INTERVAL_MS`; invalid values fall back
|
||
/// to the documented default with a tracing-warn.
|
||
fn tick_interval_from_env() -> Duration {
|
||
if let Ok(v) = std::env::var("PICLOUD_DISPATCHER_TICK_INTERVAL_MS") {
|
||
match v.trim().parse::<u64>() {
|
||
Ok(n) if n > 0 => return Duration::from_millis(n),
|
||
_ => tracing::warn!(
|
||
value = %v,
|
||
"ignoring invalid PICLOUD_DISPATCHER_TICK_INTERVAL_MS (want a positive integer)"
|
||
),
|
||
}
|
||
}
|
||
DEFAULT_TICK_INTERVAL
|
||
}
|
||
|
||
/// Read `PICLOUD_DISPATCHER_ASYNC_EXEC_TIMEOUT_SEC`; invalid values
|
||
/// fall back to the documented default with a tracing-warn.
|
||
fn async_exec_timeout_from_env() -> Duration {
|
||
if let Ok(v) = std::env::var("PICLOUD_DISPATCHER_ASYNC_EXEC_TIMEOUT_SEC") {
|
||
match v.trim().parse::<u64>() {
|
||
Ok(n) if n > 0 => return Duration::from_secs(n),
|
||
_ => tracing::warn!(
|
||
value = %v,
|
||
"ignoring invalid PICLOUD_DISPATCHER_ASYNC_EXEC_TIMEOUT_SEC (want a positive integer)"
|
||
),
|
||
}
|
||
}
|
||
DEFAULT_ASYNC_EXEC_TIMEOUT
|
||
}
|
||
|
||
/// F-P-007: per-tick concurrency cap for queue dispatches. Matches the
|
||
/// default execution-gate concurrency so the gate is what bounds real
|
||
/// parallelism, not the dispatcher's serial-await.
|
||
const QUEUE_DISPATCH_PARALLELISM: usize = 32;
|
||
|
||
impl Dispatcher {
|
||
/// Spawn the dispatcher loop as a detached `tokio::task`. Also
|
||
/// spawns the v1.1.9 queue visibility-timeout reclaim task. Both
|
||
/// run for the process lifetime; returned `JoinHandle`s are
|
||
/// dropped on purpose.
|
||
pub fn spawn(self) {
|
||
// Reclaim task: independent cadence (default 30s) so it doesn't
|
||
// contend with the per-100ms dispatcher tick.
|
||
let reclaim_queue = self.queue.clone();
|
||
let reclaim_interval =
|
||
Duration::from_millis(u64::from(self.config.queue_reclaim_interval_ms));
|
||
tokio::spawn(async move {
|
||
let mut ticker = tokio::time::interval(reclaim_interval);
|
||
ticker.tick().await;
|
||
loop {
|
||
ticker.tick().await;
|
||
match reclaim_queue.reclaim_visibility_timeouts().await {
|
||
Ok(0) => {}
|
||
Ok(n) => tracing::info!(reclaimed = n, "queue visibility-timeout reclaim"),
|
||
Err(e) => tracing::warn!(?e, "queue reclaim task errored"),
|
||
}
|
||
}
|
||
});
|
||
|
||
tokio::spawn(async move {
|
||
self.run().await;
|
||
});
|
||
}
|
||
|
||
async fn run(self) {
|
||
let mut ticker = tokio::time::interval(tick_interval_from_env());
|
||
// Skip the immediate first fire so we don't race startup.
|
||
ticker.tick().await;
|
||
loop {
|
||
ticker.tick().await;
|
||
if let Err(err) = self.tick().await {
|
||
tracing::warn!(?err, "dispatcher tick errored");
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn tick(&self) -> Result<(), DispatcherError> {
|
||
// 1) Outbox arm — KV/docs/pubsub/cron/email/files/dead-letter/http/invoke.
|
||
let rows = self
|
||
.outbox
|
||
.claim_due(&self.instance_id, CLAIM_BATCH)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
for row in rows {
|
||
// Process serially within a tick — the outer ticker is the
|
||
// pacing mechanism. Concurrent dispatchers are a cluster-
|
||
// mode concern; v1.1.1 MVP has one.
|
||
if let Err(err) = self.dispatch_one(row).await {
|
||
tracing::warn!(?err, "dispatch one errored");
|
||
}
|
||
}
|
||
|
||
// 2) Queue arm (v1.1.9). One claim per active consumer per tick;
|
||
// bounded by the number of registered consumers so a flood in
|
||
// one queue doesn't starve the rest.
|
||
if let Err(err) = self.tick_queue_arm().await {
|
||
tracing::warn!(?err, "queue arm tick errored");
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// v1.1.9. Per tick: list every `queue:receive` consumer and
|
||
/// attempt one claim per `(app_id, queue_name)`. Each successful
|
||
/// claim runs through the same executor path as outbox-backed
|
||
/// trigger dispatches, so retry / dead-letter wiring is shared.
|
||
///
|
||
/// F-P-007: bound concurrency with `for_each_concurrent` instead of
|
||
/// serial-await. With N consumers and a 100ms tick the worst-case
|
||
/// throughput was N × (claim + executor) / tick; concurrent dispatch
|
||
/// lets one slow handler not block the others. The execution gate
|
||
/// still caps real parallelism — this just removes the
|
||
/// dispatcher-side serialization.
|
||
async fn tick_queue_arm(&self) -> Result<(), DispatcherError> {
|
||
use futures::stream::{self, StreamExt};
|
||
let consumers = self
|
||
.triggers
|
||
.list_active_queue_consumers()
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
|
||
// Concurrency cap matches the execution gate's typical default;
|
||
// overflow attempts return Err(Overloaded) and back off so a
|
||
// queue burst doesn't hammer the executor.
|
||
stream::iter(consumers)
|
||
.for_each_concurrent(QUEUE_DISPATCH_PARALLELISM, |consumer| async move {
|
||
if let Err(err) = self.dispatch_one_queue(&consumer).await {
|
||
tracing::warn!(?err, queue = %consumer.queue_name, "queue dispatch errored");
|
||
}
|
||
})
|
||
.await;
|
||
Ok(())
|
||
}
|
||
|
||
#[allow(clippy::too_many_lines)]
|
||
async fn dispatch_one_queue(
|
||
&self,
|
||
consumer: &ActiveQueueConsumer,
|
||
) -> Result<(), DispatcherError> {
|
||
// Atomic claim — None → nothing pending right now for this queue.
|
||
let Some(claimed) = self
|
||
.queue
|
||
.claim(consumer.app_id, &consumer.queue_name)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
|
||
// Gate admission. If saturated, release the claim by nack-
|
||
// immediate (which lets the next tick re-claim). Mirrors the
|
||
// outbox arm.
|
||
let Ok(permit) = self.gate.try_acquire() else {
|
||
let _ = self
|
||
.queue
|
||
.nack(
|
||
claimed.id,
|
||
claimed.claim_token,
|
||
chrono::Duration::milliseconds(100),
|
||
)
|
||
.await;
|
||
return Ok(());
|
||
};
|
||
|
||
// Build the TriggerEvent and ExecRequest. Mirrors build_exec_request
|
||
// but uses the consumer + claimed-message data instead of an
|
||
// outbox row.
|
||
let event = TriggerEvent::Queue {
|
||
queue_name: claimed.queue_name.clone(),
|
||
message: claimed.payload.clone(),
|
||
enqueued_at: claimed.enqueued_at,
|
||
attempt: claimed.attempt,
|
||
message_id: claimed.id.to_string(),
|
||
};
|
||
|
||
let script = match self.scripts.get(consumer.script_id).await {
|
||
Ok(Some(s)) => s,
|
||
Ok(None) => {
|
||
tracing::warn!(script_id = %consumer.script_id, "queue trigger script missing; dead-lettering");
|
||
let _ = self
|
||
.queue
|
||
.dead_letter(
|
||
claimed.id,
|
||
claimed.claim_token,
|
||
claimed.app_id,
|
||
&claimed.queue_name,
|
||
Some(consumer.trigger_id),
|
||
Some(consumer.script_id),
|
||
claimed.attempt,
|
||
claimed.enqueued_at,
|
||
"queue trigger script not found",
|
||
)
|
||
.await;
|
||
drop(permit);
|
||
return Ok(());
|
||
}
|
||
Err(e) => return Err(DispatcherError::ResolveTrigger(e.to_string())),
|
||
};
|
||
|
||
// Audit 2026-06-11 H-F1 — same-app guard. validate_trigger_target
|
||
// rejects cross-app rows at write time; this is the runtime
|
||
// backstop for hand-edited triggers, partial backup restores, or
|
||
// a script re-pointed across apps without re-validating the
|
||
// existing consumers. Dead-letter the message so the misfire is
|
||
// observable rather than executing one app's script under
|
||
// another app's SdkCallCx.
|
||
if script.app_id != claimed.app_id {
|
||
tracing::error!(
|
||
script_app = %script.app_id,
|
||
claim_app = %claimed.app_id,
|
||
script_id = %consumer.script_id,
|
||
"queue consumer script belongs to a different app; dead-lettering"
|
||
);
|
||
let _ = self
|
||
.queue
|
||
.dead_letter(
|
||
claimed.id,
|
||
claimed.claim_token,
|
||
claimed.app_id,
|
||
&claimed.queue_name,
|
||
Some(consumer.trigger_id),
|
||
Some(consumer.script_id),
|
||
claimed.attempt,
|
||
claimed.enqueued_at,
|
||
"queue consumer target belongs to a different app",
|
||
)
|
||
.await;
|
||
drop(permit);
|
||
return Ok(());
|
||
}
|
||
|
||
let principal = self
|
||
.principals
|
||
.resolve(consumer.registered_by_principal)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
|
||
|
||
let execution_id = ExecutionId::new();
|
||
let exec_req = ExecRequest {
|
||
execution_id,
|
||
request_id: RequestId::new(),
|
||
script_id: consumer.script_id,
|
||
script_name: script.name.clone(),
|
||
invocation_type: InvocationType::Function,
|
||
path: "/trigger/queue".into(),
|
||
headers: std::collections::BTreeMap::new(),
|
||
body: serde_json::Value::Null,
|
||
params: std::collections::BTreeMap::new(),
|
||
query: std::collections::BTreeMap::new(),
|
||
rest: String::new(),
|
||
sandbox_overrides: script.sandbox,
|
||
app_id: claimed.app_id,
|
||
principal: Some(principal),
|
||
trigger_depth: 0,
|
||
root_execution_id: execution_id,
|
||
is_dead_letter_handler: false,
|
||
event: Some(event),
|
||
};
|
||
let identity = picloud_orchestrator_core::ScriptIdentity {
|
||
script_id: consumer.script_id,
|
||
updated_at: script.updated_at,
|
||
};
|
||
let outcome = self
|
||
.executor
|
||
.execute_with_identity(
|
||
identity,
|
||
&script.source,
|
||
exec_req,
|
||
async_exec_timeout_from_env(),
|
||
)
|
||
.await;
|
||
drop(permit);
|
||
|
||
// Best-effort touch on last_fired_at (a failure here doesn't
|
||
// change ack/nack behavior).
|
||
if let Err(e) = self
|
||
.triggers
|
||
.touch_queue_trigger_last_fired_at(consumer.trigger_id, Utc::now())
|
||
.await
|
||
{
|
||
tracing::debug!(?e, "touch last_fired_at failed");
|
||
}
|
||
|
||
match outcome {
|
||
Ok(_) => {
|
||
// Auto-ack on success.
|
||
if let Err(e) = self.queue.ack(claimed.id, claimed.claim_token).await {
|
||
tracing::warn!(?e, "queue ack failed");
|
||
}
|
||
}
|
||
Err(err) => {
|
||
self.handle_queue_failure(consumer, &claimed, err).await;
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
async fn handle_queue_failure(
|
||
&self,
|
||
consumer: &ActiveQueueConsumer,
|
||
claimed: &ClaimedMessage,
|
||
err: ExecError,
|
||
) {
|
||
if claimed.attempt < claimed.max_attempts {
|
||
// Retry: clear claim + set deliver_after.
|
||
let delay_ms = compute_backoff(
|
||
claimed.attempt,
|
||
consumer.retry_backoff,
|
||
consumer.retry_base_ms,
|
||
self.config.retry_jitter_pct,
|
||
);
|
||
let delay = chrono::Duration::milliseconds(i64::from(delay_ms));
|
||
if let Err(e) = self
|
||
.queue
|
||
.nack(claimed.id, claimed.claim_token, delay)
|
||
.await
|
||
{
|
||
tracing::warn!(?e, "queue nack failed");
|
||
}
|
||
return;
|
||
}
|
||
// Exhausted. Dead-letter inline (the helper deletes the queue row +
|
||
// writes a dead_letters row in one transaction). The outbox arm
|
||
// calls `fan_out_dead_letter` after writing its DL row; the queue
|
||
// arm previously omitted this, so registered `dead_letter`
|
||
// handlers filtered on `source = "queue"` sat idle. Fan out the
|
||
// same way here.
|
||
let now = Utc::now();
|
||
let last_error = err.to_string();
|
||
let dl_id = match self
|
||
.queue
|
||
.dead_letter(
|
||
claimed.id,
|
||
claimed.claim_token,
|
||
claimed.app_id,
|
||
&claimed.queue_name,
|
||
Some(consumer.trigger_id),
|
||
Some(consumer.script_id),
|
||
claimed.attempt,
|
||
claimed.enqueued_at,
|
||
&last_error,
|
||
)
|
||
.await
|
||
{
|
||
Ok(dl_id) => Some(dl_id),
|
||
Err(e) => {
|
||
tracing::error!(?e, "queue dead-letter write failed");
|
||
None
|
||
}
|
||
};
|
||
if let Some(dead_letter_id) = dl_id {
|
||
let original = TriggerEvent::Queue {
|
||
queue_name: claimed.queue_name.clone(),
|
||
message: claimed.payload.clone(),
|
||
enqueued_at: claimed.enqueued_at,
|
||
attempt: claimed.attempt,
|
||
message_id: claimed.id.to_string(),
|
||
};
|
||
// dead_letter triggers filter on the originating event's
|
||
// wire source ("queue", "kv", "http", …) — same vocabulary
|
||
// as TriggerEvent::source().
|
||
self.fan_out_dead_letter(DeadLetterFanOutCtx {
|
||
app_id: claimed.app_id,
|
||
original,
|
||
source: "queue".to_string(),
|
||
dead_letter_id,
|
||
attempts: claimed.attempt,
|
||
last_error,
|
||
trigger_id: Some(consumer.trigger_id),
|
||
script_id: Some(consumer.script_id),
|
||
first_attempt_at: claimed.enqueued_at,
|
||
last_attempt_at: now,
|
||
// Queue messages always root a depth-1 chain (the queue
|
||
// itself is depth 0; any subsequent DL handler ticks up).
|
||
trigger_depth: 1,
|
||
root_execution_id: None,
|
||
})
|
||
.await;
|
||
}
|
||
// TODO(metrics): bump picloud_queue_dead_letters_total{app_id, queue_name}.
|
||
}
|
||
|
||
async fn dispatch_one(&self, row: OutboxRow) -> Result<(), DispatcherError> {
|
||
// Depth-limit check — design notes §4: loops aren't DL'd.
|
||
if row.trigger_depth > self.config.max_trigger_depth {
|
||
tracing::warn!(
|
||
outbox_id = %row.id,
|
||
app_id = %row.app_id,
|
||
trigger_depth = row.trigger_depth,
|
||
"trigger depth exceeded; dropping row"
|
||
);
|
||
// TODO(metrics): bump `picloud_trigger_depth_exceeded{app_id,trigger_id}`.
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
return Ok(());
|
||
}
|
||
|
||
// Gate admission — non-blocking. If the gate is saturated,
|
||
// release the claim by rescheduling so another tick can pick
|
||
// it up. The row stays "due" essentially immediately.
|
||
let Ok(permit) = self.gate.try_acquire() else {
|
||
let next = Utc::now() + chrono::Duration::milliseconds(100);
|
||
self.outbox
|
||
.reschedule(row.id, row.attempt_count, next)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
return Ok(());
|
||
};
|
||
|
||
// Resolve the trigger config (KV / DL) or pull the HTTP
|
||
// payload directly off the outbox row.
|
||
let (resolved, exec_req) = match row.source_kind {
|
||
OutboxSourceKind::Http => match self.build_http_request(&row).await {
|
||
Ok(pair) => pair,
|
||
Err(err) => {
|
||
tracing::warn!(outbox_id = %row.id, ?err, "http exec build failed; dropping");
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
drop(permit);
|
||
return Ok(());
|
||
}
|
||
},
|
||
OutboxSourceKind::Invoke => match self.build_invoke_request(&row).await {
|
||
Ok(pair) => pair,
|
||
Err(err) => {
|
||
tracing::warn!(outbox_id = %row.id, ?err, "invoke exec build failed; dropping");
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
drop(permit);
|
||
return Ok(());
|
||
}
|
||
},
|
||
OutboxSourceKind::Kv
|
||
| OutboxSourceKind::Docs
|
||
| OutboxSourceKind::DeadLetter
|
||
| OutboxSourceKind::Cron
|
||
| OutboxSourceKind::Files
|
||
| OutboxSourceKind::Pubsub
|
||
| OutboxSourceKind::Email => {
|
||
let resolved = self.resolve_trigger(&row).await?;
|
||
let req = match self.build_exec_request(&row, &resolved).await {
|
||
Ok(req) => req,
|
||
Err(err) => {
|
||
tracing::warn!(outbox_id = %row.id, ?err, "exec request build failed; dropping row");
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
drop(permit);
|
||
return Ok(());
|
||
}
|
||
};
|
||
(resolved, req)
|
||
}
|
||
};
|
||
|
||
// The gate permit auto-releases when this scope ends or when
|
||
// the executor finishes. We hand control to the executor and
|
||
// wait synchronously here — sync HTTP and dispatcher share the
|
||
// semaphore so this is intentional.
|
||
let source = resolved.script_source.clone();
|
||
let identity = picloud_orchestrator_core::ScriptIdentity {
|
||
script_id: resolved.script_id,
|
||
updated_at: resolved.script_updated_at,
|
||
};
|
||
let outcome = self
|
||
.executor
|
||
.execute_with_identity(identity, &source, exec_req, async_exec_timeout_from_env())
|
||
.await;
|
||
drop(permit);
|
||
|
||
match outcome {
|
||
Ok(resp) => self.handle_success(&row, &resolved, resp).await,
|
||
Err(err) => self.handle_failure(&row, &resolved, err).await,
|
||
}
|
||
}
|
||
|
||
async fn resolve_trigger(&self, row: &OutboxRow) -> Result<ResolvedTrigger, DispatcherError> {
|
||
// For KV and DL kinds, the outbox carries `trigger_id`. Use it
|
||
// to look up the trigger row, then resolve the script.
|
||
let Some(trigger_id) = row.trigger_id else {
|
||
return Err(DispatcherError::ResolveTrigger(
|
||
"outbox row missing trigger_id".into(),
|
||
));
|
||
};
|
||
let trigger = self
|
||
.triggers
|
||
.get(trigger_id)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
|
||
.ok_or_else(|| {
|
||
DispatcherError::ResolveTrigger(format!("trigger {trigger_id} not found"))
|
||
})?;
|
||
|
||
let script = self
|
||
.scripts
|
||
.get(trigger.script_id)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
|
||
.ok_or_else(|| {
|
||
DispatcherError::ResolveTrigger(format!("script {} not found", trigger.script_id))
|
||
})?;
|
||
|
||
Ok(ResolvedTrigger {
|
||
trigger_kind: trigger.kind,
|
||
is_dead_letter_handler: matches!(trigger.kind, TriggerKind::DeadLetter),
|
||
script_id: script.id,
|
||
script_source: script.source,
|
||
script_name: script.name,
|
||
script_updated_at: script.updated_at,
|
||
sandbox_overrides: script.sandbox,
|
||
registered_by_principal: trigger.registered_by_principal,
|
||
retry_max_attempts: trigger.retry_max_attempts,
|
||
retry_backoff: trigger.retry_backoff,
|
||
retry_base_ms: trigger.retry_base_ms,
|
||
})
|
||
}
|
||
|
||
async fn build_exec_request(
|
||
&self,
|
||
row: &OutboxRow,
|
||
resolved: &ResolvedTrigger,
|
||
) -> Result<ExecRequest, DispatcherError> {
|
||
let trigger_event: TriggerEvent = serde_json::from_value(row.payload.clone())
|
||
.map_err(|e| DispatcherError::ResolveTrigger(format!("decode payload: {e}")))?;
|
||
|
||
let principal = self
|
||
.principals
|
||
.resolve(resolved.registered_by_principal)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
|
||
|
||
let execution_id = ExecutionId::new();
|
||
Ok(ExecRequest {
|
||
execution_id,
|
||
request_id: RequestId::new(),
|
||
script_id: resolved.script_id,
|
||
script_name: resolved.script_name.clone(),
|
||
invocation_type: InvocationType::Function,
|
||
path: format!("/trigger/{}", trigger_event.source()),
|
||
headers: std::collections::BTreeMap::new(),
|
||
body: serde_json::Value::Null,
|
||
params: std::collections::BTreeMap::new(),
|
||
query: std::collections::BTreeMap::new(),
|
||
rest: String::new(),
|
||
sandbox_overrides: resolved.sandbox_overrides,
|
||
app_id: row.app_id,
|
||
principal: Some(principal),
|
||
trigger_depth: row.trigger_depth,
|
||
root_execution_id: row.root_execution_id.unwrap_or(execution_id),
|
||
is_dead_letter_handler: resolved.is_dead_letter_handler,
|
||
event: Some(trigger_event),
|
||
})
|
||
}
|
||
|
||
/// Build an `(ResolvedTrigger, ExecRequest)` for an HTTP outbox
|
||
/// row. HTTP rows don't have a backing `triggers` row (the
|
||
/// `trigger_id` references `routes.id` instead). We pull the
|
||
/// script id off the outbox row, the request shape off the
|
||
/// payload, and synthesize a `ResolvedTrigger` with retry
|
||
/// settings irrelevant for HTTP (sync HTTP is never retried;
|
||
/// async HTTP uses default policy from `TriggerConfig`).
|
||
async fn build_http_request(
|
||
&self,
|
||
row: &OutboxRow,
|
||
) -> Result<(ResolvedTrigger, ExecRequest), DispatcherError> {
|
||
let Some(script_id) = row.script_id else {
|
||
return Err(DispatcherError::ResolveTrigger(
|
||
"HTTP outbox row missing script_id".into(),
|
||
));
|
||
};
|
||
let script = self
|
||
.scripts
|
||
.get(script_id)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
|
||
.ok_or_else(|| {
|
||
DispatcherError::ResolveTrigger(format!("script {script_id} not found"))
|
||
})?;
|
||
|
||
// Audit 2026-06-11 H-F1 sibling — same-app guard mirroring
|
||
// build_invoke_request and dispatch_one_queue. A hand-edited
|
||
// outbox row could otherwise execute one app's script under
|
||
// another app's SdkCallCx.
|
||
if script.app_id != row.app_id {
|
||
return Err(DispatcherError::ResolveTrigger(
|
||
"http outbox target belongs to a different app".into(),
|
||
));
|
||
}
|
||
|
||
let payload: HttpDispatchPayload = serde_json::from_value(row.payload.clone())
|
||
.map_err(|e| DispatcherError::ResolveTrigger(format!("decode http payload: {e}")))?;
|
||
|
||
let execution_id = ExecutionId::new();
|
||
let req = ExecRequest {
|
||
execution_id,
|
||
request_id: RequestId::new(),
|
||
script_id,
|
||
script_name: payload.script_name.clone(),
|
||
invocation_type: InvocationType::Http,
|
||
path: payload.path.clone(),
|
||
headers: payload.headers,
|
||
body: payload.body,
|
||
params: payload.params,
|
||
query: payload.query,
|
||
rest: payload.rest,
|
||
sandbox_overrides: script.sandbox,
|
||
app_id: row.app_id,
|
||
// HTTP outbox rows don't run as the trigger registrant —
|
||
// they run with no principal (public ingress) or the
|
||
// attached one (origin_principal forensic field is not
|
||
// promoted to execution principal in this MVP).
|
||
principal: None,
|
||
trigger_depth: row.trigger_depth,
|
||
root_execution_id: row.root_execution_id.unwrap_or(execution_id),
|
||
is_dead_letter_handler: false,
|
||
event: None,
|
||
};
|
||
|
||
let resolved = ResolvedTrigger {
|
||
trigger_kind: TriggerKind::Kv, // placeholder; HTTP doesn't have a kind
|
||
is_dead_letter_handler: false,
|
||
script_id,
|
||
script_source: script.source,
|
||
script_name: payload.script_name,
|
||
script_updated_at: script.updated_at,
|
||
sandbox_overrides: script.sandbox,
|
||
// HTTP outbox rows don't carry a registered_by_principal
|
||
// — use a sentinel zero UUID since this field isn't used
|
||
// downstream for HTTP (no retries, no inbox principal).
|
||
registered_by_principal: picloud_shared::AdminUserId::from(uuid::Uuid::nil()),
|
||
// Async HTTP uses the platform default retry policy from
|
||
// TriggerConfig. Sync HTTP (reply_to.is_some) never retries
|
||
// regardless.
|
||
retry_max_attempts: self.config.retry_max_attempts,
|
||
retry_backoff: self.config.retry_backoff,
|
||
retry_base_ms: self.config.retry_base_ms,
|
||
};
|
||
Ok((resolved, req))
|
||
}
|
||
|
||
/// v1.1.9. Build an `(ResolvedTrigger, ExecRequest)` from an
|
||
/// invoke_async outbox row. The payload was written by
|
||
/// `InvokeService::enqueue_async`. No retry policy — invoke_async
|
||
/// runs once; on failure the row is deleted and a dead_letters row
|
||
/// is written (so the misfire is observable).
|
||
async fn build_invoke_request(
|
||
&self,
|
||
row: &OutboxRow,
|
||
) -> Result<(ResolvedTrigger, ExecRequest), DispatcherError> {
|
||
// Extract the args + script_id from the payload (InvokeService
|
||
// populates these). Defensive: dead_letter the row if any
|
||
// required field is missing.
|
||
let payload = &row.payload;
|
||
let Some(script_id_str) = payload.get("script_id").and_then(|v| v.as_str()) else {
|
||
return Err(DispatcherError::ResolveTrigger(
|
||
"invoke row missing script_id".into(),
|
||
));
|
||
};
|
||
let script_id = picloud_shared::ScriptId::from(
|
||
uuid::Uuid::parse_str(script_id_str)
|
||
.map_err(|e| DispatcherError::ResolveTrigger(format!("script_id: {e}")))?,
|
||
);
|
||
let script = self
|
||
.scripts
|
||
.get(script_id)
|
||
.await
|
||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?
|
||
.ok_or_else(|| {
|
||
DispatcherError::ResolveTrigger(format!("script {script_id} not found"))
|
||
})?;
|
||
// Same-app guard — the script could have been re-assigned or
|
||
// the row could have been hand-edited. Reject mismatches.
|
||
if script.app_id != row.app_id {
|
||
return Err(DispatcherError::ResolveTrigger(
|
||
"invoke target belongs to a different app".into(),
|
||
));
|
||
}
|
||
|
||
let args = payload
|
||
.get("args")
|
||
.cloned()
|
||
.unwrap_or(serde_json::Value::Null);
|
||
let trigger_depth = payload
|
||
.get("trigger_depth")
|
||
.and_then(serde_json::Value::as_u64)
|
||
.map_or(row.trigger_depth, |n| u32::try_from(n).unwrap_or(u32::MAX));
|
||
|
||
let execution_id = ExecutionId::new();
|
||
let req = ExecRequest {
|
||
execution_id,
|
||
request_id: RequestId::new(),
|
||
script_id: script.id,
|
||
script_name: script.name.clone(),
|
||
invocation_type: InvocationType::Function,
|
||
path: "/invoke_async".into(),
|
||
headers: std::collections::BTreeMap::new(),
|
||
body: args,
|
||
params: std::collections::BTreeMap::new(),
|
||
query: std::collections::BTreeMap::new(),
|
||
rest: String::new(),
|
||
sandbox_overrides: script.sandbox,
|
||
app_id: row.app_id,
|
||
// invoke_async runs with no principal — same as HTTP outbox.
|
||
principal: None,
|
||
trigger_depth,
|
||
root_execution_id: row.root_execution_id.unwrap_or(execution_id),
|
||
is_dead_letter_handler: false,
|
||
event: None,
|
||
};
|
||
|
||
// Build a synthetic ResolvedTrigger so the dispatch_one outer
|
||
// loop can carry on through the same code path. Retry knobs
|
||
// bound to 1 attempt: invoke_async runs ONCE (callers wrap in
|
||
// retry::with for retries).
|
||
let resolved = ResolvedTrigger {
|
||
trigger_kind: TriggerKind::Cron, // placeholder; not used downstream
|
||
is_dead_letter_handler: false,
|
||
script_id: script.id,
|
||
script_source: script.source,
|
||
script_name: script.name,
|
||
script_updated_at: script.updated_at,
|
||
sandbox_overrides: script.sandbox,
|
||
registered_by_principal: picloud_shared::AdminUserId::from(uuid::Uuid::nil()),
|
||
retry_max_attempts: 1,
|
||
retry_backoff: BackoffShape::Constant,
|
||
retry_base_ms: 0,
|
||
};
|
||
Ok((resolved, req))
|
||
}
|
||
|
||
async fn handle_success(
|
||
&self,
|
||
row: &OutboxRow,
|
||
_resolved: &ResolvedTrigger,
|
||
resp: ExecResponse,
|
||
) -> Result<(), DispatcherError> {
|
||
if let Some(inbox_id) = row.reply_to {
|
||
self.deliver_inbox(row, inbox_id, InboxResult::Success(summarize(&resp)))
|
||
.await;
|
||
}
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
Ok(())
|
||
}
|
||
|
||
#[allow(clippy::too_many_lines)]
|
||
async fn handle_failure(
|
||
&self,
|
||
row: &OutboxRow,
|
||
resolved: &ResolvedTrigger,
|
||
err: ExecError,
|
||
) -> Result<(), DispatcherError> {
|
||
// Sync HTTP: always single-attempt. Always deliver outcome
|
||
// (success-or-failure) to the inbox. Never retry, never DL.
|
||
if let Some(inbox_id) = row.reply_to {
|
||
let (kind, message) = classify_exec_error(&err);
|
||
self.deliver_inbox(
|
||
row,
|
||
inbox_id,
|
||
InboxResult::Failure {
|
||
kind,
|
||
message: message.clone(),
|
||
},
|
||
)
|
||
.await;
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
return Ok(());
|
||
}
|
||
|
||
// Dead-letter handler: never retry, never DL. Failure
|
||
// annotates the original DL row + bumps a metric.
|
||
if resolved.is_dead_letter_handler {
|
||
tracing::error!(
|
||
outbox_id = %row.id,
|
||
app_id = %row.app_id,
|
||
?err,
|
||
"dead-letter handler failed; not retrying"
|
||
);
|
||
// TODO(metrics): bump `picloud_dead_letter_handler_failures{app_id}`.
|
||
// Annotate the original DL row (id is `row.payload.dead_letter.id`
|
||
// when the payload is a DeadLetter TriggerEvent). Best-effort:
|
||
// if the payload doesn't decode, just log and move on.
|
||
if let Ok(TriggerEvent::DeadLetter { dead_letter_id, .. }) =
|
||
serde_json::from_value::<TriggerEvent>(row.payload.clone())
|
||
{
|
||
if let Err(e) = self
|
||
.dead_letters
|
||
.resolve(dead_letter_id, "handler_failed")
|
||
.await
|
||
{
|
||
tracing::warn!(?e, "could not annotate DL row as handler_failed");
|
||
}
|
||
}
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
return Ok(());
|
||
}
|
||
|
||
// Async event: retry per policy, then dead-letter.
|
||
let attempt = row.attempt_count + 1;
|
||
if attempt < resolved.retry_max_attempts {
|
||
let delay = compute_backoff(
|
||
attempt,
|
||
resolved.retry_backoff,
|
||
resolved.retry_base_ms,
|
||
self.config.retry_jitter_pct,
|
||
);
|
||
let next = Utc::now() + chrono::Duration::milliseconds(i64::from(delay));
|
||
tracing::info!(
|
||
outbox_id = %row.id,
|
||
attempt,
|
||
max_attempts = resolved.retry_max_attempts,
|
||
retry_in_ms = delay,
|
||
"rescheduling outbox row"
|
||
);
|
||
self.outbox
|
||
.reschedule(row.id, attempt, next)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
return Ok(());
|
||
}
|
||
|
||
// Exhausted retries → dead-letter.
|
||
let (op, source) = describe_event(row.source_kind, &row.payload);
|
||
let now = Utc::now();
|
||
let dl_id = match self
|
||
.dead_letters
|
||
.insert(NewDeadLetter {
|
||
app_id: row.app_id,
|
||
original_event_id: row.id,
|
||
source: source.clone(),
|
||
op,
|
||
trigger_id: row.trigger_id,
|
||
script_id: Some(resolved.script_id),
|
||
payload: row.payload.clone(),
|
||
attempt_count: attempt,
|
||
first_attempt_at: row.created_at,
|
||
last_attempt_at: now,
|
||
last_error: err.to_string(),
|
||
})
|
||
.await
|
||
{
|
||
Ok(id) => Some(id),
|
||
Err(e) => {
|
||
tracing::error!(?e, "failed to write dead-letter row");
|
||
None
|
||
}
|
||
};
|
||
|
||
// v1.1.7 fix: fan the dead-letter out to matching handler triggers.
|
||
// This was missing since v1.1.1 — the row was written but
|
||
// `list_matching_dead_letter` had no production caller, so
|
||
// registered dead_letter handlers never fired. The recursion-stop
|
||
// (a dead-letter handler's own failure is not re-dead-lettered)
|
||
// is upheld by the `is_dead_letter_handler` short-circuit at the
|
||
// top of this function, so this fan-out is only reached for
|
||
// non-handler executions.
|
||
if let Some(dl_id) = dl_id {
|
||
// The DL event nests the original verbatim; if the payload
|
||
// can't be decoded back into a TriggerEvent we can't build
|
||
// the nested `original`, so skip the fan-out (the DL row is
|
||
// still written).
|
||
match serde_json::from_value::<TriggerEvent>(row.payload.clone()) {
|
||
Ok(original) => {
|
||
self.fan_out_dead_letter(DeadLetterFanOutCtx {
|
||
app_id: row.app_id,
|
||
original,
|
||
source,
|
||
dead_letter_id: dl_id,
|
||
attempts: attempt,
|
||
last_error: err.to_string(),
|
||
trigger_id: row.trigger_id,
|
||
script_id: Some(resolved.script_id),
|
||
first_attempt_at: row.created_at,
|
||
last_attempt_at: now,
|
||
trigger_depth: row.trigger_depth,
|
||
root_execution_id: row.root_execution_id,
|
||
})
|
||
.await;
|
||
}
|
||
Err(_) => {
|
||
tracing::warn!(
|
||
outbox_id = %row.id,
|
||
"dead-letter payload is not a TriggerEvent; skipping handler fan-out"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
self.outbox
|
||
.delete(row.id)
|
||
.await
|
||
.map_err(|e| DispatcherError::Outbox(e.to_string()))?;
|
||
Ok(())
|
||
}
|
||
|
||
/// Enqueue one outbox row per matching `dead_letter` trigger so its
|
||
/// handler script runs with the dead-letter event as `ctx.event`.
|
||
/// Best-effort: a lookup/insert failure is logged, not propagated
|
||
/// (the dead-letter row itself is already durably written).
|
||
///
|
||
/// Called from both the outbox arm (`handle_failure`) and the queue
|
||
/// arm (`handle_queue_failure`). The queue arm previously omitted
|
||
/// this call, so registered `dead_letter` handlers filtered on
|
||
/// `source = "queue"` sat idle even after queue retry-exhaust.
|
||
async fn fan_out_dead_letter(&self, ctx: DeadLetterFanOutCtx) {
|
||
let DeadLetterFanOutCtx {
|
||
app_id,
|
||
original,
|
||
source,
|
||
dead_letter_id,
|
||
attempts,
|
||
last_error,
|
||
trigger_id,
|
||
script_id,
|
||
first_attempt_at,
|
||
last_attempt_at,
|
||
trigger_depth,
|
||
root_execution_id,
|
||
} = ctx;
|
||
|
||
let matches = match self
|
||
.triggers
|
||
.list_matching_dead_letter(app_id, &source, trigger_id, script_id)
|
||
.await
|
||
{
|
||
Ok(m) => m,
|
||
Err(e) => {
|
||
tracing::error!(?e, "dead-letter trigger lookup failed");
|
||
return;
|
||
}
|
||
};
|
||
|
||
for m in matches {
|
||
let event = TriggerEvent::DeadLetter {
|
||
dead_letter_id,
|
||
original: Box::new(original.clone()),
|
||
attempts,
|
||
last_error: last_error.clone(),
|
||
trigger_id,
|
||
script_id,
|
||
first_attempt_at,
|
||
last_attempt_at,
|
||
};
|
||
let payload = match serde_json::to_value(&event) {
|
||
Ok(p) => p,
|
||
Err(e) => {
|
||
tracing::error!(?e, "failed to serialize dead-letter event");
|
||
continue;
|
||
}
|
||
};
|
||
if let Err(e) = self
|
||
.outbox
|
||
.insert(NewOutboxRow {
|
||
app_id,
|
||
source_kind: OutboxSourceKind::DeadLetter,
|
||
trigger_id: Some(m.trigger_id),
|
||
script_id: Some(m.script_id),
|
||
reply_to: None,
|
||
payload,
|
||
origin_principal: Some(m.registered_by_principal),
|
||
trigger_depth: trigger_depth.saturating_add(1),
|
||
root_execution_id,
|
||
})
|
||
.await
|
||
{
|
||
tracing::error!(?e, "failed to enqueue dead-letter handler delivery");
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn deliver_inbox(&self, row: &OutboxRow, inbox_id: Uuid, result: InboxResult) {
|
||
match self.inbox.deliver(inbox_id, result.clone()).await {
|
||
InboxDeliveryOutcome::Delivered => {}
|
||
InboxDeliveryOutcome::Abandoned => {
|
||
// Receiver was dropped — record forensic row + bump
|
||
// metric.
|
||
let (status_code, summary) = match &result {
|
||
InboxResult::Success(s) => (s.status_code, None),
|
||
InboxResult::Failure { kind, message } => {
|
||
(failure_kind_to_status(*kind), Some(message.clone()))
|
||
}
|
||
};
|
||
if let Err(e) = self
|
||
.abandoned
|
||
.insert(NewAbandonedExecution {
|
||
app_id: row.app_id,
|
||
outbox_id: row.id,
|
||
script_id: row.script_id,
|
||
inbox_id,
|
||
status_code,
|
||
result_summary: summary,
|
||
})
|
||
.await
|
||
{
|
||
tracing::warn!(?e, "abandoned_executions insert failed");
|
||
}
|
||
// TODO(metrics): bump `picloud_abandoned_executions_total{app_id}`.
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
pub struct ResolvedTrigger {
|
||
pub trigger_kind: TriggerKind,
|
||
pub is_dead_letter_handler: bool,
|
||
pub script_id: ScriptId,
|
||
pub script_source: String,
|
||
pub script_name: String,
|
||
/// v1.1.3: freshness comparator for the orchestrator's top-level
|
||
/// script cache. The dispatcher hands `(script_id, updated_at)`
|
||
/// in alongside the source so cached ASTs can be reused across
|
||
/// triggered invocations.
|
||
pub script_updated_at: chrono::DateTime<chrono::Utc>,
|
||
pub sandbox_overrides: ScriptSandbox,
|
||
pub registered_by_principal: picloud_shared::AdminUserId,
|
||
pub retry_max_attempts: u32,
|
||
pub retry_backoff: BackoffShape,
|
||
pub retry_base_ms: u32,
|
||
}
|
||
|
||
#[derive(Debug, thiserror::Error)]
|
||
pub enum DispatcherError {
|
||
#[error("outbox: {0}")]
|
||
Outbox(String),
|
||
#[error("resolve trigger: {0}")]
|
||
ResolveTrigger(String),
|
||
}
|
||
|
||
fn summarize(resp: &ExecResponse) -> ExecResponseSummary {
|
||
ExecResponseSummary {
|
||
status_code: resp.status_code,
|
||
headers: resp.headers.clone(),
|
||
body: resp.body.clone(),
|
||
}
|
||
}
|
||
|
||
/// Map `ExecError` onto the design-notes §3 status-code table.
|
||
fn classify_exec_error(err: &ExecError) -> (InboxFailureKind, String) {
|
||
match err {
|
||
ExecError::Parse(s) | ExecError::InvalidResponse(s) => {
|
||
(InboxFailureKind::Validation, s.clone())
|
||
}
|
||
ExecError::Timeout(_) => (InboxFailureKind::Timeout, err.to_string()),
|
||
ExecError::OperationBudgetExceeded => (InboxFailureKind::OperationBudget, err.to_string()),
|
||
ExecError::Overloaded { .. } => (InboxFailureKind::Overloaded, err.to_string()),
|
||
ExecError::Runtime(s) => (InboxFailureKind::Runtime, s.clone()),
|
||
}
|
||
}
|
||
|
||
fn failure_kind_to_status(k: InboxFailureKind) -> u16 {
|
||
match k {
|
||
InboxFailureKind::Validation => 422,
|
||
InboxFailureKind::Runtime => 502,
|
||
InboxFailureKind::Overloaded => 503,
|
||
InboxFailureKind::Timeout => 504,
|
||
InboxFailureKind::OperationBudget => 507,
|
||
InboxFailureKind::Platform => 500,
|
||
}
|
||
}
|
||
|
||
/// `(op, source)` extracted from the outbox payload. Used to seed the
|
||
/// `dead_letters` row when retries exhaust.
|
||
///
|
||
/// The shape of the payload depends on `source_kind`:
|
||
/// * `Http` rows carry an `HttpDispatchPayload` (`method`/`path`).
|
||
/// * `Invoke` rows carry an ad-hoc invoke payload (`script_id`/`args`).
|
||
/// * Every other kind carries a serialized [`TriggerEvent`], which has
|
||
/// `source` and `op` fields at the JSON top level.
|
||
///
|
||
/// F-S-009 (audit fix): previously this function unconditionally read
|
||
/// `source`/`op` from the JSON, which silently produced empty strings
|
||
/// for HTTP rows. The DL row then got written with `source = ""` and
|
||
/// `from_wire("")` returned `None` on replay, defaulting to
|
||
/// `OutboxSourceKind::Kv` — replay rerouted through `resolve_trigger`,
|
||
/// which failed because HTTP rows carry no `trigger_id`. Net result:
|
||
/// HTTP-async DL replays no-op'd silently.
|
||
fn describe_event(source_kind: OutboxSourceKind, payload: &serde_json::Value) -> (String, String) {
|
||
match source_kind {
|
||
OutboxSourceKind::Http => {
|
||
let method = payload
|
||
.get("method")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("?");
|
||
let path = payload.get("path").and_then(|v| v.as_str()).unwrap_or("?");
|
||
(
|
||
format!("{method} {path}"),
|
||
OutboxSourceKind::Http.as_str().to_string(),
|
||
)
|
||
}
|
||
OutboxSourceKind::Invoke => (
|
||
"invoke_async".to_string(),
|
||
OutboxSourceKind::Invoke.as_str().to_string(),
|
||
),
|
||
OutboxSourceKind::Kv
|
||
| OutboxSourceKind::Docs
|
||
| OutboxSourceKind::DeadLetter
|
||
| OutboxSourceKind::Cron
|
||
| OutboxSourceKind::Files
|
||
| OutboxSourceKind::Pubsub
|
||
| OutboxSourceKind::Email => {
|
||
let source = payload
|
||
.get("source")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("")
|
||
.to_string();
|
||
let op = payload
|
||
.get("op")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("")
|
||
.to_string();
|
||
(op, source)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Compute backoff (ms) for the given attempt + policy + jitter.
|
||
/// Attempt is 1-indexed (first retry = attempt 1).
|
||
#[must_use]
|
||
pub fn compute_backoff(attempt: u32, backoff: BackoffShape, base_ms: u32, jitter_pct: u32) -> u32 {
|
||
let base_ms = u64::from(base_ms);
|
||
let attempt = u64::from(attempt.saturating_sub(1));
|
||
let raw = match backoff {
|
||
BackoffShape::Constant => base_ms,
|
||
BackoffShape::Linear => base_ms * (attempt + 1),
|
||
// 1x base, 2x base, 4x base, … (saturating).
|
||
BackoffShape::Exponential => base_ms.saturating_mul(1u64 << attempt.min(20)),
|
||
};
|
||
let raw = u32::try_from(raw.min(u64::from(u32::MAX))).unwrap_or(u32::MAX);
|
||
apply_jitter(raw, jitter_pct)
|
||
}
|
||
|
||
fn apply_jitter(raw: u32, pct: u32) -> u32 {
|
||
if pct == 0 {
|
||
return raw;
|
||
}
|
||
let pct = pct.min(100);
|
||
// ±span% — bounded by raw itself so we can't underflow when
|
||
// raw + offset goes below zero.
|
||
let span = u64::from(raw) * u64::from(pct) / 100;
|
||
if span == 0 {
|
||
return raw;
|
||
}
|
||
let span_i64 = i64::try_from(span).unwrap_or(i64::MAX);
|
||
let mut rng = rand::thread_rng();
|
||
let offset = rng.gen_range(-span_i64..=span_i64);
|
||
let signed = i64::from(raw).saturating_add(offset).max(0);
|
||
u32::try_from(signed.min(i64::from(u32::MAX))).unwrap_or(u32::MAX)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn exponential_backoff_doubles_per_attempt() {
|
||
// No jitter (pct=0) for a deterministic check.
|
||
assert_eq!(compute_backoff(1, BackoffShape::Exponential, 1000, 0), 1000);
|
||
assert_eq!(compute_backoff(2, BackoffShape::Exponential, 1000, 0), 2000);
|
||
assert_eq!(compute_backoff(3, BackoffShape::Exponential, 1000, 0), 4000);
|
||
assert_eq!(compute_backoff(4, BackoffShape::Exponential, 1000, 0), 8000);
|
||
}
|
||
|
||
#[test]
|
||
fn linear_backoff_scales_with_attempt() {
|
||
assert_eq!(compute_backoff(1, BackoffShape::Linear, 100, 0), 100);
|
||
assert_eq!(compute_backoff(2, BackoffShape::Linear, 100, 0), 200);
|
||
assert_eq!(compute_backoff(5, BackoffShape::Linear, 100, 0), 500);
|
||
}
|
||
|
||
#[test]
|
||
fn constant_backoff_returns_base() {
|
||
for attempt in 1..=5 {
|
||
assert_eq!(
|
||
compute_backoff(attempt, BackoffShape::Constant, 750, 0),
|
||
750
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn jitter_within_pct_of_base() {
|
||
for _ in 0..100 {
|
||
let v = compute_backoff(1, BackoffShape::Constant, 1000, 20);
|
||
// ±20% of 1000 = 800..=1200.
|
||
assert!((800..=1200).contains(&v), "jitter out of range: {v}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn describe_event_http_uses_method_and_path() {
|
||
let payload = serde_json::json!({
|
||
"method": "POST",
|
||
"path": "/webhook",
|
||
"headers": {},
|
||
"body": null,
|
||
"params": {},
|
||
"query": {},
|
||
"rest": "",
|
||
"script_name": "hook",
|
||
"timeout_seconds": 30,
|
||
});
|
||
let (op, source) = describe_event(OutboxSourceKind::Http, &payload);
|
||
assert_eq!(op, "POST /webhook");
|
||
assert_eq!(source, "http");
|
||
// Round-trip via from_wire so the DL replay path resolves to the
|
||
// HTTP branch instead of falling back to Kv.
|
||
assert_eq!(
|
||
OutboxSourceKind::from_wire(&source),
|
||
Some(OutboxSourceKind::Http)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn describe_event_invoke_uses_constant_op() {
|
||
let payload = serde_json::json!({
|
||
"script_id": "00000000-0000-0000-0000-000000000000",
|
||
"args": {},
|
||
});
|
||
let (op, source) = describe_event(OutboxSourceKind::Invoke, &payload);
|
||
assert_eq!(op, "invoke_async");
|
||
assert_eq!(source, "invoke");
|
||
assert_eq!(
|
||
OutboxSourceKind::from_wire(&source),
|
||
Some(OutboxSourceKind::Invoke)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn describe_event_trigger_event_reads_top_level_fields() {
|
||
let payload = serde_json::json!({
|
||
"source": "kv",
|
||
"op": "set",
|
||
"collection": "users",
|
||
"key": "alice",
|
||
"value": {"name": "Alice"},
|
||
});
|
||
let (op, source) = describe_event(OutboxSourceKind::Kv, &payload);
|
||
assert_eq!(op, "set");
|
||
assert_eq!(source, "kv");
|
||
}
|
||
|
||
#[test]
|
||
fn classify_exec_error_covers_every_variant() {
|
||
let parse = classify_exec_error(&ExecError::Parse("nope".into()));
|
||
assert!(matches!(parse.0, InboxFailureKind::Validation));
|
||
let invalid = classify_exec_error(&ExecError::InvalidResponse("bad".into()));
|
||
assert!(matches!(invalid.0, InboxFailureKind::Validation));
|
||
let timeout = classify_exec_error(&ExecError::Timeout(30));
|
||
assert!(matches!(timeout.0, InboxFailureKind::Timeout));
|
||
let budget = classify_exec_error(&ExecError::OperationBudgetExceeded);
|
||
assert!(matches!(budget.0, InboxFailureKind::OperationBudget));
|
||
let runtime = classify_exec_error(&ExecError::Runtime("threw".into()));
|
||
assert!(matches!(runtime.0, InboxFailureKind::Runtime));
|
||
let overload = classify_exec_error(&ExecError::Overloaded {
|
||
retry_after_secs: 1,
|
||
});
|
||
assert!(matches!(overload.0, InboxFailureKind::Overloaded));
|
||
}
|
||
|
||
#[test]
|
||
fn failure_kind_status_codes_match_design_notes() {
|
||
assert_eq!(failure_kind_to_status(InboxFailureKind::Validation), 422);
|
||
assert_eq!(failure_kind_to_status(InboxFailureKind::Runtime), 502);
|
||
assert_eq!(failure_kind_to_status(InboxFailureKind::Overloaded), 503);
|
||
assert_eq!(failure_kind_to_status(InboxFailureKind::Timeout), 504);
|
||
assert_eq!(
|
||
failure_kind_to_status(InboxFailureKind::OperationBudget),
|
||
507
|
||
);
|
||
assert_eq!(failure_kind_to_status(InboxFailureKind::Platform), 500);
|
||
}
|
||
}
|