fix(executor): cap per-execution durable fan-out width
`trigger_depth` bounds how DEEP a trigger/invoke chain runs, but nothing bounded
how WIDE one execution could fan out. A single anonymous request running
`for i in 0..1_000_000 { invoke_async("w", #{}) }` costs a few Rhai ops plus one
cheap outbox INSERT per iteration, so within the op / wall-clock budget it could
write ~10^5 durable rows — each dispatched as its own execution — flooding the
outbox and dispatcher (a durable-amplification DoS).
Add a per-execution ceiling on DURABLE emissions (`invoke_async`,
`pubsub::publish_durable`, `queue::enqueue`, and the shared-topic/-queue
variants), env `PICLOUD_MAX_EMISSIONS_PER_EXECUTION` (default 1000). It's a
thread-local counter wrapped by a re-entrancy-aware `EmissionBudgetScope` around
every `execute_ast`: the OUTERMOST scope resets it, so a fresh dispatched
handler (a new pooled-thread task) gets a full budget while a synchronous
`invoke()` / interceptor re-entry nested in the same call stack SHARES it (fan-
out counted across the whole synchronous chain). The outermost Drop re-zeroes
the counter so a pooled thread never leaks a count into the next task.
Pinned by an `emit_budget` unit test (outermost resets, nested shares, ceiling
trips); the invoke/queue/pubsub/workflow journeys (small counts) stay green.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -121,6 +121,7 @@ Environment variables consumed by the `picloud` binary:
|
||||
|---|---|---|
|
||||
| `PICLOUD_BIND` | `0.0.0.0:8080` | HTTP listen address. Port 8080 is owned by another process on this host — override locally. |
|
||||
| `PICLOUD_MAX_CONCURRENT_EXECUTIONS` | `32` | Global concurrency cap on data-plane script executions. Overflow returns HTTP 503 with `Retry-After: 1` immediately (no queue). |
|
||||
| `PICLOUD_MAX_EMISSIONS_PER_EXECUTION` | `1000` | Per-execution fan-out ceiling: max DURABLE emissions (`invoke_async` + `pubsub::publish_durable` + `queue::enqueue`, incl. shared variants) one execution may make before the SDK call errors. Bounds a one-request outbox-amplification DoS. Re-entrancy aware — a synchronous `invoke()`/interceptor chain shares one budget; a dispatched handler starts fresh. `trigger_depth` bounds chain DEPTH, this bounds fan-out WIDTH. |
|
||||
| `DATABASE_URL` | — | Required. Postgres connection string. |
|
||||
| `PICLOUD_SECRET_KEY` | — | Master encryption key (base64). Required at startup unless dev mode is acknowledged (below). |
|
||||
| `PICLOUD_DEV_MODE` | `false` | `true` enables local-dev conveniences. Without `PICLOUD_SECRET_KEY` it ALSO requires the acknowledgement var below — `PICLOUD_DEV_MODE=true` alone aborts at startup. Also: when no SMTP relay is configured, `email::send` switches from disabled (`NotConfigured`) to an **in-memory dev sink** — sends succeed and the last 100 messages are readable at `GET /api/v1/admin/dev/emails` (instance Owner/Admin only; route exists only in this mode). Never in production. |
|
||||
|
||||
@@ -280,6 +280,13 @@ impl Engine {
|
||||
/// cache hands compiled ASTs in directly; this path skips the
|
||||
/// per-call compile.
|
||||
pub fn execute_ast(&self, ast: &Arc<AST>, req: ExecRequest) -> Result<ExecResponse, ExecError> {
|
||||
// Audit #2: bound per-execution durable fan-out. The scope is
|
||||
// re-entrancy aware — a fresh dispatched execution (a new task on a
|
||||
// pooled thread) resets the budget, while a synchronous invoke() /
|
||||
// interceptor re-entry nested in the SAME call stack shares it, so
|
||||
// fan-out is counted across the whole synchronous chain. Held to the end
|
||||
// of the call; its Drop re-zeroes the counter on the outermost exit.
|
||||
let _emit_scope = crate::sdk::emit_budget::EmissionBudgetScope::enter();
|
||||
let effective_limits = self.limits.with_overrides(&req.sandbox_overrides);
|
||||
let logs: Arc<Mutex<Vec<LogEntry>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut engine = build_engine(effective_limits, Some(logs.clone()));
|
||||
|
||||
124
crates/executor-core/src/sdk/emit_budget.rs
Normal file
124
crates/executor-core/src/sdk/emit_budget.rs
Normal file
@@ -0,0 +1,124 @@
|
||||
//! Per-execution fan-out ceiling (audit fix #2).
|
||||
//!
|
||||
//! `trigger_depth` bounds how DEEP a trigger/invoke chain can go, but nothing
|
||||
//! bounded how WIDE a single execution could fan out: one anonymous request
|
||||
//! running `for i in 0..1_000_000 { invoke_async("w", #{}) }` costs a few Rhai
|
||||
//! ops per iteration plus one cheap outbox INSERT, so within the op / wall-clock
|
||||
//! budget it could write ~10^5 durable rows — each dispatched as its own
|
||||
//! execution — flooding the outbox/queue (a durable amplification DoS).
|
||||
//!
|
||||
//! This caps the number of DURABLE emissions (`invoke_async`, `publish_durable`,
|
||||
//! `enqueue`, and their group-shared variants) a single execution may make.
|
||||
//! The counter is a thread-local. [`EmissionBudgetScope`] is created around
|
||||
//! EVERY `execute_ast` and is **re-entrancy aware**: only the OUTERMOST scope
|
||||
//! on a thread resets the counter. A dispatched handler (queue/trigger/
|
||||
//! `invoke_async`/cron/workflow) is a fresh task on a pooled thread, so its
|
||||
//! scope is outermost → it resets and gets a full budget. A synchronous
|
||||
//! `invoke()` / interceptor re-entry runs a nested `execute_ast` in the SAME
|
||||
//! call stack, so its scope is inner → it does NOT reset, and the whole
|
||||
//! synchronous chain shares one budget (fan-out counted across it). The
|
||||
//! outermost Drop re-zeroes the counter so a pooled thread never leaks a count
|
||||
//! into the next task.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use rhai::EvalAltResult;
|
||||
|
||||
use crate::sdk::bridge::runtime_err;
|
||||
|
||||
/// Default max durable emissions per execution. Generous for a legit batch
|
||||
/// producer, far below the ~10^5 a runaway loop can reach within budget.
|
||||
const DEFAULT_MAX_EMISSIONS: u32 = 1000;
|
||||
|
||||
static MAX_EMISSIONS: LazyLock<u32> = LazyLock::new(|| {
|
||||
std::env::var("PICLOUD_MAX_EMISSIONS_PER_EXECUTION")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u32>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.unwrap_or(DEFAULT_MAX_EMISSIONS)
|
||||
});
|
||||
|
||||
thread_local! {
|
||||
static EMISSIONS: Cell<u32> = const { Cell::new(0) };
|
||||
/// Scope nesting depth on this thread. `0` when no execution is active.
|
||||
static ACTIVE: Cell<u32> = const { Cell::new(0) };
|
||||
}
|
||||
|
||||
/// Charge one durable emission against the current execution's budget. Returns
|
||||
/// an `Err` (aborting the emit) once the ceiling is exceeded.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a Rhai runtime error when the per-execution emission ceiling is hit.
|
||||
pub(super) fn charge_emission(service: &str) -> Result<(), Box<EvalAltResult>> {
|
||||
EMISSIONS.with(|c| {
|
||||
let next = c.get().saturating_add(1);
|
||||
if next > *MAX_EMISSIONS {
|
||||
return Err(runtime_err(&format!(
|
||||
"{service}: per-execution emission limit exceeded (max {} durable \
|
||||
invoke_async/publish/enqueue calls per execution)",
|
||||
*MAX_EMISSIONS
|
||||
)));
|
||||
}
|
||||
c.set(next);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// RAII scope wrapping one `execute_ast`. Re-entrancy aware: the OUTERMOST scope
|
||||
/// on a thread zeroes the counter on entry and on final exit; nested scopes (a
|
||||
/// synchronous re-entry in the same call stack) leave it alone, so the chain
|
||||
/// shares one budget.
|
||||
pub(crate) struct EmissionBudgetScope;
|
||||
|
||||
impl EmissionBudgetScope {
|
||||
pub(crate) fn enter() -> Self {
|
||||
ACTIVE.with(|a| {
|
||||
if a.get() == 0 {
|
||||
EMISSIONS.with(|c| c.set(0));
|
||||
}
|
||||
a.set(a.get() + 1);
|
||||
});
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EmissionBudgetScope {
|
||||
fn drop(&mut self) {
|
||||
ACTIVE.with(|a| {
|
||||
let n = a.get().saturating_sub(1);
|
||||
a.set(n);
|
||||
if n == 0 {
|
||||
EMISSIONS.with(|c| c.set(0));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn outermost_scope_resets_but_nested_shares_budget() {
|
||||
let outer = EmissionBudgetScope::enter();
|
||||
for _ in 0..DEFAULT_MAX_EMISSIONS {
|
||||
charge_emission("test").expect("under budget");
|
||||
}
|
||||
// A NESTED scope (synchronous re-entry) must NOT reset — the budget is
|
||||
// already exhausted and stays that way.
|
||||
{
|
||||
let _nested = EmissionBudgetScope::enter();
|
||||
assert!(
|
||||
charge_emission("test").is_err(),
|
||||
"a nested re-entry shares the exhausted budget"
|
||||
);
|
||||
}
|
||||
// Still exhausted after the nested scope drops (outer still active).
|
||||
assert!(charge_emission("test").is_err());
|
||||
drop(outer);
|
||||
// A fresh OUTERMOST scope resets.
|
||||
let _fresh = EmissionBudgetScope::enter();
|
||||
charge_emission("test").expect("reset by a fresh outermost scope");
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,9 @@ pub(super) fn register(
|
||||
module.set_native_fn(
|
||||
"invoke_async",
|
||||
move |target: Dynamic, args: Dynamic| -> Result<String, Box<EvalAltResult>> {
|
||||
// Audit #2: count this durable emission against the per-execution
|
||||
// fan-out ceiling before doing any work.
|
||||
crate::sdk::emit_budget::charge_emission("invoke_async")?;
|
||||
let target = parse_target(target)?;
|
||||
let args_json = args_to_json(&args)?;
|
||||
let svc = svc.clone();
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod cx;
|
||||
pub mod dead_letters;
|
||||
pub mod docs;
|
||||
pub mod email;
|
||||
pub mod emit_budget;
|
||||
pub mod files;
|
||||
pub mod http;
|
||||
pub mod interceptor;
|
||||
|
||||
@@ -35,6 +35,7 @@ pub(super) fn register(engine: &mut RhaiEngine, services: &Services, cx: Arc<Sdk
|
||||
module.set_native_fn(
|
||||
"publish_durable",
|
||||
move |topic: &str, message: Dynamic| -> Result<(), Box<EvalAltResult>> {
|
||||
crate::sdk::emit_budget::charge_emission("pubsub::publish_durable")?;
|
||||
let json = message_to_json(&message)?;
|
||||
let svc = svc.clone();
|
||||
let cx = cx.clone();
|
||||
@@ -110,6 +111,7 @@ fn shared_publish(
|
||||
subtopic: &str,
|
||||
message: Dynamic,
|
||||
) -> Result<(), Box<EvalAltResult>> {
|
||||
crate::sdk::emit_budget::charge_emission("pubsub::shared_topic")?;
|
||||
let json = message_to_json(&message)?;
|
||||
let service = h.service.clone();
|
||||
let cx = h.cx.clone();
|
||||
|
||||
@@ -40,6 +40,7 @@ pub(super) fn register(engine: &mut RhaiEngine, services: &Services, cx: Arc<Sdk
|
||||
module.set_native_fn(
|
||||
"enqueue",
|
||||
move |name: &str, message: Dynamic| -> Result<(), Box<EvalAltResult>> {
|
||||
crate::sdk::emit_budget::charge_emission("queue::enqueue")?;
|
||||
let json = message_to_json(&message)?;
|
||||
enqueue_blocking(&svc, &cx, name, json, EnqueueOpts::default())
|
||||
},
|
||||
@@ -54,6 +55,7 @@ pub(super) fn register(engine: &mut RhaiEngine, services: &Services, cx: Arc<Sdk
|
||||
module.set_native_fn(
|
||||
"enqueue",
|
||||
move |name: &str, message: Dynamic, opts: Map| -> Result<(), Box<EvalAltResult>> {
|
||||
crate::sdk::emit_budget::charge_emission("queue::enqueue")?;
|
||||
let json = message_to_json(&message)?;
|
||||
let opts = parse_opts(&opts)?;
|
||||
enqueue_blocking(&svc, &cx, name, json, opts)
|
||||
@@ -131,6 +133,7 @@ fn group_enqueue(
|
||||
message: Dynamic,
|
||||
opts: EnqueueOpts,
|
||||
) -> Result<(), Box<EvalAltResult>> {
|
||||
crate::sdk::emit_budget::charge_emission("queue::shared_collection")?;
|
||||
let json = message_to_json(&message)?;
|
||||
let handle = TokioHandle::try_current().map_err(|e| -> Box<EvalAltResult> {
|
||||
EvalAltResult::ErrorRuntime(
|
||||
|
||||
Reference in New Issue
Block a user