executor-core/sdk/retry.rs adds the retry:: namespace with:
retry::policy(opts) -> Policy
retry::on_codes(policy, codes) -> Policy
retry::run(policy, closure) -> Dynamic (closure's return value)
NOTE: the brief specified retry::with(...), but `with` is a Rhai
reserved keyword (parse error before registration even matters). `call`
is also reserved. Shipping retry::run instead — flagged in HANDBACK §7.
Policy:
- max_attempts clamped to [1, 20]
- base_ms clamped to [1, 60_000]
- jitter_pct clamped to [0, 100]
- backoff ∈ {"exponential" | "linear" | "constant"} — bogus values
surface a clear error
- on_codes is an empty default ("retry any throw"); when non-empty,
only error messages containing one of the codes retry — others
surface immediately
retry::run uses NativeCallContext + FnPtr::call_within_context to
re-invoke the closure inside the caller's engine. Closures share the
caller's cx (Rhai semantics). If the closure calls invoke(), the inner
call gets a fresh cx with trigger_depth + 1 via the invoke bridge —
retry doesn't see or interfere with that.
Sleep between attempts: tokio::time::sleep through TokioHandle::block_on.
Safe because we're already inside the caller's spawn_blocking thread.
register_all gains retry::register positionally between queue and
secrets.
Unit tests (7): policy default, max_attempts clamping (0→1, 999→20),
base_ms + jitter clamping, bogus backoff reject, exponential doubles,
linear sums, constant.
Integration tests (sdk_retry.rs, 6 binaries):
- succeeds first try (returns 42)
- max_attempts exhausted surfaces last error ("boom")
- on_codes filters — non-matching error throws immediately
- jitter clamping is silent (script still runs)
- bogus backoff string surfaces a clear error
- "fails 2 then succeeds" — counter mutates across retries, 3rd
attempt returns 3 (validates Rhai closure-capture semantics across
re-invocations through NativeCallContext)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
172 lines
5.8 KiB
Rust
172 lines
5.8 KiB
Rust
//! `retry::*` SDK bridge integration tests — runs real Rhai scripts
|
|
//! that build a Policy, wrap a closure, and verify the retry / on_codes
|
|
//! behavior end-to-end. Sleep delays are kept to 1ms so the suite runs
|
|
//! fast.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use picloud_executor_core::{Engine, ExecRequest, InvocationType, Limits};
|
|
use picloud_shared::{
|
|
AppId, ExecutionId, NoopDeadLetterService, NoopDocsService, NoopEmailService,
|
|
NoopEventEmitter, NoopFilesService, NoopHttpService, NoopInvokeService, NoopKvService,
|
|
NoopModuleSource, NoopPubsubService, NoopQueueService, NoopSecretsService, NoopUsersService,
|
|
RequestId, ScriptId, ScriptSandbox, Services,
|
|
};
|
|
use serde_json::Value;
|
|
|
|
fn build_engine() -> Arc<Engine> {
|
|
let services = Services::new(
|
|
Arc::new(NoopKvService),
|
|
Arc::new(NoopDocsService),
|
|
Arc::new(NoopDeadLetterService),
|
|
Arc::new(NoopEventEmitter),
|
|
Arc::new(NoopModuleSource),
|
|
Arc::new(NoopHttpService),
|
|
Arc::new(NoopFilesService),
|
|
Arc::new(NoopPubsubService),
|
|
Arc::new(NoopSecretsService),
|
|
Arc::new(NoopEmailService),
|
|
Arc::new(NoopUsersService),
|
|
Arc::new(NoopQueueService),
|
|
Arc::new(NoopInvokeService),
|
|
);
|
|
Arc::new(Engine::new(Limits::default(), services))
|
|
}
|
|
|
|
fn baseline_request(app_id: AppId) -> ExecRequest {
|
|
let execution_id = ExecutionId::new();
|
|
ExecRequest {
|
|
execution_id,
|
|
request_id: RequestId::new(),
|
|
script_id: ScriptId::new(),
|
|
script_name: "retry-test".into(),
|
|
invocation_type: InvocationType::Http,
|
|
path: "/retry-test".into(),
|
|
headers: BTreeMap::new(),
|
|
body: Value::Null,
|
|
params: BTreeMap::new(),
|
|
query: BTreeMap::new(),
|
|
rest: String::new(),
|
|
sandbox_overrides: ScriptSandbox::default(),
|
|
app_id,
|
|
principal: None,
|
|
trigger_depth: 0,
|
|
root_execution_id: execution_id,
|
|
is_dead_letter_handler: false,
|
|
event: None,
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_run_succeeds_first_try() {
|
|
let engine = build_engine();
|
|
let src = r#"
|
|
let p = retry::policy(#{ max_attempts: 3, base_ms: 1 });
|
|
let v = retry::run(p, || 42);
|
|
#{ statusCode: 200, body: v }
|
|
"#
|
|
.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let resp = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(resp.body, serde_json::json!(42));
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_run_surfaces_after_max_attempts() {
|
|
let engine = build_engine();
|
|
let src = r#"
|
|
let p = retry::policy(#{ max_attempts: 3, base_ms: 1, jitter_pct: 0 });
|
|
retry::run(p, || { throw "boom" });
|
|
"#
|
|
.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let res = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap();
|
|
let err = res.unwrap_err().to_string();
|
|
assert!(err.contains("boom"), "expected 'boom' in error, got: {err}");
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_on_codes_filters_unmatched_errors() {
|
|
let engine = build_engine();
|
|
// Throw a message NOT in the filter; retry::run must surface
|
|
// immediately (no retries).
|
|
let src = r#"
|
|
let p = retry::policy(#{ max_attempts: 5, base_ms: 1, jitter_pct: 0 });
|
|
let p2 = retry::on_codes(p, ["http: 503"]);
|
|
retry::run(p2, || { throw "other failure" });
|
|
"#
|
|
.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let res = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap();
|
|
let err = res.unwrap_err().to_string();
|
|
assert!(
|
|
err.contains("other failure"),
|
|
"expected immediate surface, got: {err}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_policy_clamps_visible_at_script_layer() {
|
|
// jitter_pct = 999 should clamp to 100 silently; the policy is then
|
|
// usable.
|
|
let engine = build_engine();
|
|
let src = r#"
|
|
let p = retry::policy(#{
|
|
max_attempts: 30, base_ms: 100, jitter_pct: 999, backoff: "constant"
|
|
});
|
|
let v = retry::run(p, || 1);
|
|
#{ statusCode: 200, body: v }
|
|
"#
|
|
.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let resp = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(resp.body, serde_json::json!(1));
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_policy_rejects_bogus_backoff() {
|
|
let engine = build_engine();
|
|
let src = r#"retry::policy(#{ backoff: "bogus" });"#.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let res = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap();
|
|
assert!(res.is_err());
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn retry_run_eventually_succeeds() {
|
|
// Use an in-engine counter to simulate "fails 2 times then succeeds".
|
|
let engine = build_engine();
|
|
let counter = Arc::new(Mutex::new(0_i64));
|
|
let _ = counter; // counter via Rhai-side `let` instead
|
|
|
|
let src = r#"
|
|
let attempts = 0;
|
|
let p = retry::policy(#{ max_attempts: 5, base_ms: 1, jitter_pct: 0 });
|
|
let v = retry::run(p, || {
|
|
attempts += 1;
|
|
if attempts < 3 { throw "transient" } else { attempts }
|
|
});
|
|
#{ statusCode: 200, body: v }
|
|
"#
|
|
.to_string();
|
|
let req = baseline_request(AppId::new());
|
|
let resp = tokio::task::spawn_blocking(move || engine.execute(&src, req))
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(resp.body, serde_json::json!(3));
|
|
}
|