- F-T-003 actually closed: clients/typescript/src/subscribe.ts now bounds the 401-refresh loop at 3 consecutive refusals, surfaces an onError describing the loop, and resets on any successful stream open. New test covers the cap. The Stage 2 dashboard fix only addressed adminRequest in dashboard/src/lib/api.ts; the originally- cited TS client file was untouched. - users/invitations subtab dropped the redundant api.apps.get fetch the other 5 subtabs already shed in Stage 6. - Queue visibility-timeout validator pulled out as validate_queue_visibility_timeout, with two thresholds: hard-reject below MIN_QUEUE_VISIBILITY_TIMEOUT_SECS (30s — catches typos), warn- log between MIN and SAFE_VISIBILITY_VS_EXEC_BUDGET_SECS (300s) so operators see when their visibility is below the dispatcher's per-message executor budget. Stage 6 only had the hard floor; the reviewer caught that a 60s handler still races a 30s visibility even after the floor. Four new unit tests cover none/above-safe/ between/below-min. - pic dead-letters count subcommand: cheap headless probe for unresolved DL totals, parallels the dashboard's badge query. - pic dead-letters replay now has a happy-path integration test (replay_against_real_dl_row_succeeds): inserts a synthetic DL row directly via the rust-postgres sync driver (added as dev-dep), drives `pic dead-letters replay`, asserts the row is resolved with reason=replayed and count drops back to 0. Plus a count smoke test. All 75 CLI integration tests + 16 TS client tests + 4 new visibility-timeout unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
225 lines
7.6 KiB
Rust
225 lines
7.6 KiB
Rust
//! `pic dead-letters` smoke tests. The ls + show + resolve round trip
|
|
//! is hard to drive end-to-end without forcing a trigger to exhaust
|
|
//! retries (which the dispatcher tests already cover). Instead, write
|
|
//! a synthetic DL row via the admin API and drive the CLI against it.
|
|
|
|
use predicates::prelude::*;
|
|
use serde_json::json;
|
|
|
|
use crate::common;
|
|
use crate::common::cleanup::AppGuard;
|
|
|
|
#[ignore = "needs DATABASE_URL pointing at a running Postgres"]
|
|
#[test]
|
|
fn ls_empty_app_succeeds() {
|
|
let Some(fx) = common::fixture_or_skip() else {
|
|
return;
|
|
};
|
|
let env = common::admin_env(fx);
|
|
let slug = common::unique_slug("dl-empty");
|
|
common::pic_as(&env)
|
|
.args(["apps", "create", &slug])
|
|
.assert()
|
|
.success();
|
|
let _guard = AppGuard::new(&env.url, &env.token, &slug);
|
|
|
|
let out = common::pic_as(&env)
|
|
.args(["dead-letters", "ls", "--app", &slug])
|
|
.output()
|
|
.expect("dl ls");
|
|
assert!(out.status.success(), "ls failed: {out:?}");
|
|
let stdout = String::from_utf8(out.stdout).unwrap();
|
|
let header = stdout.lines().next().expect("header row");
|
|
assert_eq!(
|
|
common::cells(header),
|
|
vec![
|
|
"id",
|
|
"source",
|
|
"op",
|
|
"attempts",
|
|
"resolved",
|
|
"last_error",
|
|
"created_at"
|
|
]
|
|
);
|
|
// No data rows in a fresh app.
|
|
assert_eq!(stdout.lines().count(), 1, "fresh app has no DL rows");
|
|
}
|
|
|
|
#[ignore = "needs DATABASE_URL pointing at a running Postgres"]
|
|
#[test]
|
|
fn count_returns_unresolved_total() {
|
|
let Some(fx) = common::fixture_or_skip() else {
|
|
return;
|
|
};
|
|
let env = common::admin_env(fx);
|
|
let slug = common::unique_slug("dl-count");
|
|
common::pic_as(&env)
|
|
.args(["apps", "create", &slug])
|
|
.assert()
|
|
.success();
|
|
let _guard = AppGuard::new(&env.url, &env.token, &slug);
|
|
|
|
let out = common::pic_as(&env)
|
|
.args(["dead-letters", "count", "--app", &slug])
|
|
.output()
|
|
.expect("dl count");
|
|
assert!(out.status.success(), "count failed: {out:?}");
|
|
let stdout = String::from_utf8(out.stdout).unwrap();
|
|
let header = stdout.lines().next().expect("header");
|
|
assert_eq!(common::cells(header), vec!["unresolved"]);
|
|
let row = stdout.lines().nth(1).expect("data row");
|
|
assert_eq!(common::cells(row), vec!["0"]);
|
|
}
|
|
|
|
#[ignore = "needs DATABASE_URL pointing at a running Postgres"]
|
|
#[test]
|
|
fn replay_against_real_dl_row_succeeds() {
|
|
use postgres::{Client as PgClient, NoTls};
|
|
use uuid::Uuid;
|
|
|
|
let Some(fx) = common::fixture_or_skip() else {
|
|
return;
|
|
};
|
|
let env = common::admin_env(fx);
|
|
let (script_id_str, guard) = common::deploy_fixture(&env, "dl-replay", "hello.rhai");
|
|
let app_slug = guard.slug().to_string();
|
|
|
|
// Need the app's UUID to inject the DL row. Resolve via the admin
|
|
// GET — picloud accepts either slug or id, returns both.
|
|
let app_id: String = reqwest::blocking::Client::new()
|
|
.get(format!("{}/api/v1/admin/apps/{}", env.url, app_slug))
|
|
.bearer_auth(&env.token)
|
|
.send()
|
|
.expect("apps_get")
|
|
.json::<serde_json::Value>()
|
|
.expect("apps_get json")["id"]
|
|
.as_str()
|
|
.expect("app id")
|
|
.to_string();
|
|
|
|
// Direct INSERT — no admin endpoint creates DL rows; the real ones
|
|
// come from the dispatcher's retry-exhaust path, which would take
|
|
// minutes to drive. The replay path doesn't care how the row got
|
|
// there, only that it exists, belongs to this app, and is
|
|
// unresolved.
|
|
let db_url =
|
|
std::env::var("DATABASE_URL").expect("DATABASE_URL required for replay happy-path test");
|
|
let mut pg = PgClient::connect(&db_url, NoTls).expect("postgres connect");
|
|
let dl_id = Uuid::new_v4();
|
|
let app_uuid = Uuid::parse_str(&app_id).expect("app uuid");
|
|
let script_uuid = Uuid::parse_str(&script_id_str).expect("script uuid");
|
|
// Payload shape must round-trip through the dispatcher: source="kv"
|
|
// matches a KV TriggerEvent, op="set" is a KvEventOp, and the
|
|
// service_kind=`kv` outbox row gets routed by resolve_trigger.
|
|
let payload = serde_json::json!({
|
|
"source": "kv",
|
|
"op": "set",
|
|
"kv": {
|
|
"collection": "x",
|
|
"key": "y",
|
|
"value": { "v": 1 }
|
|
}
|
|
});
|
|
pg.execute(
|
|
"INSERT INTO dead_letters (
|
|
id, app_id, original_event_id, source, op, trigger_id, script_id,
|
|
payload, attempt_count, first_attempt_at, last_attempt_at, last_error
|
|
) VALUES ($1, $2, $3, 'kv', 'set', NULL, $4, $5, 1, NOW(), NOW(), 'forced')",
|
|
&[&dl_id, &app_uuid, &Uuid::new_v4(), &script_uuid, &payload],
|
|
)
|
|
.expect("insert synthetic DL row");
|
|
|
|
// Verify CLI count picks the row up.
|
|
let out = common::pic_as(&env)
|
|
.args(["dead-letters", "count", "--app", &app_slug])
|
|
.output()
|
|
.expect("dl count");
|
|
let stdout = String::from_utf8(out.stdout).unwrap();
|
|
assert!(
|
|
stdout.lines().nth(1) == Some("1"),
|
|
"count should be 1: {stdout}"
|
|
);
|
|
|
|
// Replay — pic exits 0, the server resolves the row with reason=replayed.
|
|
common::pic_as(&env)
|
|
.args([
|
|
"dead-letters",
|
|
"replay",
|
|
"--app",
|
|
&app_slug,
|
|
&dl_id.to_string(),
|
|
])
|
|
.assert()
|
|
.success()
|
|
.stdout(predicate::str::contains("Replayed dead-letter"));
|
|
|
|
// Verify the row is now resolved with reason=replayed.
|
|
let row = pg
|
|
.query_one(
|
|
"SELECT resolution FROM dead_letters WHERE id = $1",
|
|
&[&dl_id],
|
|
)
|
|
.expect("dl row exists post-replay");
|
|
let resolution: Option<String> = row.get(0);
|
|
assert_eq!(resolution.as_deref(), Some("replayed"));
|
|
|
|
// And the unresolved count drops back to 0.
|
|
let out = common::pic_as(&env)
|
|
.args(["dead-letters", "count", "--app", &app_slug])
|
|
.output()
|
|
.expect("dl count");
|
|
let stdout = String::from_utf8(out.stdout).unwrap();
|
|
assert!(
|
|
stdout.lines().nth(1) == Some("0"),
|
|
"count should be 0 after replay: {stdout}"
|
|
);
|
|
}
|
|
|
|
#[ignore = "needs DATABASE_URL pointing at a running Postgres"]
|
|
#[test]
|
|
fn resolve_marks_row_resolved() {
|
|
let Some(fx) = common::fixture_or_skip() else {
|
|
return;
|
|
};
|
|
let env = common::admin_env(fx);
|
|
let (script_id, guard) = common::deploy_fixture(&env, "dl-resolve", "hello.rhai");
|
|
let app = guard.slug().to_string();
|
|
|
|
// Synth a DL row directly through the existing admin path —
|
|
// there's no test-only insert endpoint, so we use the resolve
|
|
// API on an arbitrary id and assert 404 (the path resolves cleanly
|
|
// but the row doesn't exist). The intent here is to assert the
|
|
// CLI plumbs the URL correctly; force-exhausting retries from a
|
|
// CLI test is too slow.
|
|
let _ = script_id; // suppress unused warning
|
|
|
|
let client = reqwest::blocking::Client::new();
|
|
let resp = client
|
|
.post(format!(
|
|
"{}/api/v1/admin/apps/{}/dead_letters/{}/resolve",
|
|
env.url, app, "00000000-0000-0000-0000-000000000000"
|
|
))
|
|
.bearer_auth(&env.token)
|
|
.json(&json!({ "reason": "test" }))
|
|
.send()
|
|
.expect("resolve");
|
|
assert_eq!(resp.status().as_u16(), 404);
|
|
|
|
// The CLI mirrors that 404 — verify the wire round-trips through
|
|
// pic without mangling the path.
|
|
common::pic_as(&env)
|
|
.args([
|
|
"dead-letters",
|
|
"resolve",
|
|
"--app",
|
|
&app,
|
|
"00000000-0000-0000-0000-000000000000",
|
|
"--reason",
|
|
"test",
|
|
])
|
|
.assert()
|
|
.failure()
|
|
.stderr(predicate::str::contains("HTTP 404"));
|
|
}
|