fix(stage-3): queue DL fan-out + emit-failure visibility
Closes the queue DL fan-out gap (audit Medium) and surfaces the KV/docs/files non-transactional emit gap to operators. - handle_queue_failure previously called queue.dead_letter to write the DL row but never invoked fan_out_dead_letter. The comment claimed "the outbox arm fires registered dead_letter handlers off the new row" — but queue DL rows are written via a separate path that bypasses the outbox entirely, so handlers filtered on source="queue" sat idle forever. Refactor fan_out_dead_letter to take a DeadLetterFanOutCtx struct so both the outbox arm and the queue arm can call it; the queue arm constructs a TriggerEvent::Queue from the claimed message and passes it through. - New integration test queue_dead_letter_fans_out_to_dead_letter_handler registers a dead_letter trigger filtered on "queue", forces queue exhaustion, asserts the handler fires with the correctly-shaped event. - KV/docs/files services committed the data write then ran events.emit as a separate operation, logging-and-swallowing on failure. Bump the six call sites from tracing::warn to tracing::error with an event_emit_failure=true marker so operators can grep them. The full single-tx repo refactor (extending ServiceEventEmitter with emit_in_tx + tx-aware *_repo methods) is documented in kv_service.rs as a v1.2 follow-up — it's a meaningful redesign that deserves its own pass (pubsub_service::fan_out_publish is the reference shape). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -235,6 +235,71 @@ async fn queue_receive_dead_letters_after_max_attempts() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Audit fix: dead_letter triggers filtered on `source = "queue"` MUST
|
||||
/// fire when a queue message exhausts its retries. Before the queue arm
|
||||
/// called `fan_out_dead_letter`, the DL row was written but no handler
|
||||
/// delivery was enqueued, so registered `dead_letter` handlers sat idle.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn queue_dead_letter_fans_out_to_dead_letter_handler() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let (server, app_id) = server_for(pool.clone(), "qdlfan").await;
|
||||
|
||||
// DL handler writes to a different KV key so we don't race the
|
||||
// failing handler's own marker write.
|
||||
let dl_handler_source = r#"
|
||||
kv::collection("e2e_markers").set("dl_marker", ctx.event);
|
||||
#{ ok: true }
|
||||
"#;
|
||||
let dl_handler = create_script(&server, &app_id, "dl-handler", dl_handler_source).await;
|
||||
server
|
||||
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/dead_letter"))
|
||||
.json(&json!({ "script_id": dl_handler, "source_filter": "queue" }))
|
||||
.await
|
||||
.assert_status(axum::http::StatusCode::CREATED);
|
||||
|
||||
// Failing consumer with max_attempts=1 so we exhaust quickly.
|
||||
let failing = create_script(&server, &app_id, "failing", THROW_HANDLER).await;
|
||||
server
|
||||
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/queue"))
|
||||
.json(&json!({
|
||||
"script_id": failing,
|
||||
"queue_name": "failing-fanout",
|
||||
"visibility_timeout_secs": 30,
|
||||
"max_attempts": 1
|
||||
}))
|
||||
.await
|
||||
.assert_status(axum::http::StatusCode::CREATED);
|
||||
|
||||
enqueue_directly(&pool, &app_id, "failing-fanout", json!({ "x": 1 })).await;
|
||||
|
||||
// Poll the DL handler's marker.
|
||||
for _ in 0..200 {
|
||||
let row: Option<(Value,)> = sqlx::query_as(
|
||||
"SELECT value FROM kv_entries WHERE app_id = $1 \
|
||||
AND collection = 'e2e_markers' AND key = 'dl_marker'",
|
||||
)
|
||||
.bind(Uuid::parse_str(&app_id).expect("uuid"))
|
||||
.fetch_optional(&pool)
|
||||
.await
|
||||
.expect("kv read");
|
||||
if let Some((event,)) = row {
|
||||
// Sanity: the dead-letter event names the queue source and
|
||||
// nests the original Queue event verbatim.
|
||||
assert_eq!(event["source"], "dead_letter");
|
||||
assert_eq!(event["dead_letter"]["original"]["source"], "queue");
|
||||
assert_eq!(
|
||||
event["dead_letter"]["original"]["queue"]["queue_name"],
|
||||
"failing-fanout"
|
||||
);
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("dead_letter handler never fired for queue-exhausted message");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn queue_one_consumer_per_queue_rejected() {
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
|
||||
Reference in New Issue
Block a user