fix(stage-3): queue DL fan-out + emit-failure visibility

Closes the queue DL fan-out gap (audit Medium) and surfaces the
KV/docs/files non-transactional emit gap to operators.

- handle_queue_failure previously called queue.dead_letter to write the
  DL row but never invoked fan_out_dead_letter. The comment claimed
  "the outbox arm fires registered dead_letter handlers off the new row"
  — but queue DL rows are written via a separate path that bypasses the
  outbox entirely, so handlers filtered on source="queue" sat idle
  forever. Refactor fan_out_dead_letter to take a DeadLetterFanOutCtx
  struct so both the outbox arm and the queue arm can call it; the
  queue arm constructs a TriggerEvent::Queue from the claimed message
  and passes it through.

- New integration test queue_dead_letter_fans_out_to_dead_letter_handler
  registers a dead_letter trigger filtered on "queue", forces queue
  exhaustion, asserts the handler fires with the correctly-shaped event.

- KV/docs/files services committed the data write then ran events.emit
  as a separate operation, logging-and-swallowing on failure. Bump the
  six call sites from tracing::warn to tracing::error with an
  event_emit_failure=true marker so operators can grep them. The full
  single-tx repo refactor (extending ServiceEventEmitter with
  emit_in_tx + tx-aware *_repo methods) is documented in kv_service.rs
  as a v1.2 follow-up — it's a meaningful redesign that deserves its
  own pass (pubsub_service::fan_out_publish is the reference shape).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-10 21:22:15 +02:00
parent f466b2a15e
commit 649246213e
5 changed files with 212 additions and 47 deletions

View File

@@ -235,6 +235,71 @@ async fn queue_receive_dead_letters_after_max_attempts() {
);
}
/// Audit fix: dead_letter triggers filtered on `source = "queue"` MUST
/// fire when a queue message exhausts its retries. Before the queue arm
/// called `fan_out_dead_letter`, the DL row was written but no handler
/// delivery was enqueued, so registered `dead_letter` handlers sat idle.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn queue_dead_letter_fans_out_to_dead_letter_handler() {
let Some(pool) = pool_or_skip().await else {
return;
};
let (server, app_id) = server_for(pool.clone(), "qdlfan").await;
// DL handler writes to a different KV key so we don't race the
// failing handler's own marker write.
let dl_handler_source = r#"
kv::collection("e2e_markers").set("dl_marker", ctx.event);
#{ ok: true }
"#;
let dl_handler = create_script(&server, &app_id, "dl-handler", dl_handler_source).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/dead_letter"))
.json(&json!({ "script_id": dl_handler, "source_filter": "queue" }))
.await
.assert_status(axum::http::StatusCode::CREATED);
// Failing consumer with max_attempts=1 so we exhaust quickly.
let failing = create_script(&server, &app_id, "failing", THROW_HANDLER).await;
server
.post(&format!("/api/v1/admin/apps/{app_id}/triggers/queue"))
.json(&json!({
"script_id": failing,
"queue_name": "failing-fanout",
"visibility_timeout_secs": 30,
"max_attempts": 1
}))
.await
.assert_status(axum::http::StatusCode::CREATED);
enqueue_directly(&pool, &app_id, "failing-fanout", json!({ "x": 1 })).await;
// Poll the DL handler's marker.
for _ in 0..200 {
let row: Option<(Value,)> = sqlx::query_as(
"SELECT value FROM kv_entries WHERE app_id = $1 \
AND collection = 'e2e_markers' AND key = 'dl_marker'",
)
.bind(Uuid::parse_str(&app_id).expect("uuid"))
.fetch_optional(&pool)
.await
.expect("kv read");
if let Some((event,)) = row {
// Sanity: the dead-letter event names the queue source and
// nests the original Queue event verbatim.
assert_eq!(event["source"], "dead_letter");
assert_eq!(event["dead_letter"]["original"]["source"], "queue");
assert_eq!(
event["dead_letter"]["original"]["queue"]["queue_name"],
"failing-fanout"
);
return;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("dead_letter handler never fired for queue-exhausted message");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn queue_one_consumer_per_queue_rejected() {
let Some(pool) = pool_or_skip().await else {