feat(queue): dead-letter store for group shared queues (§11.6 D3 / Track A M2)
An exhausted SHARED durable-queue message was `drop_exhausted()`-ed with a
warning — silent data loss. Per-app queues persist to `dead_letters` (0010);
add the symmetric group store so an exhausted shared-queue message is preserved
and operator-visible.
- Migration 0068: `group_dead_letters`, keyed by (group_id, collection),
CASCADE on the group (an app delete leaves the data — it belongs to the
group, not the consuming app).
- `GroupQueueRepo::dead_letter` (replaces `drop_exhausted`): one tx that INSERTs
the dead-letter + DELETEs the live message, filtered by claim_token so a lost
lease can't dead-letter a re-claimed message (mirrors queue_repo::dead_letter).
- Dispatcher `q_terminal` shared arm now dead-letters instead of dropping. It
returns None (not the dl id) so the per-app `fan_out_dead_letter` is SKIPPED —
firing the consuming app's per-app handlers on a shared message (competing
consumers → nondeterministic app) would be wrong. Fan-out to a *shared*
dead_letter trigger is deferred (needs a new trigger kind).
- Read side: `GroupDeadLetterRepo::list_for_group` backs a new read-only
operator endpoint GET /api/v1/admin/groups/{id}/dead-letters (GroupKvRead,
mirrors the M4 group-blobs surface). `pic dead-letters ls --group` deferred
(optional; the HTTP endpoint is the operator surface).
Pinned by group_queue::dead_letter_moves_an_exhausted_message_to_the_group_store
(store move + claim-token-mismatch guard + operator read). Schema golden
reblessed.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,9 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use picloud_manager_core::group_dead_letter_repo::{
|
||||
GroupDeadLetterRepo, PostgresGroupDeadLetterRepo,
|
||||
};
|
||||
use picloud_manager_core::group_queue_repo::{
|
||||
GroupQueueRepo, NewGroupQueueMessage, PostgresGroupQueueRepo,
|
||||
};
|
||||
@@ -134,3 +137,122 @@ async fn competing_consumers_claim_each_message_exactly_once() {
|
||||
.execute(&pool)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dead_letter_moves_an_exhausted_message_to_the_group_store() {
|
||||
// §11.6 D3: an exhausted shared-queue message is MOVED to the group dead-
|
||||
// letter store (not dropped) — the row leaves the live queue and appears in
|
||||
// `group_dead_letters`, operator-visible via GroupDeadLetterRepo.
|
||||
let Some(pool) = pool_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
let sfx = Uuid::new_v4().simple().to_string();
|
||||
let g: (Uuid,) = sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
|
||||
.bind(format!("gdl-g-{sfx}"))
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let group = GroupId::from(g.0);
|
||||
let queue = PostgresGroupQueueRepo::new(pool.clone());
|
||||
let dlq = PostgresGroupDeadLetterRepo::new(pool.clone());
|
||||
|
||||
let id = queue
|
||||
.enqueue(NewGroupQueueMessage {
|
||||
group_id: group,
|
||||
collection: "jobs".into(),
|
||||
payload: serde_json::json!({ "task": "boom" }),
|
||||
deliver_after: None,
|
||||
max_attempts: 1,
|
||||
enqueued_by_principal: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let msg = queue.claim(group, "jobs").await.unwrap().expect("claimed");
|
||||
assert_eq!(msg.id, id);
|
||||
|
||||
// Exhausted → dead-letter (mirrors the dispatcher's q_terminal shared arm).
|
||||
let dl_id = queue
|
||||
.dead_letter(
|
||||
msg.id,
|
||||
msg.claim_token,
|
||||
group,
|
||||
"jobs",
|
||||
None,
|
||||
None,
|
||||
msg.attempt,
|
||||
msg.enqueued_at,
|
||||
"handler exploded",
|
||||
)
|
||||
.await
|
||||
.expect("dead_letter");
|
||||
|
||||
// The live queue no longer holds it.
|
||||
assert_eq!(
|
||||
queue.depth(group, "jobs").await.unwrap(),
|
||||
0,
|
||||
"the dead-lettered message left the live queue"
|
||||
);
|
||||
|
||||
// It is preserved + operator-visible in the group dead-letter store.
|
||||
let rows = dlq.list_for_group(group, false, 100).await.unwrap();
|
||||
assert_eq!(rows.len(), 1, "exactly one dead-letter recorded");
|
||||
let row = &rows[0];
|
||||
assert_eq!(row.id.into_inner(), dl_id.into_inner());
|
||||
assert_eq!(row.collection, "jobs");
|
||||
assert_eq!(row.source, "queue");
|
||||
assert_eq!(row.last_error, "handler exploded");
|
||||
assert_eq!(row.payload["message"]["task"], "boom");
|
||||
assert!(
|
||||
row.resolved_at.is_none(),
|
||||
"a fresh dead-letter is unresolved"
|
||||
);
|
||||
|
||||
// A claim_token mismatch cannot dead-letter a re-claimed message: enqueue
|
||||
// another, claim it, then try to DL with a bogus token → the source-row
|
||||
// fetch finds nothing (RowNotFound), leaving the live queue intact.
|
||||
let id2 = queue
|
||||
.enqueue(NewGroupQueueMessage {
|
||||
group_id: group,
|
||||
collection: "jobs".into(),
|
||||
payload: serde_json::json!({ "task": "keep" }),
|
||||
deliver_after: None,
|
||||
max_attempts: 1,
|
||||
enqueued_by_principal: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let m2 = queue
|
||||
.claim(group, "jobs")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("claimed 2");
|
||||
let bogus = Uuid::new_v4();
|
||||
assert!(
|
||||
queue
|
||||
.dead_letter(
|
||||
m2.id,
|
||||
bogus,
|
||||
group,
|
||||
"jobs",
|
||||
None,
|
||||
None,
|
||||
1,
|
||||
m2.enqueued_at,
|
||||
"x"
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"a claim-token mismatch must not dead-letter the message"
|
||||
);
|
||||
assert_eq!(
|
||||
queue.depth(group, "jobs").await.unwrap(),
|
||||
1,
|
||||
"the mismatched message is still live"
|
||||
);
|
||||
let _ = id2;
|
||||
|
||||
let _ = sqlx::query("DELETE FROM groups WHERE id = $1")
|
||||
.bind(g.0)
|
||||
.execute(&pool)
|
||||
.await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user