fabi/PiCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(queue): dead-letter store for group shared queues (§11.6 D3 / Track A M2)

Browse Source 
An exhausted SHARED durable-queue message was `drop_exhausted()`-ed with a
warning — silent data loss. Per-app queues persist to `dead_letters` (0010);
add the symmetric group store so an exhausted shared-queue message is preserved
and operator-visible.

- Migration 0068: `group_dead_letters`, keyed by (group_id, collection),
  CASCADE on the group (an app delete leaves the data — it belongs to the
  group, not the consuming app).
- `GroupQueueRepo::dead_letter` (replaces `drop_exhausted`): one tx that INSERTs
  the dead-letter + DELETEs the live message, filtered by claim_token so a lost
  lease can't dead-letter a re-claimed message (mirrors queue_repo::dead_letter).
- Dispatcher `q_terminal` shared arm now dead-letters instead of dropping. It
  returns None (not the dl id) so the per-app `fan_out_dead_letter` is SKIPPED —
  firing the consuming app's per-app handlers on a shared message (competing
  consumers → nondeterministic app) would be wrong. Fan-out to a *shared*
  dead_letter trigger is deferred (needs a new trigger kind).
- Read side: `GroupDeadLetterRepo::list_for_group` backs a new read-only
  operator endpoint GET /api/v1/admin/groups/{id}/dead-letters (GroupKvRead,
  mirrors the M4 group-blobs surface). `pic dead-letters ls --group` deferred
  (optional; the HTTP endpoint is the operator surface).

Pinned by group_queue::dead_letter_moves_an_exhausted_message_to_the_group_store
(store move + claim-token-mismatch guard + operator read). Schema golden
reblessed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-11 14:41:04 +02:00
parent c06a9e801e
commit fd4336e883
9 changed files with 536 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
crates/manager-core/tests/expected_schema.txt
View File

@@ -231,6 +231,24 @@ table: group_collections
created_at: timestamp with time zone NOT NULL default=now()
updated_at: timestamp with time zone NOT NULL default=now()
table: group_dead_letters
id: uuid NOT NULL default=gen_random_uuid()
group_id: uuid NOT NULL
collection: text NOT NULL
original_event_id: uuid NOT NULL
source: text NOT NULL
op: text NOT NULL
trigger_id: uuid NULL
script_id: uuid NULL
payload: jsonb NOT NULL
attempt_count: integer NOT NULL
first_attempt_at: timestamp with time zone NOT NULL
last_attempt_at: timestamp with time zone NOT NULL
last_error: text NOT NULL
created_at: timestamp with time zone NOT NULL default=now()
resolved_at: timestamp with time zone NULL
resolution: text NULL
table: group_docs
group_id: uuid NOT NULL
collection: text NOT NULL
@@ -567,6 +585,10 @@ indexes on group_collections:
group_collections_group_uidx: public.group_collections USING btree (group_id, lower(name), kind) WHERE (group_id IS NOT NULL)
group_collections_pkey: public.group_collections USING btree (id)
indexes on group_dead_letters:
group_dead_letters_pkey: public.group_dead_letters USING btree (id)
idx_group_dead_letters_group: public.group_dead_letters USING btree (group_id, created_at DESC)
indexes on group_docs:
group_docs_pkey: public.group_docs USING btree (group_id, collection, id)
idx_group_docs_data_gin: public.group_docs USING gin (data jsonb_path_ops)
@@ -814,6 +836,11 @@ constraints on group_collections:
[FOREIGN KEY] group_collections_group_id_fkey: FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE CASCADE
[PRIMARY KEY] group_collections_pkey: PRIMARY KEY (id)
constraints on group_dead_letters:
[CHECK] group_dead_letters_resolution_check: CHECK ((resolution = ANY (ARRAY['replayed'::text, 'ignored'::text, 'handled_by_script'::text, 'handler_failed'::text])))
[FOREIGN KEY] group_dead_letters_group_id_fkey: FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE CASCADE
[PRIMARY KEY] group_dead_letters_pkey: PRIMARY KEY (id)
constraints on group_docs:
[FOREIGN KEY] group_docs_group_id_fkey: FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE CASCADE
[PRIMARY KEY] group_docs_pkey: PRIMARY KEY (group_id, collection, id)
@@ -1005,3 +1032,4 @@ constraints on vars:
0065: group queues
0066: projects
0067: project environments
0068: group dead letters

122
crates/manager-core/tests/group_queue.rs
View File

@@ -12,6 +12,9 @@
use std::collections::HashSet;
use picloud_manager_core::group_dead_letter_repo::{
GroupDeadLetterRepo, PostgresGroupDeadLetterRepo,
};
use picloud_manager_core::group_queue_repo::{
GroupQueueRepo, NewGroupQueueMessage, PostgresGroupQueueRepo,
};
@@ -134,3 +137,122 @@ async fn competing_consumers_claim_each_message_exactly_once() {
.execute(&pool)
.await;
}
#[tokio::test]
async fn dead_letter_moves_an_exhausted_message_to_the_group_store() {
// §11.6 D3: an exhausted shared-queue message is MOVED to the group dead-
// letter store (not dropped) — the row leaves the live queue and appears in
// `group_dead_letters`, operator-visible via GroupDeadLetterRepo.
let Some(pool) = pool_or_skip().await else {
return;
};
let sfx = Uuid::new_v4().simple().to_string();
let g: (Uuid,) = sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
.bind(format!("gdl-g-{sfx}"))
.fetch_one(&pool)
.await
.unwrap();
let group = GroupId::from(g.0);
let queue = PostgresGroupQueueRepo::new(pool.clone());
let dlq = PostgresGroupDeadLetterRepo::new(pool.clone());
let id = queue
.enqueue(NewGroupQueueMessage {
group_id: group,
collection: "jobs".into(),
payload: serde_json::json!({ "task": "boom" }),
deliver_after: None,
max_attempts: 1,
enqueued_by_principal: None,
})
.await
.unwrap();
let msg = queue.claim(group, "jobs").await.unwrap().expect("claimed");
assert_eq!(msg.id, id);
// Exhausted → dead-letter (mirrors the dispatcher's q_terminal shared arm).
let dl_id = queue
.dead_letter(
msg.id,
msg.claim_token,
group,
"jobs",
None,
None,
msg.attempt,
msg.enqueued_at,
"handler exploded",
)
.await
.expect("dead_letter");
// The live queue no longer holds it.
assert_eq!(
queue.depth(group, "jobs").await.unwrap(),
0,
"the dead-lettered message left the live queue"
);
// It is preserved + operator-visible in the group dead-letter store.
let rows = dlq.list_for_group(group, false, 100).await.unwrap();
assert_eq!(rows.len(), 1, "exactly one dead-letter recorded");
let row = &rows[0];
assert_eq!(row.id.into_inner(), dl_id.into_inner());
assert_eq!(row.collection, "jobs");
assert_eq!(row.source, "queue");
assert_eq!(row.last_error, "handler exploded");
assert_eq!(row.payload["message"]["task"], "boom");
assert!(
row.resolved_at.is_none(),
"a fresh dead-letter is unresolved"
);
// A claim_token mismatch cannot dead-letter a re-claimed message: enqueue
// another, claim it, then try to DL with a bogus token → the source-row
// fetch finds nothing (RowNotFound), leaving the live queue intact.
let id2 = queue
.enqueue(NewGroupQueueMessage {
group_id: group,
collection: "jobs".into(),
payload: serde_json::json!({ "task": "keep" }),
deliver_after: None,
max_attempts: 1,
enqueued_by_principal: None,
})
.await
.unwrap();
let m2 = queue
.claim(group, "jobs")
.await
.unwrap()
.expect("claimed 2");
let bogus = Uuid::new_v4();
assert!(
queue
.dead_letter(
m2.id,
bogus,
group,
"jobs",
None,
None,
1,
m2.enqueued_at,
"x"
)
.await
.is_err(),
"a claim-token mismatch must not dead-letter the message"
);
assert_eq!(
queue.depth(group, "jobs").await.unwrap(),
1,
"the mismatched message is still live"
);
let _ = id2;
let _ = sqlx::query("DELETE FROM groups WHERE id = $1")
.bind(g.0)
.execute(&pool)
.await;
}