fabi/PiCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(shared-queues): materialized competing consumers + dispatcher branch (D3.2)

Browse Source 
The consumption side of shared durable queues:
- Thread `shared` through BundleTrigger::Queue + QueueTriggerSpec + the
  trigger identity; validate_bundle_for requires a shared queue on a group
  to name a declared kind='queue' collection (a shared queue on an app is
  rejected by the existing app-owner shared guard).
- materialize: a shared queue template materializes a consumer per
  descendant (the M5 one-consumer-slot skip is bypassed for shared —
  competing consumers are intended; each descendant gets one copy).
- dispatcher: ActiveQueueConsumer gains shared_group (from the materialized
  copy's source template via LEFT JOIN); dispatch_one_queue +
  handle_queue_failure route claim/ack/nack/terminal to the group store
  when shared_group is Some, via q_claim/q_ack/q_nack/q_terminal helpers. A
  group claim is normalized to a ClaimedMessage under the consuming app so
  the handler path is unchanged; the reclaim task also drains the group
  store. Exhausted shared messages are dropped (no group dead-letter store
  yet — documented deferral).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-02 22:33:11 +02:00
parent ccd3644aa4
commit 50205e7b7e
8 changed files with 299 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
crates/manager-core/src/dispatcher.rs
View File

@@ -31,7 +31,8 @@ use picloud_orchestrator_core::{ExecutionGate, ExecutorClient};
use picloud_shared::{
AppId, DeadLetterId, ExecResponseSummary, ExecutionId, ExecutionLogSink, ExecutionSource,
HttpDispatchPayload, InboxDeliveryOutcome, InboxFailureKind, InboxResolver, InboxResult,
RequestId, Script, ScriptId, ScriptOwner, ScriptSandbox, TriggerEvent,
QueueMessageId, RequestId, Script, ScriptId, ScriptOwner, ScriptSandbox, TriggerEvent,
TriggerId,
};
use rand::Rng;
use uuid::Uuid;
@@ -65,6 +66,10 @@ pub struct Dispatcher {
/// v1.1.9. Reads `queue_messages` for the queue arm + the reclaim
/// task. None in tests / harnesses that don't exercise queues.
pub queue: Arc<dyn QueueRepo>,
/// §11.6 D3. The group shared-queue store. A materialized consumer of a
/// SHARED group queue template (`consumer.shared_group.is_some()`) claims
/// from here instead of the per-app `queue`.
pub group_queue: Arc<dyn crate::group_queue_repo::GroupQueueRepo>,
pub config: TriggerConfig,
/// Stable id for this dispatcher instance — written into
/// `outbox.claimed_by` for forensics. In MVP this is the host's
@@ -225,6 +230,7 @@ impl Dispatcher {
// Reclaim task: independent cadence (default 30s) so it doesn't
// contend with the per-100ms dispatcher tick.
let reclaim_queue = self.queue.clone();
let reclaim_group_queue = self.group_queue.clone();
let reclaim_interval =
Duration::from_millis(u64::from(self.config.queue_reclaim_interval_ms));
tokio::spawn(async move {
@@ -237,6 +243,14 @@ impl Dispatcher {
Ok(n) => tracing::info!(reclaimed = n, "queue visibility-timeout reclaim"),
Err(e) => tracing::warn!(?e, "queue reclaim task errored"),
}
// §11.6 D3: the group shared-queue store has the same reclaim.
match reclaim_group_queue.reclaim_visibility_timeouts().await {
Ok(0) => {}
Ok(n) => {
tracing::info!(reclaimed = n, "group-queue visibility-timeout reclaim");
}
Err(e) => tracing::warn!(?e, "group-queue reclaim task errored"),
}
}
});
@@ -313,6 +327,126 @@ impl Dispatcher {
Ok(())
}
// §11.6 D3: route the four queue store operations to the per-app store or
// the group shared store, based on whether this consumer was materialized
// from a SHARED group queue template (`consumer.shared_group`). A group
// claim is normalized to a `ClaimedMessage` under the CONSUMING app so the
// rest of the queue arm (handler dispatch, event, logging) is unchanged.
async fn q_claim(&self, c: &ActiveQueueConsumer) -> Result<Option<ClaimedMessage>, String> {
match c.shared_group {
Some(g) => Ok(self
.group_queue
.claim(g, &c.queue_name)
.await
.map_err(|e| e.to_string())?
.map(|m| ClaimedMessage {
id: m.id,
app_id: c.app_id,
queue_name: m.collection,
payload: m.payload,
enqueued_at: m.enqueued_at,
attempt: m.attempt,
max_attempts: m.max_attempts,
claim_token: m.claim_token,
})),
None => self
.queue
.claim(c.app_id, &c.queue_name)
.await
.map_err(|e| e.to_string()),
}
}
async fn q_ack(
&self,
c: &ActiveQueueConsumer,
id: QueueMessageId,
token: uuid::Uuid,
) -> Result<bool, String> {
match c.shared_group {
Some(_) => self
.group_queue
.ack(id, token)
.await
.map_err(|e| e.to_string()),
None => self.queue.ack(id, token).await.map_err(|e| e.to_string()),
}
}
async fn q_nack(
&self,
c: &ActiveQueueConsumer,
id: QueueMessageId,
token: uuid::Uuid,
delay: chrono::Duration,
) -> Result<bool, String> {
match c.shared_group {
Some(_) => self
.group_queue
.nack(id, token, delay)
.await
.map_err(|e| e.to_string()),
None => self
.queue
.nack(id, token, delay)
.await
.map_err(|e| e.to_string()),
}
}
/// Terminal disposition of a message that can't be processed (script
/// missing / cross-app / exhausted). Per-app → dead-letter (+ the caller
/// fans out `dead_letter` triggers). Group shared queue → drop the row
/// (no group dead-letter store yet — documented D3 deferral) + warn.
/// Returns the dead-letter id only for the per-app path (drives fan-out).
async fn q_terminal(
&self,
c: &ActiveQueueConsumer,
claimed: &ClaimedMessage,
trigger_id: Option<TriggerId>,
script_id: Option<ScriptId>,
reason: &str,
) -> Option<DeadLetterId> {
match c.shared_group {
Some(_) => {
if let Err(e) = self
.group_queue
.drop_exhausted(claimed.id, claimed.claim_token)
.await
{
tracing::warn!(?e, "shared-queue drop failed");
}
tracing::warn!(
reason,
queue = %claimed.queue_name,
"shared-queue message dropped (no group dead-letter store yet)"
);
None
}
None => match self
.queue
.dead_letter(
claimed.id,
claimed.claim_token,
claimed.app_id,
&claimed.queue_name,
trigger_id,
script_id,
claimed.attempt,
claimed.enqueued_at,
reason,
)
.await
{
Ok(dl_id) => Some(dl_id),
Err(e) => {
tracing::error!(?e, "queue dead-letter write failed");
None
}
},
}
}
#[allow(clippy::too_many_lines)]
async fn dispatch_one_queue(
&self,
@@ -320,10 +454,9 @@ impl Dispatcher {
) -> Result<(), DispatcherError> {
// Atomic claim — None → nothing pending right now for this queue.
let Some(claimed) = self
.queue
.claim(consumer.app_id, &consumer.queue_name)
.q_claim(consumer)
.await
.map_err(|e| DispatcherError::Outbox(e.to_string()))?
.map_err(DispatcherError::Outbox)?
else {
return Ok(());
};
@@ -333,8 +466,8 @@ impl Dispatcher {
// outbox arm.
let Ok(permit) = self.gate.try_acquire() else {
let _ = self
.queue
.nack(
.q_nack(
consumer,
claimed.id,
claimed.claim_token,
chrono::Duration::milliseconds(100),
@@ -359,16 +492,11 @@ impl Dispatcher {
Ok(None) => {
tracing::warn!(script_id = %consumer.script_id, "queue trigger script missing; dead-lettering");
let _ = self
.queue
.dead_letter(
claimed.id,
claimed.claim_token,
claimed.app_id,
&claimed.queue_name,
.q_terminal(
consumer,
&claimed,
Some(consumer.trigger_id),
Some(consumer.script_id),
claimed.attempt,
claimed.enqueued_at,
"queue trigger script not found",
)
.await;
@@ -398,16 +526,11 @@ impl Dispatcher {
"queue consumer script belongs to a different app; dead-lettering"
);
let _ = self
.queue
.dead_letter(
claimed.id,
claimed.claim_token,
claimed.app_id,
&claimed.queue_name,
.q_terminal(
consumer,
&claimed,
Some(consumer.trigger_id),
Some(consumer.script_id),
claimed.attempt,
claimed.enqueued_at,
"queue consumer target belongs to a different app",
)
.await;
@@ -434,8 +557,8 @@ impl Dispatcher {
"queue consumer script disabled at fire time; releasing claim"
);
if let Err(e) = self
.queue
.nack(
.q_nack(
consumer,
claimed.id,
claimed.claim_token,
chrono::Duration::seconds(1),
@@ -515,7 +638,7 @@ impl Dispatcher {
match outcome {
Ok(_) => {
// Auto-ack on success.
if let Err(e) = self.queue.ack(claimed.id, claimed.claim_token).await {
if let Err(e) = self.q_ack(consumer, claimed.id, claimed.claim_token).await {
tracing::warn!(?e, "queue ack failed");
}
}
@@ -542,8 +665,7 @@ impl Dispatcher {
);
let delay = chrono::Duration::milliseconds(i64::from(delay_ms));
if let Err(e) = self
.queue
.nack(claimed.id, claimed.claim_token, delay)
.q_nack(consumer, claimed.id, claimed.claim_token, delay)
.await
{
tracing::warn!(?e, "queue nack failed");
@@ -558,27 +680,18 @@ impl Dispatcher {
// same way here.
let now = Utc::now();
let last_error = err.to_string();
let dl_id = match self
.queue
.dead_letter(
claimed.id,
claimed.claim_token,
claimed.app_id,
&claimed.queue_name,
// Per-app → dead-letter (+ fan out below). Shared group queue → dropped
// inside q_terminal (no group dead-letter store yet), returns None so
// the fan-out is skipped.
let dl_id = self
.q_terminal(
consumer,
claimed,
Some(consumer.trigger_id),
Some(consumer.script_id),
claimed.attempt,
claimed.enqueued_at,
&last_error,
)
.await
{
Ok(dl_id) => Some(dl_id),
Err(e) => {
tracing::error!(?e, "queue dead-letter write failed");
None
}
};
.await;
if let Some(dead_letter_id) = dl_id {
let original = TriggerEvent::Queue {
queue_name: claimed.queue_name.clone(),
@@ -1512,6 +1625,72 @@ fn apply_jitter(raw: u32, pct: u32) -> u32 {
mod tests {
use super::*;
/// §11.6 D3: a do-nothing group-queue store for dispatcher tests that don't
/// exercise SHARED queues (every consumer has `shared_group: None`, so these
/// methods are never reached — they just satisfy the struct field).
struct NoopGroupQueue;
#[async_trait::async_trait]
impl crate::group_queue_repo::GroupQueueRepo for NoopGroupQueue {
async fn enqueue(
&self,
_msg: crate::group_queue_repo::NewGroupQueueMessage,
) -> Result<QueueMessageId, crate::group_queue_repo::GroupQueueRepoError> {
unreachable!("shared queue not exercised")
}
async fn claim(
&self,
_group_id: picloud_shared::GroupId,
_collection: &str,
) -> Result<
Option<crate::group_queue_repo::ClaimedGroupMessage>,
crate::group_queue_repo::GroupQueueRepoError,
> {
Ok(None)
}
async fn ack(
&self,
_id: QueueMessageId,
_token: Uuid,
) -> Result<bool, crate::group_queue_repo::GroupQueueRepoError> {
Ok(false)
}
async fn nack(
&self,
_id: QueueMessageId,
_token: Uuid,
_delay: chrono::Duration,
) -> Result<bool, crate::group_queue_repo::GroupQueueRepoError> {
Ok(false)
}
async fn drop_exhausted(
&self,
_id: QueueMessageId,
_token: Uuid,
) -> Result<bool, crate::group_queue_repo::GroupQueueRepoError> {
Ok(false)
}
async fn reclaim_visibility_timeouts(
&self,
) -> Result<u64, crate::group_queue_repo::GroupQueueRepoError> {
Ok(0)
}
async fn depth(
&self,
_group_id: picloud_shared::GroupId,
_collection: &str,
) -> Result<u64, crate::group_queue_repo::GroupQueueRepoError> {
Ok(0)
}
async fn depth_pending(
&self,
_group_id: picloud_shared::GroupId,
_collection: &str,
) -> Result<u64, crate::group_queue_repo::GroupQueueRepoError> {
Ok(0)
}
}
#[test]
fn exponential_backoff_doubles_per_attempt() {
// No jitter (pct=0) for a deterministic check.
@@ -1778,14 +1957,14 @@ mod tests {
}
async fn ack(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
) -> Result<bool, crate::queue_repo::QueueRepoError> {
unimplemented!("not used by this test")
}
async fn nack(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
_retry_delay: chrono::Duration,
) -> Result<bool, crate::queue_repo::QueueRepoError> {
@@ -1820,7 +1999,7 @@ mod tests {
#[allow(clippy::too_many_arguments)]
async fn dead_letter(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
_app_id: AppId,
_queue_name: &str,
@@ -2205,6 +2384,7 @@ mod tests {
retry_backoff: BackoffShape::Exponential,
retry_base_ms: 1000,
registered_by_principal: AdminUserId::new(),
shared_group: None,
};
let nacked = Arc::new(AtomicBool::new(false));
@@ -2229,6 +2409,7 @@ mod tests {
claimed,
nacked: nacked.clone(),
}),
group_queue: Arc::new(NoopGroupQueue),
config: TriggerConfig::from_env(),
instance_id: "test-instance".into(),
};
@@ -2310,14 +2491,14 @@ mod tests {
}
async fn ack(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
) -> Result<bool, crate::queue_repo::QueueRepoError> {
unimplemented!("not used by this test")
}
async fn nack(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
_retry_delay: chrono::Duration,
) -> Result<bool, crate::queue_repo::QueueRepoError> {
@@ -2354,7 +2535,7 @@ mod tests {
#[allow(clippy::too_many_arguments)]
async fn dead_letter(
&self,
_message_id: picloud_shared::QueueMessageId,
_message_id: QueueMessageId,
_claim_token: Uuid,
_app_id: AppId,
_queue_name: &str,
@@ -2461,6 +2642,7 @@ mod tests {
log_sink: Arc::new(UnusedLogSink),
inbox: Arc::new(UnusedInbox),
queue: Arc::new(UnusedQueue),
group_queue: Arc::new(NoopGroupQueue),
config: TriggerConfig::from_env(),
instance_id: "test-instance".into(),
};