fix(manager-core): F-P-007 concurrent queue dispatch per tick
tick_queue_arm called list_active_queue_consumers() and then iterated serially, awaiting one queue.claim(app, queue) per consumer. With N consumers and a 100ms tick the worst-case throughput was N × (claim + executor) / tick — one slow handler blocked every other queue's progress on the dispatcher's task. Replace the for-loop with `futures::stream::iter(consumers) .for_each_concurrent(QUEUE_DISPATCH_PARALLELISM, …)`. The execution gate already caps real script-concurrency to its permits, so this just removes the dispatcher-side serialization. QUEUE_DISPATCH_PARALLELISM = 32 (matches the default gate). Workspace gains `futures = 0.3` so `for_each_concurrent` is available. AUDIT.md anchor: F-P-007. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
29
Cargo.lock
generated
29
Cargo.lock
generated
@@ -830,6 +830,21 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures"
|
||||||
|
version = "0.3.32"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
||||||
|
dependencies = [
|
||||||
|
"futures-channel",
|
||||||
|
"futures-core",
|
||||||
|
"futures-executor",
|
||||||
|
"futures-io",
|
||||||
|
"futures-sink",
|
||||||
|
"futures-task",
|
||||||
|
"futures-util",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-channel"
|
name = "futures-channel"
|
||||||
version = "0.3.32"
|
version = "0.3.32"
|
||||||
@@ -874,6 +889,17 @@ version = "0.3.32"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-macro"
|
||||||
|
version = "0.3.32"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-sink"
|
name = "futures-sink"
|
||||||
version = "0.3.32"
|
version = "0.3.32"
|
||||||
@@ -892,8 +918,10 @@ version = "0.3.32"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
|
"futures-macro",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
"memchr",
|
||||||
@@ -1862,6 +1890,7 @@ dependencies = [
|
|||||||
"chrono-tz",
|
"chrono-tz",
|
||||||
"cron",
|
"cron",
|
||||||
"data-encoding",
|
"data-encoding",
|
||||||
|
"futures",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac",
|
"hmac",
|
||||||
"lettre",
|
"lettre",
|
||||||
|
|||||||
@@ -53,8 +53,10 @@ chrono = { version = "0.4", features = ["serde"] }
|
|||||||
chrono-tz = "0.9"
|
chrono-tz = "0.9"
|
||||||
cron = "0.12"
|
cron = "0.12"
|
||||||
|
|
||||||
# Async traits
|
# Async traits + bounded-concurrency stream utilities (queue dispatcher
|
||||||
|
# uses `for_each_concurrent`).
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
|
futures = "0.3"
|
||||||
|
|
||||||
# Rhai scripting. Pinned exactly (`=1.24`) because the `internals`
|
# Rhai scripting. Pinned exactly (`=1.24`) because the `internals`
|
||||||
# feature surface is not semver-stable — future bumps must be deliberate.
|
# feature surface is not semver-stable — future bumps must be deliberate.
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ picloud-executor-core.workspace = true
|
|||||||
picloud-orchestrator-core.workspace = true
|
picloud-orchestrator-core.workspace = true
|
||||||
|
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
|
futures.workspace = true
|
||||||
axum.workspace = true
|
axum.workspace = true
|
||||||
rand.workspace = true
|
rand.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
|
|||||||
@@ -79,6 +79,11 @@ const TICK_INTERVAL: Duration = Duration::from_millis(100);
|
|||||||
/// 5-minute cap.
|
/// 5-minute cap.
|
||||||
const ASYNC_EXEC_TIMEOUT: Duration = Duration::from_secs(300);
|
const ASYNC_EXEC_TIMEOUT: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
/// F-P-007: per-tick concurrency cap for queue dispatches. Matches the
|
||||||
|
/// default execution-gate concurrency so the gate is what bounds real
|
||||||
|
/// parallelism, not the dispatcher's serial-await.
|
||||||
|
const QUEUE_DISPATCH_PARALLELISM: usize = 32;
|
||||||
|
|
||||||
impl Dispatcher {
|
impl Dispatcher {
|
||||||
/// Spawn the dispatcher loop as a detached `tokio::task`. Also
|
/// Spawn the dispatcher loop as a detached `tokio::task`. Also
|
||||||
/// spawns the v1.1.9 queue visibility-timeout reclaim task. Both
|
/// spawns the v1.1.9 queue visibility-timeout reclaim task. Both
|
||||||
@@ -149,17 +154,30 @@ impl Dispatcher {
|
|||||||
/// attempt one claim per `(app_id, queue_name)`. Each successful
|
/// attempt one claim per `(app_id, queue_name)`. Each successful
|
||||||
/// claim runs through the same executor path as outbox-backed
|
/// claim runs through the same executor path as outbox-backed
|
||||||
/// trigger dispatches, so retry / dead-letter wiring is shared.
|
/// trigger dispatches, so retry / dead-letter wiring is shared.
|
||||||
|
///
|
||||||
|
/// F-P-007: bound concurrency with `for_each_concurrent` instead of
|
||||||
|
/// serial-await. With N consumers and a 100ms tick the worst-case
|
||||||
|
/// throughput was N × (claim + executor) / tick; concurrent dispatch
|
||||||
|
/// lets one slow handler not block the others. The execution gate
|
||||||
|
/// still caps real parallelism — this just removes the
|
||||||
|
/// dispatcher-side serialization.
|
||||||
async fn tick_queue_arm(&self) -> Result<(), DispatcherError> {
|
async fn tick_queue_arm(&self) -> Result<(), DispatcherError> {
|
||||||
|
use futures::stream::{self, StreamExt};
|
||||||
let consumers = self
|
let consumers = self
|
||||||
.triggers
|
.triggers
|
||||||
.list_active_queue_consumers()
|
.list_active_queue_consumers()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
|
.map_err(|e| DispatcherError::ResolveTrigger(e.to_string()))?;
|
||||||
for consumer in consumers {
|
// Concurrency cap matches the execution gate's typical default;
|
||||||
if let Err(err) = self.dispatch_one_queue(&consumer).await {
|
// overflow attempts return Err(Overloaded) and back off so a
|
||||||
tracing::warn!(?err, queue = %consumer.queue_name, "queue dispatch errored");
|
// queue burst doesn't hammer the executor.
|
||||||
}
|
stream::iter(consumers)
|
||||||
}
|
.for_each_concurrent(QUEUE_DISPATCH_PARALLELISM, |consumer| async move {
|
||||||
|
if let Err(err) = self.dispatch_one_queue(&consumer).await {
|
||||||
|
tracing::warn!(?err, queue = %consumer.queue_name, "queue dispatch errored");
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user