fix: don't burn a job attempt when the browser is unavailable
A job's attempt is incremented at lease time, then the dispatcher acquires the headless browser. When the browser was down or mid-restart, acquire() failed and the error was ack_failed — so during an outage the whole pending backlog was chewed to `dead` within minutes while workers hot-looped. Add SyncOutcome::BrowserUnavailable: the dispatcher returns it instead of propagating the acquire error, and the daemon releases the lease (refunding the attempt, like the cancel/session paths) and backs off, so an outage is treated as an infrastructure blip rather than per-job failures. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -104,6 +104,20 @@ impl ChapterDispatcher for FailingDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Always reports the browser unavailable — models a Chromium outage where
|
||||
/// `acquire()` fails. The job must be returned to `pending` WITHOUT burning an
|
||||
/// attempt, so an outage doesn't chew the backlog to `dead`.
|
||||
struct BrowserUnavailableDispatcher {
|
||||
seen: AtomicUsize,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl ChapterDispatcher for BrowserUnavailableDispatcher {
|
||||
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
|
||||
self.seen.fetch_add(1, Ordering::AcqRel);
|
||||
Ok(SyncOutcome::BrowserUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
/// Never completes — used to verify the worker's outer dispatch timeout.
|
||||
struct HangingDispatcher {
|
||||
seen: AtomicUsize,
|
||||
@@ -204,6 +218,58 @@ async fn shutdown_mid_dispatch_releases_lease_without_burning_attempt(pool: PgPo
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn browser_unavailable_releases_lease_without_burning_attempt(pool: PgPool) {
|
||||
// During a browser outage the dispatcher reports BrowserUnavailable. The
|
||||
// worker must return the job to `pending` with the attempt refunded
|
||||
// (attempts stays 0) instead of ack-failing it toward `dead`, so the whole
|
||||
// pending backlog survives the outage.
|
||||
enqueue_chapter_job(&pool).await;
|
||||
let dispatcher = Arc::new(BrowserUnavailableDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel.clone(),
|
||||
make_cfg(None, dispatcher.clone(), session_expired, 1),
|
||||
);
|
||||
|
||||
// Wait until the dispatcher has been invoked at least once (the job was
|
||||
// leased and deferred).
|
||||
let mut dispatched = false;
|
||||
for _ in 0..40 {
|
||||
if dispatcher.seen.load(Ordering::Acquire) >= 1 {
|
||||
dispatched = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(dispatched, "dispatcher must have been invoked");
|
||||
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(
|
||||
count_state(&pool, "dead").await,
|
||||
0,
|
||||
"browser outage must not dead-letter the job"
|
||||
);
|
||||
assert_eq!(
|
||||
count_state(&pool, "pending").await,
|
||||
1,
|
||||
"job returns to pending after a browser-unavailable outcome"
|
||||
);
|
||||
let attempts: i32 = sqlx::query_scalar("SELECT attempts FROM crawler_jobs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"browser-unavailable must refund the lease attempt (no burn)"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
||||
enqueue_chapter_job(&pool).await;
|
||||
|
||||
Reference in New Issue
Block a user