fix: don't burn a job attempt when the browser is unavailable

A job's attempt is incremented at lease time, then the dispatcher acquires
the headless browser. When the browser was down or mid-restart, acquire()
failed and the error was ack_failed — so during an outage the whole pending
backlog was chewed to `dead` within minutes while workers hot-looped.

Add SyncOutcome::BrowserUnavailable: the dispatcher returns it instead of
propagating the acquire error, and the daemon releases the lease (refunding
the attempt, like the cancel/session paths) and backs off, so an outage is
treated as an infrastructure blip rather than per-job failures.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-11 13:55:50 +02:00
parent 7570524e5b
commit cef41ce76a
9 changed files with 133 additions and 5 deletions

View File

@@ -104,6 +104,20 @@ impl ChapterDispatcher for FailingDispatcher {
}
}
/// Always reports the browser unavailable — models a Chromium outage where
/// `acquire()` fails. The job must be returned to `pending` WITHOUT burning an
/// attempt, so an outage doesn't chew the backlog to `dead`.
struct BrowserUnavailableDispatcher {
seen: AtomicUsize,
}
#[async_trait::async_trait]
impl ChapterDispatcher for BrowserUnavailableDispatcher {
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
self.seen.fetch_add(1, Ordering::AcqRel);
Ok(SyncOutcome::BrowserUnavailable)
}
}
/// Never completes — used to verify the worker's outer dispatch timeout.
struct HangingDispatcher {
seen: AtomicUsize,
@@ -204,6 +218,58 @@ async fn shutdown_mid_dispatch_releases_lease_without_burning_attempt(pool: PgPo
);
}
#[sqlx::test(migrations = "./migrations")]
async fn browser_unavailable_releases_lease_without_burning_attempt(pool: PgPool) {
// During a browser outage the dispatcher reports BrowserUnavailable. The
// worker must return the job to `pending` with the attempt refunded
// (attempts stays 0) instead of ack-failing it toward `dead`, so the whole
// pending backlog survives the outage.
enqueue_chapter_job(&pool).await;
let dispatcher = Arc::new(BrowserUnavailableDispatcher {
seen: AtomicUsize::new(0),
});
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
let cancel = CancellationToken::new();
let handle = daemon::spawn(
pool.clone(),
cancel.clone(),
make_cfg(None, dispatcher.clone(), session_expired, 1),
);
// Wait until the dispatcher has been invoked at least once (the job was
// leased and deferred).
let mut dispatched = false;
for _ in 0..40 {
if dispatcher.seen.load(Ordering::Acquire) >= 1 {
dispatched = true;
break;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
assert!(dispatched, "dispatcher must have been invoked");
handle.shutdown().await;
assert_eq!(
count_state(&pool, "dead").await,
0,
"browser outage must not dead-letter the job"
);
assert_eq!(
count_state(&pool, "pending").await,
1,
"job returns to pending after a browser-unavailable outcome"
);
let attempts: i32 = sqlx::query_scalar("SELECT attempts FROM crawler_jobs")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
attempts, 0,
"browser-unavailable must refund the lease attempt (no burn)"
);
}
#[sqlx::test(migrations = "./migrations")]
async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
enqueue_chapter_job(&pool).await;