fix: don't burn a job attempt when the browser is unavailable

A job's attempt is incremented at lease time, then the dispatcher acquires
the headless browser. When the browser was down or mid-restart, acquire()
failed and the error was ack_failed — so during an outage the whole pending
backlog was chewed to `dead` within minutes while workers hot-looped.

Add SyncOutcome::BrowserUnavailable: the dispatcher returns it instead of
propagating the acquire error, and the daemon releases the lease (refunding
the attempt, like the cancel/session paths) and backs off, so an outage is
treated as an infrastructure blip rather than per-job failures.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-11 13:55:50 +02:00
parent 7570524e5b
commit cef41ce76a
9 changed files with 133 additions and 5 deletions

View File

@@ -990,7 +990,17 @@ impl ChapterDispatcher for RealChapterDispatcher {
pages_done: 0,
pages_total: None,
});
let lease = self.browser_manager.acquire().await?;
let lease = match self.browser_manager.acquire().await {
Ok(l) => l,
Err(e) => {
// Browser down / mid-restart: defer the job WITHOUT
// burning an attempt (the daemon releases it back to
// pending) rather than counting an infrastructure
// outage as a per-job failure.
tracing::warn!(error = ?e, "dispatch: browser unavailable — deferring job");
return Ok(SyncOutcome::BrowserUnavailable);
}
};
let result = content::sync_chapter_content(
&lease,
&self.db,
@@ -1061,7 +1071,15 @@ impl ChapterDispatcher for RealChapterDispatcher {
// Scope the lease so it (and the borrowing FetchContext) drop
// before any browser-restart handling in the match below.
let result = {
let lease = self.browser_manager.acquire().await?;
let lease = match self.browser_manager.acquire().await {
Ok(l) => l,
Err(e) => {
// See the SyncChapterContent arm: defer without
// burning an attempt when the browser is unavailable.
tracing::warn!(error = ?e, "dispatch: browser unavailable — deferring job");
return Ok(SyncOutcome::BrowserUnavailable);
}
};
let ctx = crate::crawler::source::FetchContext {
browser: &lease,
rate: &self.rate,