fix: don't burn a job attempt when the browser is unavailable
A job's attempt is incremented at lease time, then the dispatcher acquires the headless browser. When the browser was down or mid-restart, acquire() failed and the error was ack_failed — so during an outage the whole pending backlog was chewed to `dead` within minutes while workers hot-looped. Add SyncOutcome::BrowserUnavailable: the dispatcher returns it instead of propagating the acquire error, and the daemon releases the lease (refunding the attempt, like the cancel/session paths) and backs off, so an outage is treated as an infrastructure blip rather than per-job failures. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -71,6 +71,13 @@ pub enum SyncOutcome {
|
||||
/// Session probe failed mid-sync (avatar selector missing on the
|
||||
/// chapter page). Caller should abort the whole crawler run.
|
||||
SessionExpired,
|
||||
/// The headless browser could not be acquired (down or mid-restart).
|
||||
/// Produced only by the queue dispatcher (which calls `acquire()`); it is
|
||||
/// an *infrastructure* outage, not a job failure, so the daemon returns the
|
||||
/// job to `pending` WITHOUT burning a retry attempt. `sync_chapter_content`
|
||||
/// itself never returns this — callers that already hold a lease can treat
|
||||
/// it as unreachable.
|
||||
BrowserUnavailable,
|
||||
}
|
||||
|
||||
/// Per-chapter max fetch attempts when TOR is configured. `N = 3` means
|
||||
|
||||
@@ -66,6 +66,12 @@ const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
/// the lease window leaves two missed-beat's slack before expiry.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// How long a worker waits after a `BrowserUnavailable` outcome before looping
|
||||
/// back to lease again. The job was released (not failed) so it stays pending;
|
||||
/// this backoff keeps the worker from hot-looping lease→acquire→release while
|
||||
/// the browser is down or mid-restart.
|
||||
const BROWSER_UNAVAILABLE_BACKOFF: Duration = Duration::from_secs(5);
|
||||
|
||||
#[async_trait]
|
||||
pub trait MetadataPass: Send + Sync {
|
||||
async fn run(&self) -> anyhow::Result<pipeline::MetadataStats>;
|
||||
@@ -502,6 +508,24 @@ impl WorkerContext {
|
||||
self.status.poke();
|
||||
let _ = jobs::release(&self.pool, lease.id, lease.lease_generation).await;
|
||||
}
|
||||
Ok(Ok(SyncOutcome::BrowserUnavailable)) => {
|
||||
// Infrastructure outage, not a job failure: the browser was
|
||||
// down or mid-restart when the dispatcher tried to acquire it.
|
||||
// Return the job to `pending` WITHOUT burning an attempt (like
|
||||
// the cancel/session paths) so an outage doesn't chew the whole
|
||||
// backlog to `dead`, then back off to avoid hot-looping while
|
||||
// the browser recovers.
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
lease_id = %lease.id,
|
||||
"worker: browser unavailable — released lease without burning an attempt"
|
||||
);
|
||||
let _ = jobs::release(&self.pool, lease.id, lease.lease_generation).await;
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(BROWSER_UNAVAILABLE_BACKOFF) => {}
|
||||
_ = self.cancel.cancelled() => {}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
|
||||
@@ -280,6 +280,12 @@ impl ResyncService for RealResyncService {
|
||||
SyncOutcome::SessionExpired => {
|
||||
anyhow::bail!("source session expired — operator must refresh PHPSESSID")
|
||||
}
|
||||
// Unreachable here: resync already holds its own browser lease and
|
||||
// `sync_chapter_content` never acquires one, so it can't report the
|
||||
// browser unavailable. Handled defensively for exhaustiveness.
|
||||
SyncOutcome::BrowserUnavailable => {
|
||||
anyhow::bail!("crawler browser unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user