fix(crawler): harden crash & shutdown job recovery
Three failure-mode fixes surfaced by a crawler recovery audit: - Graceful shutdown mid-dispatch now releases the in-flight job back to pending without burning a retry attempt. process_lease wraps the dispatch in a biased tokio::select! cancel arm that aborts the heartbeat and calls jobs::release; previously a mid-job SIGTERM left the row 'running' until lease expiry and cost one of max_attempts. - Boot-time jobs::reclaim_orphaned resets 'running' jobs with an expired lease back to pending (attempt refunded), run once at daemon startup before workers start. Crash recovery is now immediate instead of waiting a full lease window for the lazy lease-expiry path. Safe under multi-replica: only already-expired leases are touched, which a healthy heartbeating peer never has. - Manga-list parsing now warns when listing anchors are dropped for a missing/empty href or title (split into parse_manga_list_anchors so the drop count is testable), turning silent source markup drift into an observable signal. Returned refs are byte-identical to before. Tests added: shutdown_mid_dispatch_releases_lease_without_burning_attempt, reclaim_orphaned_resets_only_expired_running_jobs, and a drop-count unit test. Patch bump 0.81.0 -> 0.81.1 (both manifests). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -136,6 +136,60 @@ async fn worker_times_out_a_hung_dispatch_and_acks_failed(pool: PgPool) {
|
||||
assert!(dispatcher.seen.load(Ordering::Acquire) >= 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn shutdown_mid_dispatch_releases_lease_without_burning_attempt(pool: PgPool) {
|
||||
// A worker that is mid-dispatch when shutdown is signalled must return
|
||||
// its job to `pending` with the attempt refunded (attempts back to 0),
|
||||
// rather than leaving it `running` until lease expiry with a burned
|
||||
// attempt. Uses a never-completing dispatcher so the job is provably
|
||||
// in-flight when we cancel.
|
||||
enqueue_chapter_job(&pool).await;
|
||||
let dispatcher = Arc::new(HangingDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let mut cfg = make_cfg(None, dispatcher.clone(), session_expired, 1);
|
||||
// Long timeout so the dispatch can't be acked-failed by the outer
|
||||
// timeout before we cancel.
|
||||
cfg.job_timeout = Duration::from_secs(60);
|
||||
let handle = daemon::spawn(pool.clone(), cancel.clone(), cfg);
|
||||
|
||||
// Wait until the job is actually leased (running) and the dispatch has
|
||||
// entered the hanging future.
|
||||
let mut entered = false;
|
||||
for _ in 0..40 {
|
||||
if dispatcher.seen.load(Ordering::Acquire) >= 1 && count_state(&pool, "running").await == 1
|
||||
{
|
||||
entered = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(entered, "dispatch must be in-flight (running) before shutdown");
|
||||
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(
|
||||
count_state(&pool, "running").await,
|
||||
0,
|
||||
"no job left running after graceful shutdown"
|
||||
);
|
||||
assert_eq!(
|
||||
count_state(&pool, "pending").await,
|
||||
1,
|
||||
"the in-flight job is released back to pending"
|
||||
);
|
||||
let attempts: i32 = sqlx::query_scalar("SELECT attempts FROM crawler_jobs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"graceful shutdown must refund the lease attempt (no burn)"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
||||
enqueue_chapter_job(&pool).await;
|
||||
|
||||
Reference in New Issue
Block a user