From f83d49b83ee81ffab523aeeed6ac0890dc0db63f Mon Sep 17 00:00:00 2001
From: MechaCat02 <fabian@diekaulbachs.de>
Date: Sat, 11 Jul 2026 13:58:39 +0200
Subject: [PATCH] fix: exponential backoff for idle crawler workers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An idle worker re-issued a row-locking lease query every 1s per worker with
no backoff. Replace the flat sleep with idle_backoff: 1s, 2s, 4s, … capped at
30s, reset to 0 the moment a job is leased, so a quiet daemon stops hammering
the crawler_jobs table.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 backend/Cargo.lock            |  2 +-
 backend/Cargo.toml            |  2 +-
 backend/src/crawler/daemon.rs | 47 ++++++++++++++++++++++++++++++++++-
 frontend/package.json         |  2 +-
 4 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index adb6422..00174bc 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.124.17"
+version = "0.124.18"
 dependencies = [
  "anyhow",
  "argon2",
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 67632ec..1ebe6af 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.124.17"
+version = "0.124.18"
 edition = "2021"
 default-run = "mangalord"
 
diff --git a/backend/src/crawler/daemon.rs b/backend/src/crawler/daemon.rs
index 9c23d5a..487349c 100644
--- a/backend/src/crawler/daemon.rs
+++ b/backend/src/crawler/daemon.rs
@@ -72,6 +72,24 @@ const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
 /// the browser is down or mid-restart.
 const BROWSER_UNAVAILABLE_BACKOFF: Duration = Duration::from_secs(5);
 
+/// Longest an idle worker waits between lease polls. Bounds the exponential
+/// [`idle_backoff`] so a worker still notices freshly-enqueued work reasonably
+/// soon after a quiet spell.
+const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
+
+/// Backoff for a worker that keeps finding no work: 1s, 2s, 4s, … capped at
+/// [`IDLE_BACKOFF_CAP`]. `consecutive_empty` is the number of empty polls seen
+/// so far (0 on the first miss); reset to 0 the moment a job is leased. Replaces
+/// the old flat 1s sleep so an idle daemon isn't firing a row-locking `SELECT …
+/// FOR UPDATE SKIP LOCKED` lease query every second per worker.
+fn idle_backoff(consecutive_empty: u32) -> Duration {
+    let cap = IDLE_BACKOFF_CAP.as_secs();
+    // 1 << n grows the interval; saturate to the cap once the shift overflows
+    // or the value exceeds the cap.
+    let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
+    Duration::from_secs(secs)
+}
+
 #[async_trait]
 pub trait MetadataPass: Send + Sync {
     async fn run(&self) -> anyhow::Result<pipeline::MetadataStats>;
@@ -362,6 +380,9 @@ struct WorkerContext {
 
 impl WorkerContext {
     async fn run(self) {
+        // Consecutive empty lease polls, driving the idle backoff. Reset to 0
+        // the moment any job is leased.
+        let mut idle_streak: u32 = 0;
         loop {
             if self.cancel.is_cancelled() {
                 tracing::info!(worker = self.id, "worker: shutdown");
@@ -391,11 +412,14 @@ impl WorkerContext {
                 }
             };
             let Some(lease) = leases.into_iter().next() else {
+                let backoff = idle_backoff(idle_streak);
+                idle_streak = idle_streak.saturating_add(1);
                 tokio::select! {
-                    _ = tokio::time::sleep(Duration::from_secs(1)) => continue,
+                    _ = tokio::time::sleep(backoff) => continue,
                     _ = self.cancel.cancelled() => return,
                 }
             };
+            idle_streak = 0;
             self.process_lease(lease).await;
         }
     }
@@ -774,6 +798,27 @@ mod tests {
         Utc.with_ymd_and_hms(y, mo, d, h, mi, 0).unwrap()
     }
 
+    #[test]
+    fn idle_backoff_grows_then_caps() {
+        // First miss is a short 1s poll; the interval doubles each empty poll…
+        assert_eq!(idle_backoff(0), Duration::from_secs(1));
+        assert_eq!(idle_backoff(1), Duration::from_secs(2));
+        assert_eq!(idle_backoff(2), Duration::from_secs(4));
+        assert_eq!(idle_backoff(3), Duration::from_secs(8));
+        assert_eq!(idle_backoff(4), Duration::from_secs(16));
+        // …and saturates at the cap rather than growing unbounded.
+        assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
+        assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
+        // Never exceeds the cap and is monotonic non-decreasing.
+        let mut prev = Duration::ZERO;
+        for n in 0..40 {
+            let b = idle_backoff(n);
+            assert!(b >= prev, "backoff must be non-decreasing at n={n}");
+            assert!(b <= IDLE_BACKOFF_CAP, "backoff must never exceed the cap");
+            prev = b;
+        }
+    }
+
     #[test]
     fn next_fire_in_utc_at_midnight_advances_one_day() {
         let now = dt_utc(2026, 5, 25, 12, 0); // noon UTC
diff --git a/frontend/package.json b/frontend/package.json
index 2a0c67a..3ae0676 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.124.17",
+  "version": "0.124.18",
   "private": true,
   "type": "module",
   "scripts": {