fix: exponential backoff for idle crawler workers

An idle worker re-issued a row-locking lease query every 1s per worker with
no backoff. Replace the flat sleep with idle_backoff: 1s, 2s, 4s, … capped at
30s, reset to 0 the moment a job is leased, so a quiet daemon stops hammering
the crawler_jobs table.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/Cargo.lock

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.124.17"
+version = "0.124.18"
 dependencies = [
  "anyhow",
  "argon2",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.124.17"
+version = "0.124.18"
 edition = "2021"
 default-run = "mangalord"
 

backend/src/crawler/daemon.rs

@@ -72,6 +72,24 @@ const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
 /// the browser is down or mid-restart.
 const BROWSER_UNAVAILABLE_BACKOFF: Duration = Duration::from_secs(5);
 
+/// Longest an idle worker waits between lease polls. Bounds the exponential
+/// [`idle_backoff`] so a worker still notices freshly-enqueued work reasonably
+/// soon after a quiet spell.
+const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
+
+/// Backoff for a worker that keeps finding no work: 1s, 2s, 4s, … capped at
+/// [`IDLE_BACKOFF_CAP`]. `consecutive_empty` is the number of empty polls seen
+/// so far (0 on the first miss); reset to 0 the moment a job is leased. Replaces
+/// the old flat 1s sleep so an idle daemon isn't firing a row-locking `SELECT …
+/// FOR UPDATE SKIP LOCKED` lease query every second per worker.
+fn idle_backoff(consecutive_empty: u32) -> Duration {
+    let cap = IDLE_BACKOFF_CAP.as_secs();
+    // 1 << n grows the interval; saturate to the cap once the shift overflows
+    // or the value exceeds the cap.
+    let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
+    Duration::from_secs(secs)
+}
+
 #[async_trait]
 pub trait MetadataPass: Send + Sync {
     async fn run(&self) -> anyhow::Result<pipeline::MetadataStats>;
@@ -362,6 +380,9 @@ struct WorkerContext {
 
 impl WorkerContext {
     async fn run(self) {
+        // Consecutive empty lease polls, driving the idle backoff. Reset to 0
+        // the moment any job is leased.
+        let mut idle_streak: u32 = 0;
         loop {
             if self.cancel.is_cancelled() {
                 tracing::info!(worker = self.id, "worker: shutdown");
@@ -391,11 +412,14 @@ impl WorkerContext {
                 }
             };
             let Some(lease) = leases.into_iter().next() else {
+                let backoff = idle_backoff(idle_streak);
+                idle_streak = idle_streak.saturating_add(1);
                 tokio::select! {
-                    _ = tokio::time::sleep(Duration::from_secs(1)) => continue,
+                    _ = tokio::time::sleep(backoff) => continue,
                     _ = self.cancel.cancelled() => return,
                 }
             };
+            idle_streak = 0;
             self.process_lease(lease).await;
         }
     }
@@ -774,6 +798,27 @@ mod tests {
         Utc.with_ymd_and_hms(y, mo, d, h, mi, 0).unwrap()
     }
 
+    #[test]
+    fn idle_backoff_grows_then_caps() {
+        // First miss is a short 1s poll; the interval doubles each empty poll…
+        assert_eq!(idle_backoff(0), Duration::from_secs(1));
+        assert_eq!(idle_backoff(1), Duration::from_secs(2));
+        assert_eq!(idle_backoff(2), Duration::from_secs(4));
+        assert_eq!(idle_backoff(3), Duration::from_secs(8));
+        assert_eq!(idle_backoff(4), Duration::from_secs(16));
+        // …and saturates at the cap rather than growing unbounded.
+        assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
+        assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
+        // Never exceeds the cap and is monotonic non-decreasing.
+        let mut prev = Duration::ZERO;
+        for n in 0..40 {
+            let b = idle_backoff(n);
+            assert!(b >= prev, "backoff must be non-decreasing at n={n}");
+            assert!(b <= IDLE_BACKOFF_CAP, "backoff must never exceed the cap");
+            prev = b;
+        }
+    }
+
     #[test]
     fn next_fire_in_utc_at_midnight_advances_one_day() {
         let now = dt_utc(2026, 5, 25, 12, 0); // noon UTC

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.124.17",
+  "version": "0.124.18",
   "private": true,
   "type": "module",
   "scripts": {