fix: batch and index the crawler_jobs retention reaper

reap_terminal was one unbatched DELETE with no updated_at index — a full scan and
long-held lock over a large backlog (audit M5). Migration 0040 adds a partial
index on (updated_at) WHERE state IN ('done','dead'); reap_terminal now deletes in
bounded 5k batches. Existing tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/src/crawler/jobs.rs

@@ -495,15 +495,33 @@ pub async fn reap_terminal(pool: &PgPool, retention_days: u32) -> sqlx::Result<u
     if retention_days == 0 {
         return Ok(0);
     }
-    let result = sqlx::query(
-        "DELETE FROM crawler_jobs \
-          WHERE state IN ('done', 'dead') \
-            AND updated_at < now() - ($1::bigint || ' days')::interval",
-    )
-    .bind(retention_days as i64)
-    .execute(pool)
-    .await?;
-    Ok(result.rows_affected())
+    // Delete in bounded batches rather than one statement. A single unbatched
+    // DELETE over an unbounded backlog takes a long-held lock and one huge
+    // transaction; batching keeps each delete short (index-backed by
+    // crawler_jobs_terminal_reap_idx, 0040) so the reaper never pins the table
+    // (audit M5). Loop until a batch comes back short.
+    const BATCH: i64 = 5_000;
+    let mut total: u64 = 0;
+    loop {
+        let result = sqlx::query(
+            "DELETE FROM crawler_jobs \
+              WHERE ctid IN ( \
+                    SELECT ctid FROM crawler_jobs \
+                     WHERE state IN ('done', 'dead') \
+                       AND updated_at < now() - ($1::bigint || ' days')::interval \
+                     LIMIT $2 )",
+        )
+        .bind(retention_days as i64)
+        .bind(BATCH)
+        .execute(pool)
+        .await?;
+        let n = result.rows_affected();
+        total += n;
+        if n < BATCH as u64 {
+            break;
+        }
+    }
+    Ok(total)
 }
 
 #[cfg(test)]