From bde72e47f44f5bfdf921dd42790d1d529c32edb8 Mon Sep 17 00:00:00 2001
From: MechaCat02 <MechaCat02@example.com>
Date: Mon, 13 Jul 2026 21:46:01 +0200
Subject: [PATCH] fix: batch and index the crawler_jobs retention reaper
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

reap_terminal was one unbatched DELETE with no updated_at index — a full scan and
long-held lock over a large backlog (audit M5). Migration 0040 adds a partial
index on (updated_at) WHERE state IN ('done','dead'); reap_terminal now deletes in
bounded 5k batches. Existing tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 backend/Cargo.lock                            |  2 +-
 backend/Cargo.toml                            |  2 +-
 .../0040_crawler_jobs_terminal_reap_idx.sql   |  9 +++++
 backend/src/crawler/jobs.rs                   | 36 ++++++++++++++-----
 frontend/package.json                         |  2 +-
 5 files changed, 39 insertions(+), 12 deletions(-)
 create mode 100644 backend/migrations/0040_crawler_jobs_terminal_reap_idx.sql

diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 6657db8..093276a 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.128.19"
+version = "0.128.20"
 dependencies = [
  "anyhow",
  "argon2",
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 480e3c8..dcd918c 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.128.19"
+version = "0.128.20"
 edition = "2021"
 default-run = "mangalord"
 
diff --git a/backend/migrations/0040_crawler_jobs_terminal_reap_idx.sql b/backend/migrations/0040_crawler_jobs_terminal_reap_idx.sql
new file mode 100644
index 0000000..5619b95
--- /dev/null
+++ b/backend/migrations/0040_crawler_jobs_terminal_reap_idx.sql
@@ -0,0 +1,9 @@
+-- Index the retention reaper's scan. reap_terminal deletes
+--   WHERE state IN ('done','dead') AND updated_at < now() - interval
+-- which had no supporting index, so each daily sweep sequential-scanned the whole
+-- crawler_jobs table to find expired terminal rows (audit M5). A partial index on
+-- updated_at over just the terminal states makes the batched reaper's per-batch
+-- SELECT index-backed.
+CREATE INDEX crawler_jobs_terminal_reap_idx
+    ON crawler_jobs (updated_at)
+    WHERE state IN ('done', 'dead');
diff --git a/backend/src/crawler/jobs.rs b/backend/src/crawler/jobs.rs
index d5ba276..f8714f1 100644
--- a/backend/src/crawler/jobs.rs
+++ b/backend/src/crawler/jobs.rs
@@ -495,15 +495,33 @@ pub async fn reap_terminal(pool: &PgPool, retention_days: u32) -> sqlx::Result<u
     if retention_days == 0 {
         return Ok(0);
     }
-    let result = sqlx::query(
-        "DELETE FROM crawler_jobs \
-          WHERE state IN ('done', 'dead') \
-            AND updated_at < now() - ($1::bigint || ' days')::interval",
-    )
-    .bind(retention_days as i64)
-    .execute(pool)
-    .await?;
-    Ok(result.rows_affected())
+    // Delete in bounded batches rather than one statement. A single unbatched
+    // DELETE over an unbounded backlog takes a long-held lock and one huge
+    // transaction; batching keeps each delete short (index-backed by
+    // crawler_jobs_terminal_reap_idx, 0040) so the reaper never pins the table
+    // (audit M5). Loop until a batch comes back short.
+    const BATCH: i64 = 5_000;
+    let mut total: u64 = 0;
+    loop {
+        let result = sqlx::query(
+            "DELETE FROM crawler_jobs \
+              WHERE ctid IN ( \
+                    SELECT ctid FROM crawler_jobs \
+                     WHERE state IN ('done', 'dead') \
+                       AND updated_at < now() - ($1::bigint || ' days')::interval \
+                     LIMIT $2 )",
+        )
+        .bind(retention_days as i64)
+        .bind(BATCH)
+        .execute(pool)
+        .await?;
+        let n = result.rows_affected();
+        total += n;
+        if n < BATCH as u64 {
+            break;
+        }
+    }
+    Ok(total)
 }
 
 #[cfg(test)]
diff --git a/frontend/package.json b/frontend/package.json
index 29b92af..01759be 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.128.19",
+  "version": "0.128.20",
   "private": true,
   "type": "module",
   "scripts": {