From cb757e7b69839998a4d5935974bd633b13241c88 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Mon, 13 Jul 2026 21:43:33 +0200 Subject: [PATCH] fix: index the running/leased_until arm of the job-lease query The lease predicate's `state='running' AND leased_until < now()` arm had no index, so every lease poll seq-scanned the whole crawler_jobs table (audit H2). migration 0039 adds a partial index on (leased_until) WHERE state='running'. Behavior-neutral; 29 lease tests green. Co-Authored-By: Claude Opus 4.8 --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- .../0039_crawler_jobs_running_lease_idx.sql | 18 ++++++++++++++++++ frontend/package.json | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 backend/migrations/0039_crawler_jobs_running_lease_idx.sql diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b3aea27..6657db8 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.128.18" +version = "0.128.19" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 089da93..480e3c8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.128.18" +version = "0.128.19" edition = "2021" default-run = "mangalord" diff --git a/backend/migrations/0039_crawler_jobs_running_lease_idx.sql b/backend/migrations/0039_crawler_jobs_running_lease_idx.sql new file mode 100644 index 0000000..f021483 --- /dev/null +++ b/backend/migrations/0039_crawler_jobs_running_lease_idx.sql @@ -0,0 +1,18 @@ +-- Index the crashed-worker-recovery arm of the job lease predicate. +-- +-- lease/lease_kinds match: +-- WHERE (state = 'pending' OR (state = 'running' AND leased_until < now())) +-- crawler_jobs_ready_idx (0016) is `ON (scheduled_at) WHERE state = 'pending'`, +-- so it covers only the pending arm. The `state = 'running' AND leased_until` +-- arm had no usable index, so Postgres could not BitmapOr the two arms and +-- degraded to a sequential scan of the ENTIRE crawler_jobs table — including all +-- done/dead rows not yet reaped — on every lease poll, by every worker, +-- continuously (audit H2, the headline performance finding). +-- +-- A partial index on leased_until over just the running rows makes the recovery +-- arm index-backed. `state = 'running'` is an immutable predicate (now() stays +-- in the query, not the index). The running set is tiny (in-flight jobs only), +-- so the index is cheap to maintain. +CREATE INDEX crawler_jobs_running_lease_idx + ON crawler_jobs (leased_until) + WHERE state = 'running'; diff --git a/frontend/package.json b/frontend/package.json index 47f71dd..29b92af 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.128.18", + "version": "0.128.19", "private": true, "type": "module", "scripts": {