fix: make the genre dedup migration collision-proof (0038)

Review caught a boot-crash risk: 0038's repoint-UPDATE could set two
non-canonical variant links of one manga to the same canonical id in a single
statement (NOT EXISTS sees the pre-statement snapshot) -> manga_genres PK
violation -> migration rollback -> startup failure. Replace with INSERT ... SELECT
DISTINCT ... ON CONFLICT DO NOTHING (collision-proof) then delete variants. Amended
in place (0038 is unpushed). Tested with the exact collision scenario.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -118,8 +118,10 @@ MAX_REQUEST_BYTES=209715200
 MAX_FILE_BYTES=20971520
 # Max page images accepted in one chapter upload. Bounds how many parts
 # the handler will stage before rejecting the request with 413, so a
-# client can't pin a worker with an unbounded page count. 0 disables the
+# client can't pin a worker with an unbounded page count. Default 2000.
-# cap. Default 2000.
+# Setting 0 disables THIS cap — the total is then bounded only by
+# MAX_REQUEST_BYTES (the whole-request body limit above), so leave 0 only
+# if you intend that body limit to be the sole backstop.
 MAX_PAGES_PER_CHAPTER=2000
 
 # ----- Crawler download safety -----
@@ -191,12 +193,19 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
 # exhausted) that trigger an automatic coordinated browser restart.
 # Default 3.
 CRAWLER_BROWSER_RESTART_THRESHOLD=3
-# Opt-in CDP Fetch interception that re-validates every headless-browser
+# CDP Fetch interception that re-validates every headless-browser
-# navigation/redirect against the SSRF check (blocks a scraped page that
+# navigation/redirect/subresource against the SSRF check (blocks a scraped page
-# redirects the browser to an internal target). Default false — enabling
+# that drives the browser — via a redirect OR page JS/subresource — to an
-# Fetch is a fragile hook in the crawler's critical path and is not
+# internal target such as the cloud metadata service or postgres). Default TRUE:
-# CI-verified; validate with a manual crawl before turning on.
+# with it off, only the top-level URL string is checked and page JS/subresources
-CRAWLER_SSRF_INTERCEPT=false
+# reach internal IPs (the reqwest SafeResolver does not cover Chromium's own
+# stack). The CDP Fetch hook can't be exercised in CI (no Chromium); before
+# relying on a fresh deploy, validate it doesn't wedge navigation:
+#   CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium \
+#     cargo test -p mangalord --test crawler_browser_smoke -- --ignored \
+#     ssrf_interception_does_not_wedge_allowed_navigation
+# Set to false only as a break-glass if the hook destabilizes a deployment.
+CRAWLER_SSRF_INTERCEPT=true
 # Path to a system Chromium binary. When set, the crawler skips the
 # bundled-fetcher download. Required on platforms without a usable
 # upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On
@@ -418,3 +427,14 @@ VISION_MANAGER_DATABASE_URL=
 # VISION_MEM_LOW_WATERMARK_PCT  used% >= this → inhibit starts (keep HIGH - LOW >= ~10 so the band beats jitter). Default 80.
 # VISION_MEM_YIELD_COOLDOWN     Seconds after a pressure-stop during which restart is refused regardless of backlog. Default 300.
 # VISION_MEM_POLL_INTERVAL      Mem sub-poll cadence, seconds (<= VISION_POLL_INTERVAL); catches spikes between backlog ticks. Default 5.
+
+# ---- Deployment resource bounds & exposure (docker-compose.yml) ----
+# Host interface the frontend port publishes on. Default 127.0.0.1 so SvelteKit
+# (plain HTTP) is reachable only by a host-local reverse proxy. Set 0.0.0.0 only
+# if your TLS terminator runs on a different host.
+FRONTEND_PUBLISH_ADDR=127.0.0.1
+# Per-container memory ceilings (docker mem_limit). Generous defaults; tune to
+# your host. The backend runs OCR + headless Chromium and is the heavy one.
+BACKEND_MEM_LIMIT=4g
+FRONTEND_MEM_LIMIT=512m
+POSTGRES_MEM_LIMIT=1g

backend/Cargo.lock

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.128.10"
+version = "0.128.25"
 dependencies = [
  "anyhow",
  "argon2",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.128.10"
+version = "0.128.25"
 edition = "2021"
 default-run = "mangalord"
 

backend/migrations/0038_genres_name_lower_unique.sql

@@ -0,0 +1,44 @@
+-- Enforce case-insensitive genre uniqueness, matching `authors` and `tags`
+-- (both got `UNIQUE (lower(name))` in 0009). `genres` only had a case-SENSITIVE
+-- `name UNIQUE`, but the crawler's `sync_genres` treats genres as
+-- case-insensitive (`WHERE lower(name) = lower($1)`) and can INSERT a new row
+-- from a source string. Under a race (or across ticks) two different-cased
+-- strings — "Isekai" vs "isekai" — could both miss the pre-check and both
+-- insert, since the exact-name unique doesn't collide. That leaves duplicate
+-- genre rows for one logical genre.
+
+-- Heal any pre-existing case-variant duplicates before adding the index, so
+-- `sqlx::migrate!` can't crash on a dirty table (mirrors 0031's pre-dedup).
+-- Canonical = lowest id per lower(name).
+
+-- 1. Ensure every manga linked to any case-variant also has the canonical link.
+--    INSERT ... ON CONFLICT DO NOTHING is collision-proof: SELECT DISTINCT
+--    collapses multiple variants of one manga to a single canonical row, and the
+--    ON CONFLICT absorbs a canonical link that already exists. (A prior
+--    UPDATE-repoint could set two non-canonical rows of the SAME manga to the
+--    same canonical id in a single statement and violate the manga_genres PK,
+--    rolling the migration back and wedging startup.)
+INSERT INTO manga_genres (manga_id, genre_id)
+SELECT DISTINCT mg.manga_id, k.keep_id
+  FROM manga_genres mg
+  JOIN genres g ON mg.genre_id = g.id
+  JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
+          FROM genres GROUP BY lower(name)) k
+    ON lower(g.name) = k.lname
+ WHERE g.id <> k.keep_id
+ON CONFLICT (manga_id, genre_id) DO NOTHING;
+
+-- 2. Every non-canonical link now has a canonical sibling — drop the variants.
+DELETE FROM manga_genres mg
+ USING genres g,
+       (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
+          FROM genres GROUP BY lower(name)) k
+ WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id;
+
+-- Remove the now-orphaned duplicate genre rows.
+DELETE FROM genres g
+ USING (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
+          FROM genres GROUP BY lower(name)) k
+ WHERE lower(g.name) = k.lname AND g.id <> k.keep_id;
+
+CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name));

backend/migrations/0039_crawler_jobs_running_lease_idx.sql

@@ -0,0 +1,18 @@
+-- Index the crashed-worker-recovery arm of the job lease predicate.
+--
+-- lease/lease_kinds match:
+--   WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
+-- crawler_jobs_ready_idx (0016) is `ON (scheduled_at) WHERE state = 'pending'`,
+-- so it covers only the pending arm. The `state = 'running' AND leased_until`
+-- arm had no usable index, so Postgres could not BitmapOr the two arms and
+-- degraded to a sequential scan of the ENTIRE crawler_jobs table — including all
+-- done/dead rows not yet reaped — on every lease poll, by every worker,
+-- continuously (audit H2, the headline performance finding).
+--
+-- A partial index on leased_until over just the running rows makes the recovery
+-- arm index-backed. `state = 'running'` is an immutable predicate (now() stays
+-- in the query, not the index). The running set is tiny (in-flight jobs only),
+-- so the index is cheap to maintain.
+CREATE INDEX crawler_jobs_running_lease_idx
+    ON crawler_jobs (leased_until)
+    WHERE state = 'running';

backend/migrations/0040_crawler_jobs_terminal_reap_idx.sql

@@ -0,0 +1,9 @@
+-- Index the retention reaper's scan. reap_terminal deletes
+--   WHERE state IN ('done','dead') AND updated_at < now() - interval
+-- which had no supporting index, so each daily sweep sequential-scanned the whole
+-- crawler_jobs table to find expired terminal rows (audit M5). A partial index on
+-- updated_at over just the terminal states makes the batched reaper's per-batch
+-- SELECT index-backed.
+CREATE INDEX crawler_jobs_terminal_reap_idx
+    ON crawler_jobs (updated_at)
+    WHERE state IN ('done', 'dead');

backend/src/analysis/daemon.rs

@@ -37,6 +37,22 @@ const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
 /// long enough not to hammer `/health` while vision is down.
 const READINESS_POLL: Duration = Duration::from_secs(2);
 
+/// Longest an idle analysis worker waits between lease polls, bounding the
+/// exponential [`idle_backoff`].
+const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
+
+/// Exponential idle backoff (1s, 2s, 4s … capped at [`IDLE_BACKOFF_CAP`]).
+/// `consecutive_empty` is the number of empty lease polls so far (0 on the first
+/// miss); reset to 0 the moment a job is leased. Replaces the old flat 1s sleep
+/// so an idle worker isn't firing a `SELECT … FOR UPDATE SKIP LOCKED` lease
+/// query every second (audit: analysis daemon idle poll). Mirrors the crawler
+/// daemon's backoff.
+fn idle_backoff(consecutive_empty: u32) -> Duration {
+    let cap = IDLE_BACKOFF_CAP.as_secs();
+    let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
+    Duration::from_secs(secs)
+}
+
 /// The unit of work: analyze one page. Implemented by
 /// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
 #[async_trait]
@@ -135,6 +151,9 @@ impl WorkerContext {
         // Last observed readiness, so we log only on transitions (not every
         // poll). `None` until the first probe.
         let mut was_ready: Option<bool> = None;
+        // Empty lease polls seen in a row, driving the idle backoff. Reset to 0
+        // the moment a job is leased.
+        let mut consecutive_empty: u32 = 0;
         loop {
             if self.cancel.is_cancelled() {
                 tracing::info!(worker = self.id, "analysis worker: shutdown");
@@ -178,11 +197,15 @@ impl WorkerContext {
                     }
                 };
             let Some(lease) = leases.into_iter().next() else {
-                if self.sleep_or_cancel(Duration::from_secs(1)).await {
+                let wait = idle_backoff(consecutive_empty);
+                consecutive_empty = consecutive_empty.saturating_add(1);
+                if self.sleep_or_cancel(wait).await {
                     return;
                 }
                 continue;
             };
+            // Leased work — drop back to a tight poll cadence.
+            consecutive_empty = 0;
             self.process_lease(lease).await;
         }
     }
@@ -510,6 +533,17 @@ mod tests {
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use tokio::net::TcpListener;
 
+    #[test]
+    fn idle_backoff_grows_exponentially_and_caps() {
+        assert_eq!(idle_backoff(0), Duration::from_secs(1));
+        assert_eq!(idle_backoff(1), Duration::from_secs(2));
+        assert_eq!(idle_backoff(2), Duration::from_secs(4));
+        assert_eq!(idle_backoff(4), Duration::from_secs(16));
+        // Caps at IDLE_BACKOFF_CAP (30s) and never overflows on large counts.
+        assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
+        assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
+    }
+
     /// Bind an ephemeral port that answers every request with `status_line`
     /// (e.g. `"200 OK"`), and return its `/health` URL. The probe under test
     /// only inspects the status code, so a zero-length body is enough.

backend/src/api/admin/analysis.rs

@@ -328,10 +328,21 @@ fn ensure_enabled(state: &AppState) -> AppResult<()> {
 async fn reenqueue(
     State(state): State<AppState>,
     admin: RequireAdmin,
-    body: Option<Json<ReenqueueBody>>,
+    raw: axum::body::Bytes,
 ) -> AppResult<Json<ReenqueueResponse>> {
     ensure_enabled(&state)?;
-    let body = body.map(|b| b.0).unwrap_or_default();
+    // An ABSENT/empty body means the default "All" scope. A PRESENT body must be
+    // valid JSON — `Option<Json<T>>` would silently collapse a malformed body to
+    // None and run the full-library default the caller never intended, so parse
+    // the raw bytes ourselves and 422 on a real parse error.
+    let body: ReenqueueBody = if raw.iter().all(u8::is_ascii_whitespace) {
+        ReenqueueBody::default()
+    } else {
+        serde_json::from_slice(&raw).map_err(|e| AppError::ValidationFailed {
+            message: "reenqueue body is not valid JSON".into(),
+            details: json!({ "body": e.to_string() }),
+        })?
+    };
 
     // Resolve the scope. manga_id and chapter_id are mutually exclusive;
     // an unknown target is a 404 rather than a silent zero-enqueue.

backend/src/api/files.rs

@@ -118,9 +118,20 @@ async fn serve_thumbnail(
         Err(e) => return Err(e.into()),
     };
     // Resizing is CPU-bound; keep it off the async worker threads.
-    let thumb = tokio::task::spawn_blocking(move || make_thumbnail(&original, width, fmt))
+    let thumb = match tokio::task::spawn_blocking(move || make_thumbnail(&original, width, fmt))
         .await
-        .map_err(|e| AppError::Other(anyhow::anyhow!("thumbnail task join: {e}")))??;
+        .map_err(|e| AppError::Other(anyhow::anyhow!("thumbnail task join: {e}")))?
+    {
+        Ok(t) => t,
+        Err(e) => {
+            // The blob's magic bytes passed upload's sniff but the body won't
+            // decode (corrupt/truncated, or an encoder feature we don't support).
+            // Don't 500 the reader — fall back to streaming the original, which
+            // serves without decoding.
+            tracing::warn!(key, error = %format!("{e:#}"), "thumbnail decode failed; serving original");
+            return serve_original(state, key).await;
+        }
+    };
 
     // Best-effort cache; a write failure just means we regenerate next time.
     let _ = state.storage.put(&derived, &thumb).await;

backend/src/api/mangas.rs

@@ -409,6 +409,17 @@ async fn put_cover(
     let old_key = repo::manga::get(&state.db, id).await?.cover_image_path;
     let new_key = format!("mangas/{}/cover.{}", id, img.ext);
     state.storage.put(&new_key, &img.bytes).await?;
+    // Cover keys are reused (mangas/{id}/cover.{ext}), so a same-extension
+    // replacement overwrites the blob at an existing key — drop any thumbnails
+    // cached for it so we don't serve a stale variant of the old cover.
+    crate::api::files::purge_thumbnails(state.storage.as_ref(), &new_key).await;
+
+    // Commit the DB pointer to the new blob BEFORE removing the old one. If we
+    // deleted the old blob first and this write then failed, the row would point
+    // at a deleted blob (a cover that 404s) and the new blob would be orphaned.
+    // Ordering it first means a failure here leaves the row pointing at the
+    // still-present old blob; only a harmless orphan (new blob) can result.
+    repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
 
     if let Some(prev) = old_key.as_deref() {
         if prev != new_key {
@@ -423,12 +434,6 @@ async fn put_cover(
             crate::api::files::purge_thumbnails(state.storage.as_ref(), prev).await;
         }
     }
-    // Cover keys are reused (mangas/{id}/cover.{ext}), so a same-extension
-    // replacement overwrites the blob at an existing key — drop any thumbnails
-    // cached for it so we don't serve a stale variant of the old cover.
-    crate::api::files::purge_thumbnails(state.storage.as_ref(), &new_key).await;
-
-    repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
     Ok(Json(repo::manga::get_detail(&state.db, id).await?))
 }
 
@@ -446,12 +451,15 @@ async fn delete_cover(
     }
     require_can_edit(&state, id, user.id, admin_via_session(&session)).await?;
     if let Some(key) = repo::manga::get(&state.db, id).await?.cover_image_path {
+        // Clear the DB pointer BEFORE deleting the blob, so a storage-delete
+        // failure can't leave the row pointing at a removed blob. A failure
+        // after the clear only orphans the blob (harmless).
+        repo::manga::clear_cover_image_path(&state.db, id).await?;
         match state.storage.delete(&key).await {
             Ok(()) | Err(StorageError::NotFound) => {}
             Err(e) => return Err(e.into()),
         }
         crate::api::files::purge_thumbnails(state.storage.as_ref(), &key).await;
-        repo::manga::clear_cover_image_path(&state.db, id).await?;
     }
     Ok(Json(repo::manga::get_detail(&state.db, id).await?))
 }

backend/src/bin/crawler.rs

@@ -133,13 +133,16 @@ async fn main() -> anyhow::Result<()> {
     if let Some(proxy) = &proxy_url {
         http_builder = http_builder
             .proxy(reqwest::Proxy::all(proxy).with_context(|| format!("parse proxy URL: {proxy}"))?);
-    } else {
+    }
-        // DNS-rebinding guard, direct (unproxied) path only — see the matching
+    // DNS-rebinding guard: attached on the direct path AND on http(s) proxies
-        // note in `spawn_crawler_daemon` (app.rs). With a `socks5h://` proxy the
+    // (reqwest resolves the target itself there), skipped only for SOCKS proxies
-        // target is resolved by the proxy, so this resolver would only ever
+    // where the proxy resolves the target. Use the shared predicate so this CLI
-        // reject the proxy's own private-IP host (e.g. `tor`), breaking every
+    // and the daemon (app.rs) stay in lockstep — previously the CLI attached the
-        // fetch for no security gain.
+    // resolver only on the fully-direct path, so an http(s) proxy silently lost
-        http_builder = http_builder.dns_resolver(mangalord::crawler::safety::safe_dns_resolver());
+    // the rebinding guard.
+    if mangalord::crawler::safety::should_attach_safe_resolver(proxy_url.as_deref()) {
+        http_builder =
+            http_builder.dns_resolver(mangalord::crawler::safety::safe_dns_resolver());
     }
     let http = http_builder.build().context("build http client")?;
 

backend/src/config.rs

@@ -66,8 +66,10 @@ pub struct UploadConfig {
     pub max_file_bytes: usize,
     /// Max page images accepted in one chapter upload. Bounds how many
     /// parts the handler will stage before giving up, so a client can't
-    /// pin a worker streaming an unbounded page count. `0` disables the
+    /// pin a worker streaming an unbounded page count. `0` disables THIS
-    /// cap. Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
+    /// cap — the total is then bounded only by `max_request_bytes` (the
+    /// whole-request body limit), which stays the backstop either way.
+    /// Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
     pub max_pages_per_chapter: usize,
 }
 
@@ -520,12 +522,15 @@ pub struct CrawlerConfig {
     /// exhausted) that trigger an automatic coordinated browser restart.
     /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
     pub browser_restart_threshold: u32,
-    /// Opt-in CDP `Fetch` interception that re-validates every headless-browser
+    /// CDP `Fetch` interception that re-validates every headless-browser
-    /// navigation/redirect against the SSRF check (blocks a scraped page that
+    /// navigation/redirect/subresource against the SSRF check. Default `true`:
-    /// redirects the browser to an internal target). Default `false`: enabling
+    /// with it off, only the top-level URL string is validated, so a scraped
-    /// `Fetch` is a fragile hook in the crawler's critical path and the wiring
+    /// page's JS/subresources (which use Chromium's own network stack, not the
-    /// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl
+    /// reqwest `SafeResolver`) can reach internal targets like the cloud
-    /// before turning on. `CRAWLER_SSRF_INTERCEPT`.
+    /// metadata service or postgres. The Fetch hook can't be exercised in CI (no
+    /// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation`
+    /// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`;
+    /// set `false` only as a break-glass if the hook destabilizes a deployment.
     pub ssrf_intercept: bool,
 }
 
@@ -559,7 +564,7 @@ impl Default for CrawlerConfig {
             job_timeout: Duration::from_secs(600),
             metadata_max_consecutive_failures: 10,
             browser_restart_threshold: 3,
-            ssrf_intercept: false,
+            ssrf_intercept: true,
         }
     }
 }
@@ -725,7 +730,7 @@ impl CrawlerConfig {
             ) as u32,
             browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
                 as u32,
-            ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false),
+            ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true),
         })
     }
 }
@@ -1091,6 +1096,13 @@ mod tests {
         "BACKEND_URL",
         "BACKEND_PROXY_TIMEOUT_MS",
         "VISION_MANAGER_DATABASE_URL",
+        // Compose-level deployment knobs (port publish interface + per-container
+        // memory ceilings) — consumed by docker-compose.yml itself, never read
+        // by the backend process, so they don't belong in its environment block.
+        "FRONTEND_PUBLISH_ADDR",
+        "BACKEND_MEM_LIMIT",
+        "FRONTEND_MEM_LIMIT",
+        "POSTGRES_MEM_LIMIT",
     ];
 
     /// Keys whose compose RHS is intentionally NOT a `${KEY...}`

backend/src/crawler/content.rs

@@ -595,15 +595,22 @@ pub(crate) async fn persist_pages(
     let mut tx = db.begin().await.context("open chapter sync tx")?;
     let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
     let mut page_numbers: Vec<i32> = Vec::with_capacity(stored.len());
+    // Pages whose row already existed and was overwritten by this re-crawl.
+    // Their image changed but pages.id (and any analysis keyed to it) survived,
+    // so the derived analysis is now stale and must be invalidated below.
+    let mut updated_page_ids: Vec<Uuid> = Vec::new();
     for page in stored {
-        let (id,): (Uuid,) = sqlx::query_as(
+        // `xmax <> 0` on the RETURNING row distinguishes a conflict UPDATE
+        // (existing row overwritten) from a fresh INSERT — the standard upsert
+        // idiom. Cast through text/bigint since there is no direct xid<>int op.
+        let (id, was_update): (Uuid, bool) = sqlx::query_as(
             "INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes)
              VALUES ($1, $2, $3, $4, $5)
              ON CONFLICT (chapter_id, page_number) DO UPDATE
              SET storage_key = EXCLUDED.storage_key,
                  content_type = EXCLUDED.content_type,
                  size_bytes = EXCLUDED.size_bytes
-             RETURNING id",
+             RETURNING id, (xmax::text::bigint <> 0)",
         )
         .bind(chapter_id)
         .bind(page.page_number)
@@ -615,6 +622,32 @@ pub(crate) async fn persist_pages(
         .with_context(|| format!("insert page row {}", page.page_number))?;
         page_ids.push(id);
         page_numbers.push(page.page_number);
+        if was_update {
+            updated_page_ids.push(id);
+        }
+    }
+    // Invalidate stale analysis for re-crawled (overwritten) pages. Storage keys
+    // are deterministic per page number, so the row survives the upsert and its
+    // page_analysis / OCR / auto-tags / content-warnings would otherwise reflect
+    // the OLD image — poisoning search results and the derived
+    // manga_content_warnings. Clearing them (the DELETE on page_content_warnings
+    // fires the mcw_* triggers so manga_content_warnings recomputes) drops the
+    // pages back to "unanalyzed" so the enqueue below — and the admin
+    // only-unanalyzed backfill — re-derive them. Newly inserted pages have no
+    // prior analysis, so they're untouched.
+    if !updated_page_ids.is_empty() {
+        for table in [
+            "page_ocr_text",
+            "page_auto_tags",
+            "page_content_warnings",
+            "page_analysis",
+        ] {
+            sqlx::query(&format!("DELETE FROM {table} WHERE page_id = ANY($1::uuid[])"))
+                .bind(&updated_page_ids)
+                .execute(&mut *tx)
+                .await
+                .with_context(|| format!("invalidate {table} for re-crawled pages"))?;
+        }
     }
     // Drop any rows left over from a prior, larger crawl of this chapter
     // (re-crawl that now yields fewer pages). Without this the stale rows —
@@ -1200,4 +1233,91 @@ mod tests {
         assert_eq!(fetch_n, 1);
         assert!(format!("{err:#}").contains("nav timeout"));
     }
+
+    #[sqlx::test(migrations = "./migrations")]
+    async fn persist_pages_invalidates_stale_analysis_on_recrawl_update(pool: PgPool) {
+        // Page storage keys are deterministic per page number, so a re-crawl
+        // overwrites the image but keeps pages.id. Analysis keyed to page_id
+        // would otherwise stay stale (old OCR/search_doc/warnings). persist_pages
+        // must clear the derived analysis for updated pages so it re-derives.
+        let manga_id = Uuid::new_v4();
+        let chapter_id = Uuid::new_v4();
+        sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
+            .bind(manga_id)
+            .execute(&pool)
+            .await
+            .unwrap();
+        sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
+            .bind(chapter_id)
+            .bind(manga_id)
+            .execute(&pool)
+            .await
+            .unwrap();
+
+        let v1 = vec![StoredPage {
+            page_number: 1,
+            storage_key: "k/0001.jpg".into(),
+            content_type: "image/jpeg".into(),
+            size_bytes: 10,
+        }];
+        persist_pages(&pool, chapter_id, &v1, false).await.unwrap();
+        let page_id: Uuid = sqlx::query_scalar(
+            "SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
+        )
+        .bind(chapter_id)
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+
+        // Simulate a completed prior analysis pass for that page.
+        sqlx::query(
+            "INSERT INTO page_analysis (page_id, status, is_nsfw, analyzed_at) \
+             VALUES ($1, 'done', true, now())",
+        )
+        .bind(page_id)
+        .execute(&pool)
+        .await
+        .unwrap();
+        sqlx::query("INSERT INTO page_ocr_text (page_id, kind, text) VALUES ($1, 'speech', 'stale')")
+            .bind(page_id)
+            .execute(&pool)
+            .await
+            .unwrap();
+        sqlx::query("INSERT INTO page_content_warnings (page_id, warning) VALUES ($1, 'gore')")
+            .bind(page_id)
+            .execute(&pool)
+            .await
+            .unwrap();
+
+        // Re-crawl page 1 with a new image (same deterministic key, new size).
+        let v2 = vec![StoredPage {
+            page_number: 1,
+            storage_key: "k/0001.jpg".into(),
+            content_type: "image/jpeg".into(),
+            size_bytes: 999,
+        }];
+        persist_pages(&pool, chapter_id, &v2, false).await.unwrap();
+
+        // The page row survived (deterministic key).
+        let same_id: Uuid = sqlx::query_scalar(
+            "SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
+        )
+        .bind(chapter_id)
+        .fetch_one(&pool)
+        .await
+        .unwrap();
+        assert_eq!(same_id, page_id, "re-crawl keeps the page id");
+
+        // ...but its stale analysis is cleared so it gets re-derived.
+        for table in ["page_analysis", "page_ocr_text", "page_content_warnings"] {
+            let n: i64 = sqlx::query_scalar(&format!(
+                "SELECT COUNT(*) FROM {table} WHERE page_id = $1"
+            ))
+            .bind(page_id)
+            .fetch_one(&pool)
+            .await
+            .unwrap();
+            assert_eq!(n, 0, "{table} must be invalidated when the re-crawl updates the page");
+        }
+    }
 }

backend/src/crawler/intercept.rs

@@ -82,6 +82,11 @@ pub(crate) fn verdict(url: &str) -> Verdict {
     };
     match parsed.scheme() {
         "http" | "https" => {}
+        // Non-http(s) subresource schemes: block the ones that can reach the
+        // local filesystem or pivot to another protocol; continue the rest
+        // (data:/blob:/about: and Chrome-internal schemes) which legitimate page
+        // loads depend on — blocking those would wedge navigation.
+        "file" | "ftp" | "gopher" => return Verdict::Block,
         _ => return Verdict::Allow,
     }
     // Literal private IP / localhost / missing host → block (string check).
@@ -210,6 +215,15 @@ mod tests {
         }
     }
 
+    #[test]
+    fn verdict_blocks_dangerous_non_http_schemes() {
+        // file:/ftp:/gopher: can reach the local filesystem or pivot protocols;
+        // block them explicitly rather than falling through to Allow.
+        for url in ["file:///etc/passwd", "ftp://internal/x", "gopher://internal:70/1"] {
+            assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
+        }
+    }
+
     #[test]
     fn verdict_blocks_private_ip_literals_and_localhost() {
         for url in [

backend/src/crawler/jobs.rs

@@ -495,15 +495,33 @@ pub async fn reap_terminal(pool: &PgPool, retention_days: u32) -> sqlx::Result<u
     if retention_days == 0 {
         return Ok(0);
     }
-    let result = sqlx::query(
+    // Delete in bounded batches rather than one statement. A single unbatched
-        "DELETE FROM crawler_jobs \
+    // DELETE over an unbounded backlog takes a long-held lock and one huge
-          WHERE state IN ('done', 'dead') \
+    // transaction; batching keeps each delete short (index-backed by
-            AND updated_at < now() - ($1::bigint || ' days')::interval",
+    // crawler_jobs_terminal_reap_idx, 0040) so the reaper never pins the table
-    )
+    // (audit M5). Loop until a batch comes back short.
-    .bind(retention_days as i64)
+    const BATCH: i64 = 5_000;
-    .execute(pool)
+    let mut total: u64 = 0;
-    .await?;
+    loop {
-    Ok(result.rows_affected())
+        let result = sqlx::query(
+            "DELETE FROM crawler_jobs \
+              WHERE ctid IN ( \
+                    SELECT ctid FROM crawler_jobs \
+                     WHERE state IN ('done', 'dead') \
+                       AND updated_at < now() - ($1::bigint || ' days')::interval \
+                     LIMIT $2 )",
+        )
+        .bind(retention_days as i64)
+        .bind(BATCH)
+        .execute(pool)
+        .await?;
+        let n = result.rows_affected();
+        total += n;
+        if n < BATCH as u64 {
+            break;
+        }
+    }
+    Ok(total)
 }
 
 #[cfg(test)]

backend/src/crawler/safety.rs

@@ -507,6 +507,26 @@ mod tests {
         DownloadAllowlist::new().allow(host)
     }
 
+    #[test]
+    fn rejects_alternate_ipv4_encodings_of_loopback() {
+        // The `url` crate normalizes special-scheme IPv4 hosts per WHATWG, so
+        // decimal/hex/octal/short forms all collapse to 127.0.0.1 and must be
+        // rejected. This normalization is load-bearing (it's what stops these
+        // encodings smuggling past the literal-IP check) but was untested — pin
+        // it so a future URL-parser swap can't silently reopen the hole.
+        for url in [
+            "http://2130706433/", // decimal 127.0.0.1
+            "http://0x7f000001/",  // hex
+            "http://0177.0.0.1/",  // octal first octet
+            "http://127.1/",       // short form
+        ] {
+            assert!(
+                ensure_public_target(url).is_err(),
+                "{url} normalizes to loopback and must be rejected"
+            );
+        }
+    }
+
     #[test]
     fn allow_any_admits_arbitrary_public_host() {
         // Operators who can't pre-enumerate a numbered-CDN fleet

backend/src/error.rs

@@ -207,4 +207,68 @@ mod tests {
             "text_search_not_yet_supported"
         );
     }
+
+    #[test]
+    fn status_mapping_is_stable() {
+        // Pin the code -> HTTP status mapping so a refactor of into_response
+        // can't silently change a status (e.g. 501 -> 500). The 501 NotImplemented
+        // path and a deterministic 429 Retry-After were previously unexercised
+        // (audit GAP B).
+        let cases: Vec<(AppError, StatusCode)> = vec![
+            (AppError::NotFound, StatusCode::NOT_FOUND),
+            (AppError::InvalidInput("x".into()), StatusCode::BAD_REQUEST),
+            (AppError::Unauthenticated, StatusCode::UNAUTHORIZED),
+            (AppError::Forbidden, StatusCode::FORBIDDEN),
+            (AppError::Conflict("x".into()), StatusCode::CONFLICT),
+            (AppError::PayloadTooLarge("x".into()), StatusCode::PAYLOAD_TOO_LARGE),
+            (
+                AppError::UnsupportedMediaType("x".into()),
+                StatusCode::UNSUPPORTED_MEDIA_TYPE,
+            ),
+            (
+                AppError::ServiceUnavailable("x".into()),
+                StatusCode::SERVICE_UNAVAILABLE,
+            ),
+            (
+                AppError::ValidationFailed {
+                    message: "x".into(),
+                    details: json!({}),
+                },
+                StatusCode::UNPROCESSABLE_ENTITY,
+            ),
+            (
+                AppError::NotImplemented {
+                    code: "c",
+                    message: "m",
+                },
+                StatusCode::NOT_IMPLEMENTED,
+            ),
+            (
+                AppError::TooManyRequests {
+                    retry_after_secs: None,
+                },
+                StatusCode::TOO_MANY_REQUESTS,
+            ),
+            (
+                AppError::Other(anyhow::anyhow!("boom")),
+                StatusCode::INTERNAL_SERVER_ERROR,
+            ),
+        ];
+        for (err, want) in cases {
+            let code = err.code();
+            assert_eq!(err.into_response().status(), want, "status for code {code}");
+        }
+
+        // A 429 with retry_after carries the Retry-After header deterministically
+        // (without needing a live rate limiter).
+        let resp = AppError::TooManyRequests {
+            retry_after_secs: Some(7),
+        }
+        .into_response();
+        assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
+        assert_eq!(
+            resp.headers().get(axum::http::header::RETRY_AFTER).unwrap(),
+            "7"
+        );
+    }
 }

backend/src/repo/crawler.rs

@@ -247,10 +247,13 @@ async fn sync_genres(
         let genre_id = match existing {
             Some((id,)) => id,
             None => {
+                // Conflict on lower(name) (0038) not name, so a racing insert of
+                // a differently-cased variant resolves to the existing row
+                // instead of raising a unique violation.
                 let (id,): (Uuid,) = sqlx::query_as(
                     r#"
                     INSERT INTO genres (name) VALUES ($1)
-                    ON CONFLICT (name) DO UPDATE SET name = genres.name
+                    ON CONFLICT (lower(name)) DO UPDATE SET name = genres.name
                     RETURNING id
                     "#,
                 )

backend/tests/api_admin_analysis.rs

@@ -189,6 +189,51 @@ async fn reenqueue_returns_503_when_analysis_disabled(pool: PgPool) {
     assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
 }
 
+#[sqlx::test(migrations = "./migrations")]
+async fn reenqueue_rejects_a_malformed_body(pool: PgPool) {
+    // A present-but-malformed JSON body must be a 422 — NOT silently coerced to
+    // an absent body, which would run the default full "All" scope the caller
+    // never asked for and enqueue jobs across the whole library.
+    let h = common::harness_with_analysis(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    seed_pages(&pool, 2).await;
+
+    let resp = h
+        .app
+        .clone()
+        .oneshot(common::post_raw_with_cookie(
+            "/api/v1/admin/analysis/reenqueue",
+            "{ not valid json",
+            &cookie,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
+    assert_eq!(analyze_job_count(&pool).await, 0, "a rejected body enqueues nothing");
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn reenqueue_treats_empty_body_as_default_all_scope(pool: PgPool) {
+    // An absent/empty body is still valid: it means the default "All" scope.
+    let h = common::harness_with_analysis(pool.clone());
+    let cookie = seed_admin(&pool, &h.app).await;
+    seed_pages(&pool, 2).await;
+
+    let resp = h
+        .app
+        .clone()
+        .oneshot(common::post_raw_with_cookie(
+            "/api/v1/admin/analysis/reenqueue",
+            "",
+            &cookie,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::OK);
+    let body = common::body_json(resp).await;
+    assert_eq!(body["enqueued"], 2, "empty body backfills the whole library");
+}
+
 #[sqlx::test(migrations = "./migrations")]
 async fn reenqueue_backfills_existing_pages(pool: PgPool) {
     let h = common::harness_with_analysis(pool.clone());

backend/tests/api_auth.rs

@@ -9,6 +9,73 @@ fn creds(username: &str) -> serde_json::Value {
     json!({ "username": username, "password": "hunter2hunter2" })
 }
 
+/// A login request carrying a chosen `X-Forwarded-For`. Empty password so the
+/// handler consumes a rate-limit token then short-circuits with 400 before any
+/// argon2/DB work — the burst drains near-instantly regardless of CI load.
+fn login_from(xff: &str) -> axum::http::Request<axum::body::Body> {
+    axum::http::Request::builder()
+        .method("POST")
+        .uri("/api/v1/auth/login")
+        .header(header::CONTENT_TYPE, "application/json")
+        .header("x-forwarded-for", xff)
+        .body(axum::body::Body::from(
+            json!({ "username": "victim", "password": "" }).to_string(),
+        ))
+        .unwrap()
+}
+
+// The per-IP rate limiter's soundness hinges on one gate: X-Forwarded-For is
+// honored ONLY when AUTH_TRUSTED_PROXY is set. These two tests pin both sides of
+// that gate at the request level (the pure parser is unit-tested separately).
+
+#[sqlx::test(migrations = "./migrations")]
+async fn trusted_proxy_gives_each_forwarded_ip_its_own_bucket(pool: PgPool) {
+    let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, true);
+    // Drain IP A's bucket (per_sec=1, burst=2) until it 429s.
+    let mut a_saw_429 = false;
+    for _ in 0..8 {
+        let resp = h.app.clone().oneshot(login_from("203.0.113.10")).await.unwrap();
+        if resp.status() == StatusCode::TOO_MANY_REQUESTS {
+            a_saw_429 = true;
+            break;
+        }
+    }
+    assert!(a_saw_429, "IP A must be rate-limited after draining its own bucket");
+    // A different forwarded IP has an independent bucket — its first hit is not 429.
+    let resp_b = h.app.clone().oneshot(login_from("198.51.100.20")).await.unwrap();
+    assert_ne!(
+        resp_b.status(),
+        StatusCode::TOO_MANY_REQUESTS,
+        "a distinct X-Forwarded-For hop must get its own bucket, not A's drained one"
+    );
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn untrusted_proxy_ignores_forwarded_ip_and_shares_one_bucket(pool: PgPool) {
+    let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, false);
+    // Every request carries a DISTINCT (spoofed) X-Forwarded-For, but the backend
+    // doesn't trust it — so they all fall back to the single global bucket, which
+    // drains and starts returning 429. If XFF were wrongly honored here, each
+    // unique IP would get a fresh bucket and none of these would ever 429.
+    let mut saw_429 = false;
+    for i in 0..12 {
+        let resp = h
+            .app
+            .clone()
+            .oneshot(login_from(&format!("10.0.0.{i}")))
+            .await
+            .unwrap();
+        if resp.status() == StatusCode::TOO_MANY_REQUESTS {
+            saw_429 = true;
+            break;
+        }
+    }
+    assert!(
+        saw_429,
+        "with trusted_proxy off, spoofed X-Forwarded-For must NOT dodge the shared limiter"
+    );
+}
+
 #[sqlx::test(migrations = "./migrations")]
 async fn register_creates_user_and_sets_session_cookie(pool: PgPool) {
     let h = common::harness(pool);

backend/tests/api_files.rs

@@ -55,3 +55,20 @@ async fn image_blobs_are_served_inline(pool: sqlx::PgPool) {
         "images must render inline, not download"
     );
 }
+
+#[sqlx::test(migrations = "./migrations")]
+async fn corrupt_image_thumbnail_falls_back_to_original_not_500(pool: sqlx::PgPool) {
+    // A blob with valid PNG magic bytes but an undecodable body passes upload's
+    // magic-byte sniff; a ?w= thumbnail request then tries to decode it. That
+    // must not 500 the reader — fall back to streaming the original bytes.
+    let h = common::harness(pool);
+    write_blob(&h, "misc/corrupt.png", &common::fake_png_bytes());
+
+    let resp = h
+        .app
+        .oneshot(common::get("/api/v1/files/misc/corrupt.png?w=320"))
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), StatusCode::OK, "corrupt thumbnail must not 500");
+    assert_eq!(resp.headers().get("content-type").unwrap(), "image/png");
+}

backend/tests/common/mod.rs

@@ -148,6 +148,27 @@ pub fn harness_with_auth_rate_limit(
     harness_with_auth_config(pool, storage, storage_dir, auth)
 }
 
+/// Like [`harness_with_auth_rate_limit`] but also sets whether the backend
+/// trusts `X-Forwarded-For` (`AUTH_TRUSTED_PROXY`). Used to prove the per-IP
+/// rate-limit gate: with `trusted_proxy` on, distinct XFF hops get independent
+/// buckets; with it off, XFF is ignored and everything shares the global bucket.
+pub fn harness_with_auth_rate_limit_proxy(
+    pool: PgPool,
+    per_sec: u32,
+    burst: u32,
+    trusted_proxy: bool,
+) -> Harness {
+    let storage_dir = tempfile::tempdir().expect("tempdir");
+    let storage = Arc::new(LocalStorage::new(storage_dir.path()));
+    let auth = AuthConfig {
+        cookie_secure: false,
+        trusted_proxy,
+        rate_limit: mangalord::auth::rate_limit::RateLimitConfig { per_sec, burst },
+        ..AuthConfig::default()
+    };
+    harness_with_auth_config(pool, storage, storage_dir, auth)
+}
+
 /// Like [`harness`] but slots a caller-supplied [`ResyncService`] stub
 /// into `AppState.resync`. Used by the admin resync tests so the
 /// endpoint path is exercised without standing up a real Chromium.
@@ -475,6 +496,20 @@ pub fn post_json_with_cookie(
         .unwrap()
 }
 
+/// Like [`post_json_with_cookie`] but sends a raw (possibly malformed) body,
+/// for exercising body-parse error handling. Content-Type stays JSON and the
+/// CSRF Origin is attached so the request reaches the handler.
+pub fn post_raw_with_cookie(uri: &str, body: &str, cookie: &str) -> Request<Body> {
+    Request::builder()
+        .method("POST")
+        .uri(uri)
+        .header(header::CONTENT_TYPE, "application/json")
+        .header(header::COOKIE, cookie)
+        .header(header::ORIGIN, TEST_ORIGIN)
+        .body(Body::from(body.to_string()))
+        .unwrap()
+}
+
 /// Same as [`post_json_with_cookie`] but also attaches `Origin` (and
 /// optionally `Referer`) headers. Used by the admin CSRF tests to drive
 /// the cross-origin reject + allowed-origin accept paths.

backend/tests/crawler_sync.rs

@@ -674,6 +674,110 @@ async fn arbitrary_genres_from_source_get_inserted(pool: PgPool) {
     assert_eq!(webtoons_count.0, 1, "case-insensitive lookup reuses the existing row");
 }
 
+#[sqlx::test(migrations = "./migrations")]
+async fn genre_dedup_survives_a_manga_linked_to_two_variants(pool: PgPool) {
+    // Regression for a migration-0038 collision: a manga linked to TWO
+    // non-canonical case-variants of one genre. A repoint-UPDATE would set both
+    // rows to the canonical id in one statement -> manga_genres PK violation ->
+    // migration rollback -> boot failure. #[sqlx::test] migrates a clean DB, so
+    // recreate the dirty pre-index state and replay 0038's healing statements;
+    // they must complete and leave exactly the one canonical link. (Mirrors the
+    // healing SQL in migrations/0038_genres_name_lower_unique.sql.)
+    let manga = Uuid::new_v4();
+    sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
+        .bind(manga)
+        .execute(&pool)
+        .await
+        .unwrap();
+
+    // Drop the live guard so we can plant case-variant genres.
+    sqlx::query("DROP INDEX genres_name_lower_uniq").execute(&pool).await.unwrap();
+    let keep = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
+    let dup2 = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
+    let dup3 = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
+    for (id, name) in [(keep, "Zzz"), (dup2, "zzz"), (dup3, "ZZZ")] {
+        sqlx::query("INSERT INTO genres (id, name) VALUES ($1, $2)")
+            .bind(id)
+            .bind(name)
+            .execute(&pool)
+            .await
+            .unwrap();
+    }
+    // The manga links to the two NON-canonical variants, not the canonical one.
+    for gid in [dup2, dup3] {
+        sqlx::query("INSERT INTO manga_genres (manga_id, genre_id) VALUES ($1, $2)")
+            .bind(manga)
+            .bind(gid)
+            .execute(&pool)
+            .await
+            .unwrap();
+    }
+
+    // Step 1 — collision-proof canonical-link backfill (the crux of the fix).
+    sqlx::query(
+        "INSERT INTO manga_genres (manga_id, genre_id) \
+         SELECT DISTINCT mg.manga_id, k.keep_id \
+           FROM manga_genres mg JOIN genres g ON mg.genre_id = g.id \
+           JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
+                   FROM genres GROUP BY lower(name)) k ON lower(g.name) = k.lname \
+          WHERE g.id <> k.keep_id \
+         ON CONFLICT (manga_id, genre_id) DO NOTHING",
+    )
+    .execute(&pool)
+    .await
+    .expect("collision-proof INSERT must not raise a manga_genres PK violation");
+    // Step 2 — drop the non-canonical links.
+    sqlx::query(
+        "DELETE FROM manga_genres mg USING genres g, \
+         (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
+            FROM genres GROUP BY lower(name)) k \
+         WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id",
+    )
+    .execute(&pool)
+    .await
+    .unwrap();
+    // Step 3 — remove orphaned duplicate genres.
+    sqlx::query(
+        "DELETE FROM genres g USING \
+         (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
+            FROM genres GROUP BY lower(name)) k \
+         WHERE lower(g.name) = k.lname AND g.id <> k.keep_id",
+    )
+    .execute(&pool)
+    .await
+    .unwrap();
+
+    // Exactly the canonical link remains, and the guard re-creates cleanly.
+    let links: Vec<Uuid> =
+        sqlx::query_scalar("SELECT genre_id FROM manga_genres WHERE manga_id = $1")
+            .bind(manga)
+            .fetch_all(&pool)
+            .await
+            .unwrap();
+    assert_eq!(links, vec![keep], "manga keeps exactly the canonical genre link");
+    sqlx::query("CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name))")
+        .execute(&pool)
+        .await
+        .expect("no residual case-variant duplicates remain");
+}
+
+#[sqlx::test(migrations = "./migrations")]
+async fn genres_reject_case_variant_duplicates_at_the_db(pool: PgPool) {
+    // The sequential pre-check in sync_genres dedups the common case, but two
+    // concurrent inserts could each miss it. The lower(name) unique index (0038)
+    // is the race backstop: a case variant of an existing genre must be rejected
+    // by the DB itself. "Action" is seeded by 0009.
+    let dup = sqlx::query("INSERT INTO genres (name) VALUES ('action')")
+        .execute(&pool)
+        .await;
+    let err = dup.expect_err("a case-variant of a seeded genre must violate the unique index");
+    let msg = err.to_string();
+    assert!(
+        msg.contains("genres_name_lower_uniq") || msg.contains("unique") || msg.contains("duplicate"),
+        "expected a unique-violation, got: {msg}"
+    );
+}
+
 /// User-attached tags (rows with non-NULL `added_by` in `manga_tags`)
 /// must survive a crawler upsert. The crawler owns source-attached tags
 /// (added_by IS NULL); user attachments are owned by the user who made

docker-compose.yml

@@ -23,6 +23,12 @@ services:
     # takes the whole stack offline. Match the policy already set on
     # tor / docker-socket-proxy / vision-manager.
     restart: unless-stopped
+    # Bound Postgres so a runaway query/backlog can't consume all host RAM.
+    # Tune to your host via .env (should comfortably exceed shared_buffers +
+    # work_mem * max_connections).
+    mem_limit: ${POSTGRES_MEM_LIMIT:-1g}
+    security_opt:
+      - no-new-privileges:true
 
   tor:
     # SOCKS5 proxy for the crawler, plus a control port so the backend
@@ -112,7 +118,7 @@ services:
       CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600}
       CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10}
       CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3}
-      CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false}
+      CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-true}
       # Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps
       # the in-process scheduler off; the dashboard force-resync still works.
       CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true}
@@ -197,6 +203,13 @@ services:
     expose:
       - "8080"
     restart: unless-stopped
+    # Bound the heaviest consumer: the backend hosts the OCR/analysis worker and
+    # drives a headless Chromium crawler, either of which can balloon and
+    # OOM-kill the host (taking Postgres with it) with no limit. Tune to your
+    # host via .env — this default is a generous ceiling, not a target.
+    mem_limit: ${BACKEND_MEM_LIMIT:-4g}
+    security_opt:
+      - no-new-privileges:true
 
   frontend:
     build: ./frontend
@@ -213,8 +226,17 @@ services:
       # to 300000 (5 min) in hooks.server.ts; raise/lower via .env.
       BACKEND_PROXY_TIMEOUT_MS: ${BACKEND_PROXY_TIMEOUT_MS:-300000}
     ports:
-      - "3000:3000"
+      # Bind to loopback by default so SvelteKit (plain HTTP; COOKIE_SECURE=true
+      # expects TLS in front) is reachable ONLY by a host-local reverse proxy,
+      # not cleartext over the LAN/internet. A host-external TLS terminator can
+      # set FRONTEND_PUBLISH_ADDR=0.0.0.0 (or a specific interface) in .env.
+      - "${FRONTEND_PUBLISH_ADDR:-127.0.0.1}:3000:3000"
     restart: unless-stopped
+    # Cap runaway memory (SvelteKit is light, but bound it anyway) and forbid
+    # privilege escalation. Tune the limit to your host via .env.
+    mem_limit: ${FRONTEND_MEM_LIMIT:-512m}
+    security_opt:
+      - no-new-privileges:true
 
   # ----- Vision autoscaling (profile: ai) -----------------------------------
   # Two extra containers that idle-stop the mangalord-vision (llama.cpp)

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.109.1",
+  "version": "0.128.24",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mangalord-frontend",
-      "version": "0.109.1",
+      "version": "0.128.24",
       "devDependencies": {
         "@lucide/svelte": "^1.16.0",
         "@playwright/test": "^1.48.0",
@@ -169,7 +169,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -193,7 +192,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       }
@@ -1157,7 +1155,6 @@
       "integrity": "sha512-mQjlkNo+rJvpln7V2IGY2j99BqhcFbS4UN0AQNKNYfhBAFZTuCDAdW3a1sgf330mvtNvsBXn3HpAhcmvdJTcIQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@standard-schema/spec": "^1.0.0",
         "@sveltejs/acorn-typescript": "^1.0.5",
@@ -1200,7 +1197,6 @@
       "integrity": "sha512-0ba1RQ/PHen5FGpdSrW7Y3fAMQjrXantECALeOiOdBdzR5+5vPP6HVZRLmZaQL+W8m++o+haIAKq5qT+MiZ7VA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@sveltejs/vite-plugin-svelte-inspector": "^3.0.0-next.0||^3.0.0",
         "debug": "^4.3.7",
@@ -1359,7 +1355,6 @@
       "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "undici-types": "~6.21.0"
       }
@@ -1507,7 +1502,6 @@
       "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -2249,7 +2243,6 @@
       "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "cssstyle": "^4.1.0",
         "data-urls": "^5.0.0",
@@ -2638,7 +2631,6 @@
       "integrity": "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/estree": "1.0.8"
       },
@@ -2810,7 +2802,6 @@
       "integrity": "sha512-ymI5ykLPwIHW839E053FQbI1G+jnRFJEw3Kv5Y4njixVWywQBx+NUFpkkKyk5LIb36Fg9DVXSYpqiGekLD0hyw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@jridgewell/remapping": "^2.3.4",
         "@jridgewell/sourcemap-codec": "^1.5.0",
@@ -2997,7 +2988,6 @@
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -3019,7 +3009,6 @@
       "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "esbuild": "^0.21.3",
         "postcss": "^8.4.43",
@@ -3138,7 +3127,6 @@
       "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@vitest/expect": "2.1.9",
         "@vitest/mocker": "2.1.9",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.128.10",
+  "version": "0.128.25",
   "private": true,
   "type": "module",
   "scripts": {

frontend/src/hooks.server.test.ts

@@ -370,6 +370,16 @@ describe('applySecurityHeaders', () => {
         expect(pp).toContain('geolocation=()');
     });
 
+    it('emits HSTS only when the request was secure', () => {
+        const insecure = applySecurityHeaders(new Headers());
+        expect(insecure.get('strict-transport-security')).toBeNull();
+
+        const secure = applySecurityHeaders(new Headers(), { hsts: true });
+        expect(secure.get('strict-transport-security')).toBe(
+            'max-age=63072000; includeSubDomains'
+        );
+    });
+
     it('does not clobber a header the response already set', () => {
         // If a downstream route deliberately set a stricter Referrer-Policy,
         // the helper must not override it.

frontend/src/hooks.server.ts

@@ -124,13 +124,23 @@ export function shouldBypassProxyTimeout(headers: Headers): boolean {
  * stricter value keeps it. Mutates and returns the passed `Headers`. Exported
  * for unit-test coverage.
  */
-export function applySecurityHeaders(headers: Headers): Headers {
+export function applySecurityHeaders(
+    headers: Headers,
+    opts: { hsts?: boolean } = {}
+): Headers {
     const defaults: Record<string, string> = {
         'x-frame-options': 'DENY',
         'referrer-policy': 'strict-origin-when-cross-origin',
         'x-content-type-options': 'nosniff',
         'permissions-policy': 'camera=(), microphone=(), geolocation=(), interest-cohort=()'
     };
+    // HSTS ONLY when the browser reached us over HTTPS (the caller derives this
+    // from x-forwarded-proto). Sending it over plain HTTP — a dev run or a
+    // misconfigured deploy — would pin the browser to HTTPS for this host for
+    // two years and could lock users out, so it's opt-in per request.
+    if (opts.hsts) {
+        defaults['strict-transport-security'] = 'max-age=63072000; includeSubDomains';
+    }
     for (const [name, value] of Object.entries(defaults)) {
         if (!headers.has(name)) headers.set(name, value);
     }
@@ -232,6 +242,9 @@ export const handle: Handle = async ({ event, resolve }) => {
         });
     }
     const response = await resolve(event);
-    applySecurityHeaders(response.headers);
+    // The TLS terminator (reverse proxy) sets x-forwarded-proto; only emit HSTS
+    // when the browser's hop to it was HTTPS.
+    const secure = event.request.headers.get('x-forwarded-proto') === 'https';
+    applySecurityHeaders(response.headers, { hsts: secure });
     return response;
 };

frontend/src/lib/components/AddToCollectionModal.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
     import AdaptiveDialog from './AdaptiveDialog.svelte';
+    import { CancellableLoader } from '$lib/util/cancellable';
     import {
         addMangaToCollection,
         createCollection,
@@ -74,19 +75,30 @@
             : removePageFromCollection(collectionId, t.id);
     }
 
+    // Reopening for a different target (or an in-flight retarget) fires load()
+    // again; without this guard the two responses can resolve out of order and
+    // the first target's membership overwrites the second's (audit FE-2). The
+    // CancellableLoader drops a superseded call's result — the newer load then
+    // owns both the state and the spinner.
+    const loader = new CancellableLoader();
+
     async function load() {
         loading = true;
         error = null;
         try {
-            const [page, ids] = await Promise.all([
+            const result = await loader.run(async () => {
-                listMyCollections({ limit: 200 }),
+                const [page, ids] = await Promise.all([
-                loadContaining(target)
+                    listMyCollections({ limit: 200 }),
-            ]);
+                    loadContaining(target)
-            collections = page.items;
+                ]);
-            containingIds = new Set(ids);
+                return { items: page.items, ids };
+            });
+            if (result === null) return; // superseded — a newer load owns the state
+            collections = result.items;
+            containingIds = new Set(result.ids);
+            loading = false;
         } catch (e) {
             error = (e as Error).message;
-        } finally {
             loading = false;
         }
     }

frontend/src/lib/components/AddToCollectionModal.svelte.test.ts

@@ -84,7 +84,11 @@ describe('AddToCollectionModal', () => {
 
         expect(pageCollectionsApi.getMyCollectionsContainingPage).toHaveBeenCalledWith('p1');
         expect(collectionsApi.getMyCollectionsContaining).not.toHaveBeenCalled();
-        expect(screen.getByText('Favorite panels')).toBeTruthy();
+        // Await the async render: load() resolves the collections over a few
+        // microtasks (through the CancellableLoader), so assert with findByText
+        // (retries until it appears) rather than a synchronous getByText tied to
+        // a fixed tick budget.
+        expect(await screen.findByText('Favorite panels')).toBeTruthy();
     });
 
     it('does not load when closed', () => {

frontend/src/lib/components/crawler/CrawlerHistoryTable.svelte

@@ -2,6 +2,7 @@
     import { onMount } from 'svelte';
     import Pager from '$lib/components/Pager.svelte';
     import { fmtDuration } from '$lib/format';
+    import { safeExternalHref } from '$lib/safeHref';
     import SearchBar from './SearchBar.svelte';
     import {
         listCrawlerJobHistory,
@@ -175,9 +176,14 @@
                         <td class="kind">{r.kind ?? '—'}</td>
                         <td>
                             {#if !r.manga_title && r.payload_title && r.source_url}
-                                <a href={r.source_url} target="_blank" rel="noreferrer noopener"
+                                {@const href = safeExternalHref(r.source_url)}
-                                    >{target(r)}</a
+                                {#if href}
-                                >
+                                    <a {href} target="_blank" rel="noreferrer noopener"
+                                        >{target(r)}</a
+                                    >
+                                {:else}
+                                    {target(r)}
+                                {/if}
                             {:else}
                                 {target(r)}
                             {/if}

frontend/src/lib/components/crawler/DeadJobsTable.svelte

@@ -2,6 +2,7 @@
     import Pager from '$lib/components/Pager.svelte';
     import type { DeadJob, RequeueScope } from '$lib/api/admin';
     import SearchBar from './SearchBar.svelte';
+    import { safeExternalHref } from '$lib/safeHref';
 
     let {
         jobs,
@@ -60,9 +61,14 @@
                                 {j.manga_title}
                             {:else if j.payload_title}
                                 {#if j.source_url}
-                                    <a href={j.source_url} target="_blank" rel="noreferrer noopener"
+                                    {@const href = safeExternalHref(j.source_url)}
-                                        >{j.payload_title}</a
+                                    {#if href}
-                                    >
+                                        <a {href} target="_blank" rel="noreferrer noopener"
+                                            >{j.payload_title}</a
+                                        >
+                                    {:else}
+                                        {j.payload_title}
+                                    {/if}
                                 {:else}
                                     {j.payload_title}
                                 {/if}

frontend/src/lib/readerKeys.test.ts

@@ -0,0 +1,23 @@
+import { describe, it, expect } from 'vitest';
+import { isTypingTarget } from './readerKeys';
+
+function el(tag: string, contentEditable = false): HTMLElement {
+    const node = document.createElement(tag);
+    if (contentEditable) node.setAttribute('contenteditable', 'true');
+    return node;
+}
+
+describe('isTypingTarget', () => {
+    it('is true for text-entry and native-key elements', () => {
+        expect(isTypingTarget(el('input'))).toBe(true);
+        expect(isTypingTarget(el('textarea'))).toBe(true);
+        expect(isTypingTarget(el('select'))).toBe(true);
+        expect(isTypingTarget(el('div', true))).toBe(true);
+    });
+
+    it('is false for ordinary elements and null', () => {
+        expect(isTypingTarget(el('div'))).toBe(false);
+        expect(isTypingTarget(el('button'))).toBe(false);
+        expect(isTypingTarget(null)).toBe(false);
+    });
+});

frontend/src/lib/readerKeys.ts

@@ -0,0 +1,19 @@
+/**
+ * Whether a keyboard event's target is a place the user is typing (or otherwise
+ * expects native key handling), so the reader must NOT hijack arrows/j/k/Home/End
+ * for page navigation. Covers text inputs, textareas, native `<select>` dropdowns
+ * (whose Home/End/arrows are their own), and any contenteditable region.
+ */
+export function isTypingTarget(target: EventTarget | null): boolean {
+    const el = target as HTMLElement | null;
+    if (!el) return false;
+    const tag = el.tagName;
+    // `isContentEditable` is the correct runtime signal (accounts for inherited
+    // editability); the attribute check is a robust fallback for the explicit
+    // case and for environments that don't compute the property (jsdom).
+    const editable =
+        el.isContentEditable === true ||
+        el.getAttribute?.('contenteditable') === 'true' ||
+        el.getAttribute?.('contenteditable') === '';
+    return tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || editable;
+}

frontend/src/lib/safeHref.test.ts

@@ -0,0 +1,29 @@
+import { describe, it, expect } from 'vitest';
+import { safeExternalHref } from './safeHref';
+
+describe('safeExternalHref', () => {
+    it('passes through http and https URLs', () => {
+        expect(safeExternalHref('http://example.com/manga/1')).toBe(
+            'http://example.com/manga/1'
+        );
+        expect(safeExternalHref('https://example.com/x?y=1#z')).toBe(
+            'https://example.com/x?y=1#z'
+        );
+    });
+
+    it('rejects dangerous schemes', () => {
+        expect(safeExternalHref('javascript:alert(document.cookie)')).toBeNull();
+        expect(safeExternalHref('JavaScript:alert(1)')).toBeNull();
+        expect(safeExternalHref('data:text/html,<script>alert(1)</script>')).toBeNull();
+        expect(safeExternalHref('vbscript:msgbox(1)')).toBeNull();
+        expect(safeExternalHref('file:///etc/passwd')).toBeNull();
+    });
+
+    it('rejects relative, malformed, empty, and nullish input', () => {
+        expect(safeExternalHref('/relative/path')).toBeNull();
+        expect(safeExternalHref('not a url')).toBeNull();
+        expect(safeExternalHref('')).toBeNull();
+        expect(safeExternalHref(null)).toBeNull();
+        expect(safeExternalHref(undefined)).toBeNull();
+    });
+});

frontend/src/lib/safeHref.ts

@@ -0,0 +1,21 @@
+/**
+ * Return `url` only if it is a well-formed `http(s)` URL that is safe to place
+ * in an `href`, otherwise `null`.
+ *
+ * Crawler `source_url` values are external/untrusted content harvested from
+ * scraped source sites. Svelte does NOT sanitize URL schemes in `href`
+ * bindings, so a harvested `javascript:`/`data:`/`vbscript:` link would execute
+ * (or phish) in the admin origin on click. Callers render the title as plain
+ * text when this returns `null`.
+ */
+export function safeExternalHref(url: string | null | undefined): string | null {
+    if (!url) return null;
+    let parsed: URL;
+    try {
+        parsed = new URL(url);
+    } catch {
+        // Relative or malformed — not a safe external link.
+        return null;
+    }
+    return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? url : null;
+}

frontend/src/routes/manga/[id]/chapter/[chapter_id]/+page.svelte

@@ -3,6 +3,7 @@
     import { browser } from '$app/environment';
     import { afterNavigate, goto, invalidateAll } from '$app/navigation';
     import { fileUrl, ApiError } from '$lib/api/client';
+    import { isTypingTarget } from '$lib/readerKeys';
     import { GAP_PX, type ReaderPageGap } from '$lib/api/preferences';
     import { preferences } from '$lib/preferences.svelte';
     import { updateReadProgress } from '$lib/api/read_progress';
@@ -675,9 +676,9 @@
     );
 
     function onKeydown(e: KeyboardEvent) {
-        // Don't hijack keys while the user is typing in an input.
+        // Don't hijack keys while the user is typing or focused on a native
-        const target = e.target as HTMLElement | null;
+        // control (input/textarea/select/contenteditable own their own keys).
-        if (target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) return;
+        if (isTypingTarget(e.target)) return;
         // `?` opens the keyboard-shortcut help overlay. While it's open,
         // swallow every key here so paging can't happen behind it — the
         // Sheet owns its own Escape/close.
@@ -687,6 +688,20 @@
             return;
         }
         if (shortcutsOpen) return;
+        // Any other overlay (jump/settings/action/collections/tags/context menu)
+        // owns its own keys; swallow reader paging so arrows/j/k/Home/End can't
+        // act on the chapter *behind* the overlay. Those overlays only
+        // stopPropagation() on Escape, so navigation keys otherwise bubble here.
+        if (
+            chapterJumpOpen ||
+            settingsOpen ||
+            contextMenuOpen ||
+            actionSheetOpen ||
+            collectionsModalOpen ||
+            tagsModalOpen
+        ) {
+            return;
+        }
         // Esc always exits fullscreen if active — applies in both
         // modes, including when the bars are hidden. Handled before
         // the per-mode switches so it doesn't get shadowed.