bugfix: SSRF allowlist, image size cap, robust session detect (0.34.1)

Four crawler defences in one PR (all four threats the review flagged in §3 of REVIEW.md): - New crawler::safety module with is_safe_url + accumulate_capped + fetch_bytes_capped. Rejects non-http(s) schemes, RFC1918 / loopback / link-local / CGNAT / ULA / IPv6-link-local hosts, and any host not on the operator's allowlist (defaults to CRAWLER_START_URL host + CRAWLER_CDN_HOST + CRAWLER_DOWNLOAD_ALLOWLIST extras). - Streaming size cap (CRAWLER_MAX_IMAGE_BYTES, default 32 MiB) so a 10 GiB \"image\" can't fill memory before disk. - looks_like_image() reject path: non-image bytes fail the chapter or cover instead of being stored as .bin and served as application/octet-stream. - session::classify_chapter_probe: three-way classifier replaces the binary #avatar_menu check at content.rs:115. A transient hiccup (broken-page body, or logged-in-but-no-reader) now retries with backoff instead of falsely freezing every worker on session_expired. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-28 20:00:22 +02:00
21 changed files with 956 additions and 325 deletions
--- a/.env.example
+++ b/.env.example
@@ -44,6 +44,14 @@ MAX_REQUEST_BYTES=209715200
 # Default 20 MiB.
 MAX_FILE_BYTES=20971520

+# ----- Crawler download safety -----
+# Hosts the crawler is allowed to fetch images/covers from, in addition
+# to CRAWLER_START_URL's host and CRAWLER_CDN_HOST. Comma-separated.
+# Defends against SSRF via scraped <img src="http://10.0.0.1/...">.
+CRAWLER_DOWNLOAD_ALLOWLIST=
+# Hard cap on a single image body. Default 32 MiB.
+CRAWLER_MAX_IMAGE_BYTES=33554432
+
 # ----- Frontend -----
 # The frontend container runs SvelteKit's Node adapter on :3000 and
 # proxies /api/* to BACKEND_URL via src/hooks.server.ts. In compose the
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -1470,7 +1470,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"

 [[package]]
 name = "mangalord"
-version = "0.34.0"
+version = "0.34.1"
 dependencies = [
 "anyhow",
 "argon2",
@@ -2324,6 +2324,7 @@ dependencies = [
 "cookie",
 "cookie_store",
 "futures-core",
+ "futures-util",
 "http",
 "http-body",
 "http-body-util",
@@ -2343,12 +2344,14 @@ dependencies = [
 "sync_wrapper",
 "tokio",
 "tokio-rustls",
+ "tokio-util",
 "tower",
 "tower-http",
 "tower-service",
 "url",
 "wasm-bindgen",
 "wasm-bindgen-futures",
+ "wasm-streams",
 "web-sys",
 "webpki-roots",
 ]
@@ -3527,6 +3530,19 @@ dependencies = [
 "wasmparser",
 ]

+[[package]]
+name = "wasm-streams"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "wasmparser"
 version = "0.244.0"
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.34.0"
+version = "0.34.1"
 edition = "2021"
 default-run = "mangalord"

@@ -46,7 +46,7 @@ futures-util = "0.3"
 bytes = "1"
 chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttls-tokio"], default-features = false }
 scraper = "0.20"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies"] }
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream"] }

 [dev-dependencies]
 tempfile = "3"
--- a/backend/src/api/bookmarks.rs
+++ b/backend/src/api/bookmarks.rs
@@ -67,7 +67,14 @@ async fn create(
    // the foreign-key violation collapse into a generic 500.
    repo::manga::get(&state.db, input.manga_id).await?;
    if let Some(chapter_id) = input.chapter_id {
-        if !repo::chapter::belongs_to_manga(&state.db, chapter_id, input.manga_id).await? {
+        let exists: Option<(Uuid,)> = sqlx::query_as(
+            "SELECT id FROM chapters WHERE id = $1 AND manga_id = $2",
+        )
+        .bind(chapter_id)
+        .bind(input.manga_id)
+        .fetch_optional(&state.db)
+        .await?;
+        if exists.is_none() {
            return Err(AppError::NotFound);
        }
    }
--- a/backend/src/app.rs
+++ b/backend/src/app.rs
@@ -19,6 +19,7 @@ use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass
 use crate::crawler::jobs::JobPayload;
 use crate::crawler::pipeline::{self, MetadataStats};
 use crate::crawler::rate_limit::HostRateLimiters;
+use crate::crawler::safety::DownloadAllowlist;
 use crate::crawler::session;
 use crate::crawler::source::{target as target_source, DiscoverMode};
 use crate::repo;
@@ -153,6 +154,8 @@ async fn spawn_crawler_daemon(
            start_url: url.clone(),
            mode_pref: cfg.mode,
            incremental_stop_after: cfg.incremental_stop_after,
+            download_allowlist: cfg.download_allowlist.clone(),
+            max_image_bytes: cfg.max_image_bytes,
        });
        m
    });
@@ -163,6 +166,8 @@ async fn spawn_crawler_daemon(
        storage: Arc::clone(&storage),
        http,
        rate: Arc::clone(&rate),
+        download_allowlist: cfg.download_allowlist.clone(),
+        max_image_bytes: cfg.max_image_bytes,
    });

    // Shared cancellation: daemon shutdown cancels the BrowserManager's
@@ -216,6 +221,8 @@ struct RealMetadataPass {
    start_url: String,
    mode_pref: CrawlerModePref,
    incremental_stop_after: usize,
+    download_allowlist: DownloadAllowlist,
+    max_image_bytes: usize,
 }

 #[async_trait]
@@ -238,6 +245,8 @@ impl MetadataPass for RealMetadataPass {
            0,
            false,
            mode,
+            &self.download_allowlist,
+            self.max_image_bytes,
        )
        .await
    }
@@ -293,6 +302,8 @@ struct RealChapterDispatcher {
    storage: Arc<dyn Storage>,
    http: reqwest::Client,
    rate: Arc<HostRateLimiters>,
+    download_allowlist: DownloadAllowlist,
+    max_image_bytes: usize,
 }

 #[async_trait]
@@ -304,9 +315,18 @@ impl ChapterDispatcher for RealChapterDispatcher {
                chapter_id,
                source_chapter_key: _,
            } => {
-                let row = repo::chapter::dispatch_target(&self.db, chapter_id)
-                    .await
-                    .context("look up chapter for dispatch")?;
+                // Look up manga_id + source_url for this chapter.
+                let row: Option<(uuid::Uuid, String)> = sqlx::query_as(
+                    "SELECT c.manga_id, cs.source_url \
+                       FROM chapters c \
+                       JOIN chapter_sources cs ON cs.chapter_id = c.id \
+                      WHERE c.id = $1 \
+                      LIMIT 1",
+                )
+                .bind(chapter_id)
+                .fetch_optional(&self.db)
+                .await
+                .context("look up chapter for dispatch")?;
                let Some((manga_id, source_url)) = row else {
                    // Chapter (or its source row) is gone — ack done.
                    return Ok(SyncOutcome::Skipped);
@@ -322,6 +342,8 @@ impl ChapterDispatcher for RealChapterDispatcher {
                    manga_id,
                    &source_url,
                    false,
+                    &self.download_allowlist,
+                    self.max_image_bytes,
                )
                .await?;
                drop(lease);
--- a/backend/src/bin/crawler.rs
+++ b/backend/src/bin/crawler.rs
@@ -229,6 +229,33 @@ async fn run(
    }
    let rate = Arc::new(rate);

+    // SSRF defence: only download from the catalog host + CDN host
+    // (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
+    // single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
+    let mut allowlist =
+        mangalord::crawler::safety::DownloadAllowlist::new();
+    if let Ok(parsed) = reqwest::Url::parse(start_url) {
+        if let Some(h) = parsed.host_str() {
+            allowlist = allowlist.allow(h);
+        }
+    }
+    if let Some(host) = cdn_host {
+        allowlist = allowlist.allow(host);
+    }
+    if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
+        for piece in extras.split(',') {
+            let trimmed = piece.trim();
+            if !trimmed.is_empty() {
+                allowlist = allowlist.allow(trimmed);
+            }
+        }
+    }
+    let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
+    let allowlist = Arc::new(allowlist);
+
    let stats = pipeline::run_metadata_pass(
        manager.as_ref(),
        db,
@@ -239,6 +266,8 @@ async fn run(
        limit,
        skip_chapters,
        mode,
+        allowlist.as_ref(),
+        max_image_bytes,
    )
    .await?;
    tracing::info!(?stats, "metadata pass complete");
@@ -253,6 +282,8 @@ async fn run(
            "target",
            chapter_workers,
            force_refetch_chapters,
+            Arc::clone(&allowlist),
+            max_image_bytes,
        )
        .await?;
    }
@@ -276,6 +307,8 @@ async fn sync_bookmarked_chapter_content(
    source_id: &str,
    workers: usize,
    force_refetch: bool,
+    allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
+    max_image_bytes: usize,
 ) -> anyhow::Result<()> {
    let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
        r#"
@@ -312,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
            let storage = Arc::clone(&storage);
            let rate = Arc::clone(&rate);
            let manager = Arc::clone(&manager);
+            let allowlist = Arc::clone(&allowlist);
            let stats = &stats;
            async move {
                if session_expired.load(std::sync::atomic::Ordering::Relaxed) {
@@ -336,6 +370,8 @@ async fn sync_bookmarked_chapter_content(
                    manga_id,
                    &source_url,
                    force_refetch,
+                    allowlist.as_ref(),
+                    max_image_bytes,
                )
                .await;
                drop(lease);
--- a/backend/src/config.rs
+++ b/backend/src/config.rs
@@ -5,6 +5,7 @@ use chrono::NaiveTime;
 use chrono_tz::Tz;

 use crate::crawler::browser::LaunchOptions;
+use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
 use crate::crawler::source::DiscoverMode;

 /// What `CRAWLER_MODE` was set to. `Auto` is the daemon's default —
@@ -93,6 +94,13 @@ pub struct CrawlerConfig {
    /// `stop_after_unchanged` threshold supplied to Incremental in both
    /// `Auto` (post-seed) and `Explicit(Incremental)` modes.
    pub incremental_stop_after: usize,
+    /// Hosts the crawler is allowed to download images / covers from.
+    /// Always seeded with the host of `start_url` and (when set) the
+    /// configured `cdn_host`. Additional hosts can be added via
+    /// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated).
+    pub download_allowlist: DownloadAllowlist,
+    /// Hard upper bound on a single image download. Defaults to 32 MiB.
+    pub max_image_bytes: usize,
 }

 impl Default for CrawlerConfig {
@@ -115,6 +123,8 @@ impl Default for CrawlerConfig {
            browser: LaunchOptions::headless(),
            mode: CrawlerModePref::Auto,
            incremental_stop_after: 20,
+            download_allowlist: DownloadAllowlist::new(),
+            max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
        }
    }
 }
@@ -172,6 +182,14 @@ impl CrawlerConfig {
        let incremental_stop_after =
            env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
        let mode = parse_mode_env(incremental_stop_after)?;
+        let start_url = std::env::var("CRAWLER_START_URL")
+            .ok()
+            .filter(|s| !s.trim().is_empty());
+        let cdn_host = std::env::var("CRAWLER_CDN_HOST")
+            .ok()
+            .filter(|s| !s.trim().is_empty());
+        let download_allowlist =
+            build_download_allowlist(start_url.as_deref(), cdn_host.as_deref());
        Ok(Self {
            daemon_enabled: env_bool("CRAWLER_DAEMON", true),
            daily_at,
@@ -179,13 +197,9 @@ impl CrawlerConfig {
            idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
            chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
            retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
-            start_url: std::env::var("CRAWLER_START_URL")
-                .ok()
-                .filter(|s| !s.trim().is_empty()),
+            start_url,
            rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
-            cdn_host: std::env::var("CRAWLER_CDN_HOST")
-                .ok()
-                .filter(|s| !s.trim().is_empty()),
+            cdn_host,
            cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)),
            phpsessid: std::env::var("CRAWLER_PHPSESSID")
                .ok()
@@ -202,10 +216,45 @@ impl CrawlerConfig {
            browser: LaunchOptions::from_env(),
            mode,
            incremental_stop_after,
+            download_allowlist,
+            max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
        })
    }
 }

+/// Build the download allowlist from env. Always includes
+/// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from
+/// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional
+/// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-
+/// separated). Empty by default — meaning the crawler refuses to
+/// download anything when no source is configured, which is the safe
+/// fail-closed posture.
+fn build_download_allowlist(
+    start_url: Option<&str>,
+    cdn_host: Option<&str>,
+) -> DownloadAllowlist {
+    let mut allow = DownloadAllowlist::new();
+    if let Some(url) = start_url {
+        if let Ok(parsed) = reqwest::Url::parse(url) {
+            if let Some(h) = parsed.host_str() {
+                allow = allow.allow(h);
+            }
+        }
+    }
+    if let Some(host) = cdn_host {
+        allow = allow.allow(host);
+    }
+    if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
+        for piece in extras.split(',') {
+            let trimmed = piece.trim();
+            if !trimmed.is_empty() {
+                allow = allow.allow(trimmed);
+            }
+        }
+    }
+    allow
+}
+
 /// Parse `CRAWLER_MODE`. Empty/unset → `Auto`. Recognized values are
 /// `auto`, `backfill`, and `incremental` (case-insensitive). Anything
 /// else is a hard error so a typo can't silently fall through to the
--- a/backend/src/crawler/content.rs
+++ b/backend/src/crawler/content.rs
@@ -18,7 +18,8 @@ use uuid::Uuid;

 use crate::crawler::detect::PageError;
 use crate::crawler::rate_limit::HostRateLimiters;
-use crate::crawler::session;
+use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
+use crate::crawler::session::{self, ChapterProbe};
 use crate::storage::Storage;

 /// Parse the chapter page DOM and return the page images in `pageN`
@@ -88,6 +89,8 @@ pub async fn sync_chapter_content(
    manga_id: Uuid,
    source_url: &str,
    force_refetch: bool,
+    allowlist: &DownloadAllowlist,
+    max_image_bytes: usize,
 ) -> anyhow::Result<SyncOutcome> {
    // Skip if already fetched, unless caller explicitly forces.
    if !force_refetch {
@@ -110,16 +113,28 @@ pub async fn sync_chapter_content(
        .with_context(|| format!("open chapter page {source_url}"))?;
    page.wait_for_navigation().await.context("wait for chapter nav")?;

-    // Session probe: avatar present == still logged in. Missing means
-    // PHPSESSID expired; bail the entire crawler run.
-    if page.find_element("#avatar_menu").await.is_err() {
-        page.close().await.ok();
-        return Ok(SyncOutcome::SessionExpired);
-    }
-
    let html = page.content().await.context("read chapter html")?;
    page.close().await.ok();

+    // Three-way session classification: distinguishes a transient
+    // hiccup (broken-page body or logged-in-but-no-reader) from a
+    // genuine PHPSESSID expiry (no reader and no avatar widget). The
+    // earlier binary `#avatar_menu` check conflated both and froze
+    // every worker on a layout shift.
+    match session::classify_chapter_probe(&html) {
+        ChapterProbe::Unauthenticated => return Ok(SyncOutcome::SessionExpired),
+        ChapterProbe::Transient => {
+            // Surface as a typed Err so the dispatcher path runs
+            // ack_failed with exponential backoff (rather than the
+            // session-expired sticky flag).
+            anyhow::bail!(
+                "chapter page at {source_url} returned a transient response \
+                 (broken-page body or reader didn't render); will retry"
+            );
+        }
+        ChapterProbe::Ok => {}
+    }
+
    let images = parse_chapter_pages(&html)
        .with_context(|| format!("parse chapter pages at {source_url}"))?;
    if images.is_empty() {
@@ -138,18 +153,29 @@ pub async fn sync_chapter_content(
            format!("join image URL {} onto {source_url}", img.url)
        })?;
        rate.wait_for(url.as_str()).await?;
-        let resp = http
-            .get(url.clone())
-            // Source CDNs commonly check Referer. Set it to the
-            // chapter page — matches what the browser would send.
-            .header(reqwest::header::REFERER, source_url)
-            .send()
-            .await
-            .with_context(|| format!("GET {url}"))?
-            .error_for_status()
-            .with_context(|| format!("non-2xx for {url}"))?;
-        let bytes = resp.bytes().await.context("read image body")?.to_vec();
-        let ext = infer::get(&bytes).map(|k| k.extension()).unwrap_or("bin");
+        let bytes = fetch_bytes_capped(
+            http,
+            url.as_str(),
+            Some(source_url),
+            allowlist,
+            max_image_bytes,
+        )
+        .await?
+        .to_vec();
+        // Reject any non-image response: the only valid output of an
+        // image URL is an image. `infer` returns None on truncated
+        // bytes too, which also wants to be a failure not a silent
+        // `.bin` extension.
+        if !looks_like_image(&bytes) {
+            anyhow::bail!(
+                "image URL {url} returned non-image bytes \
+                 (first 16: {:?}); refusing to store as binary blob",
+                &bytes.get(..16.min(bytes.len()))
+            );
+        }
+        let ext = infer::get(&bytes)
+            .map(|k| k.extension())
+            .expect("looks_like_image asserted infer succeeded");
        fetched.push((img.page_number, bytes, ext));
    }

@@ -194,8 +220,9 @@ pub async fn sync_chapter_content(
    Ok(SyncOutcome::Fetched { pages: fetched.len() })
 }

-// Suppress unused-import warning for `session` until the bin/crawler
-// wiring lands in this branch and uses it through this module.
+// Suppress unused-import warning for `session::registrable_domain`
+// until the bin/crawler wiring lands in this branch and uses it
+// through this module.
 #[allow(dead_code)]
 fn _keep_session_in_scope() {
    let _ = session::registrable_domain;
--- a/backend/src/crawler/daemon.rs
+++ b/backend/src/crawler/daemon.rs
@@ -317,10 +317,14 @@ impl WorkerContext {
        // (because a force-refetch race or a job that was re-enqueued
        // after a previous one finished), ack done without re-fetching.
        if let JobPayload::SyncChapterContent { chapter_id, .. } = &lease.payload {
-            let page_count = crate::repo::chapter::page_count(&self.pool, *chapter_id)
-                .await
-                .ok()
-                .flatten();
+            let page_count: Option<i32> = sqlx::query_scalar(
+                "SELECT page_count FROM chapters WHERE id = $1",
+            )
+            .bind(chapter_id)
+            .fetch_optional(&self.pool)
+            .await
+            .ok()
+            .flatten();
            if matches!(page_count, Some(n) if n > 0) {
                let _ = jobs::ack_done(&self.pool, lease.id).await;
                return;
--- a/backend/src/crawler/mod.rs
+++ b/backend/src/crawler/mod.rs
@@ -22,6 +22,6 @@ pub mod diff;
 pub mod jobs;
 pub mod pipeline;
 pub mod rate_limit;
+pub mod safety;
 pub mod session;
 pub mod source;
-pub mod url_utils;
--- a/backend/src/crawler/pipeline.rs
+++ b/backend/src/crawler/pipeline.rs
@@ -9,6 +9,7 @@ use uuid::Uuid;
 use crate::crawler::browser_manager::BrowserManager;
 use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
 use crate::crawler::rate_limit::HostRateLimiters;
+use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
 use crate::crawler::source::target::TargetSource;
 use crate::crawler::source::{DiscoverMode, FetchContext, Source};
 use crate::repo;
@@ -62,6 +63,8 @@ pub async fn run_metadata_pass(
    limit: usize,
    skip_chapters: bool,
    mode: DiscoverMode,
+    allowlist: &DownloadAllowlist,
+    max_image_bytes: usize,
 ) -> anyhow::Result<MetadataStats> {
    let lease = browser_manager
        .acquire()
@@ -181,6 +184,8 @@ pub async fn run_metadata_pass(
                        &r.url,
                        upsert.manga_id,
                        cover_url,
+                        allowlist,
+                        max_image_bytes,
                    )
                    .await
                    {
@@ -382,6 +387,7 @@ pub struct EnqueueSummary {
 /// pipeline because the CLI still calls it from its inline chapter-content
 /// loop; once the worker pool fully replaces that path we can fold this
 /// into `pipeline` proper.
+#[allow(clippy::too_many_arguments)]
 async fn download_and_store_cover(
    db: &PgPool,
    storage: &dyn Storage,
@@ -390,6 +396,8 @@ async fn download_and_store_cover(
    manga_url: &str,
    manga_id: Uuid,
    cover_url: &str,
+    allowlist: &DownloadAllowlist,
+    max_image_bytes: usize,
 ) -> anyhow::Result<()> {
    let absolute = reqwest::Url::parse(manga_url)
        .context("parse manga URL")?
@@ -397,17 +405,22 @@ async fn download_and_store_cover(
        .context("join cover URL onto manga URL")?;

    rate.wait_for(absolute.as_str()).await?;
-    let resp = http
-        .get(absolute.clone())
-        .header(reqwest::header::REFERER, manga_url)
-        .send()
-        .await
-        .with_context(|| format!("GET {absolute}"))?
-        .error_for_status()
-        .with_context(|| format!("non-2xx for {absolute}"))?;
-    let bytes = resp.bytes().await.context("read cover body")?;
-    let kind = infer::get(&bytes);
-    let ext = kind.map(|k| k.extension()).unwrap_or("bin");
+    let bytes = fetch_bytes_capped(
+        http,
+        absolute.as_str(),
+        Some(manga_url),
+        allowlist,
+        max_image_bytes,
+    )
+    .await?;
+    if !looks_like_image(&bytes) {
+        anyhow::bail!(
+            "cover URL {absolute} returned non-image bytes; refusing to store as binary blob"
+        );
+    }
+    let ext = infer::get(&bytes)
+        .map(|k| k.extension())
+        .expect("looks_like_image asserted infer succeeded");
    let key = format!("mangas/{manga_id}/cover.{ext}");

    storage
@@ -427,7 +440,11 @@ async fn download_and_store_cover(
    Ok(())
 }

-use crate::crawler::url_utils::origin_of;
+fn origin_of(url: &str) -> Option<String> {
+    let (scheme, rest) = url.split_once("://")?;
+    let host = rest.split('/').next()?;
+    Some(format!("{scheme}://{host}"))
+}

 #[cfg(test)]
 mod tests {
--- a/backend/src/crawler/rate_limit.rs
+++ b/backend/src/crawler/rate_limit.rs
@@ -98,9 +98,15 @@ impl HostRateLimiters {
    }
 }

-// `host_of` was duplicated across session/rate_limit/pipeline; the
-// canonical version now lives in `crawler::url_utils`.
-use crate::crawler::url_utils::host_of;
+/// Extract the host (no port) from a URL string. Returns `None` for
+/// inputs without a `scheme://host` shape — those would never have
+/// reached the network layer anyway.
+fn host_of(url: &str) -> Option<String> {
+    let after_scheme = url.split_once("://")?.1;
+    let host_with_port = after_scheme.split('/').next()?;
+    let host = host_with_port.rsplit_once(':').map_or(host_with_port, |(h, _)| h);
+    (!host.is_empty()).then(|| host.to_ascii_lowercase())
+}

 #[cfg(test)]
 mod tests {
--- a/backend/src/crawler/safety.rs
+++ b/backend/src/crawler/safety.rs
@@ -0,0 +1,486 @@
+//! Defensive helpers for the image-download paths.
+//!
+//! Two threats this module addresses:
+//!
+//! - **SSRF**: a scraped chapter or manga page can embed an absolute
+//!   `<img src="http://10.0.0.1/...">`. The crawler runs inside the
+//!   backend container with intra-compose access to `postgres:5432`
+//!   and possibly other internal services; without a host check the
+//!   crawler would happily probe them. [`is_safe_url`] rejects
+//!   anything whose host isn't on the operator-configured allowlist,
+//!   plus any IP literal in RFC1918 / loopback / link-local / unique-
+//!   local space (including IPv4-mapped IPv6 like `::ffff:127.0.0.1`)
+//!   as a second defence for the case where an allowlisted hostname's
+//!   DNS happens to resolve to a literal private address.
+//!
+//!   **DNS rebinding is not covered.** A hostname like `cdn.allowed.com`
+//!   that *resolves* to `127.0.0.1` via hostile DNS bypasses the IP
+//!   check entirely — `is_safe_url` only inspects URL strings, not
+//!   resolved IPs. Mitigating that requires a custom reqwest resolver
+//!   that filters IPs after DNS, which would mean rebuilding reqwest's
+//!   connector. The allowlist + good operator DNS hygiene is the
+//!   realistic mitigation today.
+//!
+//! - **Unbounded download**: `Response::bytes().await` reads the full
+//!   body before returning. A malicious source serving a 10 GiB image
+//!   would fill memory and then disk. [`accumulate_capped`] streams
+//!   the body chunk-by-chunk into a [`bytes::BytesMut`] and bails as
+//!   soon as the running total exceeds the cap.
+//!
+//! Both helpers are pure-data: the SSRF check is keyed off a parsed
+//! URL string, and the byte accumulator is keyed off a generic stream.
+//! Easy to unit-test without a live network or browser.
+
+use std::net::IpAddr;
+
+use anyhow::{bail, Context};
+use bytes::BytesMut;
+use futures_util::StreamExt;
+use reqwest::Url;
+
+/// Default per-image download cap. A page image is generally <2 MiB;
+/// 32 MiB leaves headroom for high-resolution covers while still
+/// stopping a misbehaving CDN dead. Override via `CRAWLER_MAX_IMAGE_BYTES`.
+pub const DEFAULT_MAX_IMAGE_BYTES: usize = 32 * 1024 * 1024;
+
+/// Hosts that are always allowed in addition to the operator's
+/// configured allowlist. None by default — keeping the surface area
+/// minimal so the only way a URL gets through is if it matches an
+/// explicit catalog/CDN entry.
+#[derive(Clone, Debug, Default)]
+pub struct DownloadAllowlist {
+    hosts: Vec<String>,
+}
+
+impl DownloadAllowlist {
+    pub fn new() -> Self {
+        Self { hosts: Vec::new() }
+    }
+
+    /// Add a host (case-insensitive match). Sub-domains are *not*
+    /// implied: pass `cdn.example.com` and `example.com` separately
+    /// if both should be reachable.
+    pub fn allow(mut self, host: impl Into<String>) -> Self {
+        let h = host.into().to_ascii_lowercase();
+        if !h.is_empty() && !self.hosts.iter().any(|existing| existing == &h) {
+            self.hosts.push(h);
+        }
+        self
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.hosts.is_empty()
+    }
+
+    pub fn contains(&self, host: &str) -> bool {
+        let lower = host.to_ascii_lowercase();
+        self.hosts.iter().any(|h| h == &lower)
+    }
+}
+
+/// Verify a URL is safe for the crawler to fetch.
+///
+/// Rejects:
+/// - non-http(s) schemes (file://, gopher://, …),
+/// - any IP literal in private / loopback / link-local / unique-local
+///   space (defense in depth — a DNS allowlist alone wouldn't cover an
+///   attacker that places an entry like `cdn.evil` pointing at
+///   `192.168.1.1`),
+/// - the literal hostname `localhost`,
+/// - hosts that aren't on the supplied allowlist.
+///
+/// An empty allowlist rejects everything (the conservative default —
+/// callers must explicitly allow the catalog and CDN hosts).
+pub fn is_safe_url(raw_url: &str, allow: &DownloadAllowlist) -> Result<(), UrlSafetyError> {
+    let url = Url::parse(raw_url).map_err(|_| UrlSafetyError::Unparseable)?;
+    let scheme = url.scheme();
+    if scheme != "http" && scheme != "https" {
+        return Err(UrlSafetyError::BadScheme(scheme.to_string()));
+    }
+    let host = url.host_str().ok_or(UrlSafetyError::NoHost)?;
+    let lower_host = host.to_ascii_lowercase();
+    if lower_host == "localhost" {
+        return Err(UrlSafetyError::Loopback);
+    }
+    // Reject IP literals in private/loopback ranges regardless of the
+    // allowlist — if someone puts an IP literal on the allowlist they
+    // almost certainly didn't mean a private range.
+    // reqwest::Url normalises IPv6 literals as `[::1]` (brackets
+    // included) in `host_str()`. Strip the brackets before parsing.
+    let ip_candidate = lower_host
+        .strip_prefix('[')
+        .and_then(|s| s.strip_suffix(']'))
+        .unwrap_or(&lower_host);
+    if let Ok(ip) = ip_candidate.parse::<IpAddr>() {
+        if is_private_ip(&ip) {
+            return Err(UrlSafetyError::PrivateIp(ip));
+        }
+    }
+    if !allow.contains(&lower_host) {
+        return Err(UrlSafetyError::HostNotAllowed(lower_host));
+    }
+    Ok(())
+}
+
+fn is_private_ip(ip: &IpAddr) -> bool {
+    match ip {
+        IpAddr::V4(v4) => {
+            v4.is_loopback()
+                || v4.is_private()
+                || v4.is_link_local()
+                || v4.is_unspecified()
+                || v4.is_broadcast()
+                // CGNAT 100.64.0.0/10
+                || (v4.octets()[0] == 100 && (v4.octets()[1] & 0xC0) == 64)
+                // 169.254/16 link-local already covered, but 0.0.0.0/8 is special-use
+                || v4.octets()[0] == 0
+        }
+        IpAddr::V6(v6) => {
+            // IPv4-mapped IPv6 (::ffff:0:0/96): unwrap to the embedded
+            // IPv4 and recurse so `::ffff:127.0.0.1` is caught by the
+            // IPv4 loopback check rather than passing through.
+            // `Ipv6Addr::is_loopback()` only matches `::1` exactly.
+            if let Some(v4) = v6.to_ipv4_mapped() {
+                return is_private_ip(&IpAddr::V4(v4));
+            }
+            v6.is_loopback()
+                || v6.is_unspecified()
+                // fc00::/7 unique-local
+                || (v6.segments()[0] & 0xfe00) == 0xfc00
+                // fe80::/10 link-local
+                || (v6.segments()[0] & 0xffc0) == 0xfe80
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum UrlSafetyError {
+    #[error("URL is not parseable")]
+    Unparseable,
+    #[error("scheme {0:?} is not http or https")]
+    BadScheme(String),
+    #[error("URL is missing a host")]
+    NoHost,
+    #[error("host points at the loopback interface")]
+    Loopback,
+    #[error("host is a private/internal IP: {0}")]
+    PrivateIp(IpAddr),
+    #[error("host {0:?} is not on the crawler download allowlist")]
+    HostNotAllowed(String),
+}
+
+/// Drain a byte stream into a single buffer, bailing out as soon as
+/// the running total exceeds `max_bytes`. Generic over the stream so
+/// it's testable without a live HTTP response.
+pub async fn accumulate_capped<S, E>(stream: S, max_bytes: usize) -> anyhow::Result<bytes::Bytes>
+where
+    S: futures_core::Stream<Item = Result<bytes::Bytes, E>>,
+    E: std::error::Error + Send + Sync + 'static,
+{
+    let mut buf = BytesMut::new();
+    let mut stream = std::pin::pin!(stream);
+    while let Some(chunk) = stream.next().await {
+        let chunk = chunk.map_err(|e| anyhow::anyhow!("stream chunk: {e}"))?;
+        if buf.len().saturating_add(chunk.len()) > max_bytes {
+            bail!(
+                "response exceeds {max_bytes}-byte cap (received >{}+{})",
+                buf.len(),
+                chunk.len()
+            );
+        }
+        buf.extend_from_slice(&chunk);
+    }
+    Ok(buf.freeze())
+}
+
+/// Send `req` and stream the response into a length-limited buffer.
+/// Combines [`is_safe_url`] check + [`accumulate_capped`] so each
+/// call-site is one line.
+pub async fn fetch_bytes_capped(
+    http: &reqwest::Client,
+    url: &str,
+    referer: Option<&str>,
+    allow: &DownloadAllowlist,
+    max_bytes: usize,
+) -> anyhow::Result<bytes::Bytes> {
+    is_safe_url(url, allow).with_context(|| format!("reject unsafe URL {url}"))?;
+    let mut req = http.get(url);
+    if let Some(r) = referer {
+        req = req.header(reqwest::header::REFERER, r);
+    }
+    let resp = req
+        .send()
+        .await
+        .with_context(|| format!("GET {url}"))?
+        .error_for_status()
+        .with_context(|| format!("non-2xx for {url}"))?;
+    accumulate_capped(resp.bytes_stream(), max_bytes)
+        .await
+        .with_context(|| format!("download body for {url}"))
+}
+
+/// True when `bytes` sniffs as one of the *renderable* image formats
+/// the `/files/*key` endpoint can serve with a correct Content-Type:
+/// JPEG, PNG, WebP, GIF, AVIF. Matches the upload pipeline's
+/// whitelist in `upload::parse_image`.
+///
+/// `infer::MatcherType::Image` is intentionally NOT used — it also
+/// matches BMP, TIFF, HEIF, ICO, PSD, and JP2. Those would sniff as
+/// "image" here but [`api::files::content_type_for`] would fall back
+/// to `application/octet-stream`, prompting browsers to download
+/// instead of render. Keep the two layers aligned.
+pub fn looks_like_image(bytes: &[u8]) -> bool {
+    matches!(
+        infer::get(bytes).map(|k| k.mime_type()),
+        Some("image/jpeg" | "image/png" | "image/webp" | "image/gif" | "image/avif")
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_util::stream;
+
+    fn allow_just(host: &str) -> DownloadAllowlist {
+        DownloadAllowlist::new().allow(host)
+    }
+
+    #[test]
+    fn safe_url_allows_listed_host() {
+        let allow = allow_just("cdn.example.com");
+        assert!(is_safe_url("https://cdn.example.com/img.jpg", &allow).is_ok());
+    }
+
+    #[test]
+    fn safe_url_blocks_unlisted_host() {
+        let allow = allow_just("cdn.example.com");
+        let err = is_safe_url("https://evil.example.org/img.jpg", &allow).unwrap_err();
+        assert!(matches!(err, UrlSafetyError::HostNotAllowed(h) if h == "evil.example.org"));
+    }
+
+    #[test]
+    fn safe_url_blocks_localhost_even_if_allowlisted() {
+        let allow = allow_just("localhost");
+        assert!(matches!(
+            is_safe_url("http://localhost:8080/", &allow).unwrap_err(),
+            UrlSafetyError::Loopback
+        ));
+    }
+
+    #[test]
+    fn safe_url_blocks_loopback_ipv4() {
+        let allow = allow_just("127.0.0.1");
+        assert!(matches!(
+            is_safe_url("http://127.0.0.1/", &allow).unwrap_err(),
+            UrlSafetyError::PrivateIp(_)
+        ));
+    }
+
+    #[test]
+    fn safe_url_blocks_rfc1918() {
+        let allow = allow_just("10.0.0.1");
+        for url in [
+            "http://10.0.0.1/",
+            "http://192.168.1.1/",
+            "http://172.16.0.5/",
+            "http://172.31.255.255/",
+        ] {
+            assert!(
+                matches!(
+                    is_safe_url(url, &allow).unwrap_err(),
+                    UrlSafetyError::PrivateIp(_)
+                ),
+                "should reject {url}"
+            );
+        }
+    }
+
+    #[test]
+    fn safe_url_blocks_link_local() {
+        let allow = allow_just("169.254.169.254");
+        // 169.254.169.254 is the AWS/GCP metadata service — the most
+        // dangerous SSRF target on a default cloud VM.
+        assert!(matches!(
+            is_safe_url("http://169.254.169.254/", &allow).unwrap_err(),
+            UrlSafetyError::PrivateIp(_)
+        ));
+    }
+
+    #[test]
+    fn safe_url_blocks_ipv6_loopback_and_ula() {
+        // Debug what host_str returns first — reqwest::Url normalises
+        // IPv6 literals as `[::1]` with brackets, which doesn't parse
+        // as `IpAddr` directly. The implementation strips them.
+        let allow = allow_just("[::1]");
+        let err = is_safe_url("http://[::1]/", &allow).unwrap_err();
+        assert!(
+            matches!(err, UrlSafetyError::PrivateIp(_)),
+            "expected PrivateIp, got {err:?}"
+        );
+        let allow = allow_just("[fd00::1]");
+        let err = is_safe_url("http://[fd00::1]/", &allow).unwrap_err();
+        assert!(
+            matches!(err, UrlSafetyError::PrivateIp(_)),
+            "expected PrivateIp, got {err:?}"
+        );
+    }
+
+    #[test]
+    fn safe_url_blocks_ipv4_mapped_ipv6_loopback() {
+        // `Ipv6Addr::is_loopback()` only matches `::1` exactly, so
+        // `::ffff:127.0.0.1` would slip through without the
+        // to_ipv4_mapped() unwrap in is_private_ip.
+        let allow = allow_just("[::ffff:127.0.0.1]");
+        let err = is_safe_url("http://[::ffff:127.0.0.1]/", &allow).unwrap_err();
+        assert!(
+            matches!(err, UrlSafetyError::PrivateIp(_)),
+            "expected PrivateIp, got {err:?}"
+        );
+    }
+
+    #[test]
+    fn safe_url_blocks_ipv4_mapped_ipv6_rfc1918() {
+        let allow = allow_just("[::ffff:10.0.0.1]");
+        let err = is_safe_url("http://[::ffff:10.0.0.1]/", &allow).unwrap_err();
+        assert!(matches!(err, UrlSafetyError::PrivateIp(_)));
+    }
+
+    #[test]
+    fn safe_url_blocks_non_http_schemes() {
+        let allow = allow_just("anywhere");
+        assert!(matches!(
+            is_safe_url("file:///etc/passwd", &allow).unwrap_err(),
+            UrlSafetyError::BadScheme(_)
+        ));
+        assert!(matches!(
+            is_safe_url("gopher://anywhere:70/", &allow).unwrap_err(),
+            UrlSafetyError::BadScheme(_)
+        ));
+    }
+
+    #[test]
+    fn safe_url_rejects_unparseable() {
+        let allow = allow_just("anywhere");
+        assert!(matches!(
+            is_safe_url("not a url", &allow).unwrap_err(),
+            UrlSafetyError::Unparseable
+        ));
+    }
+
+    #[test]
+    fn safe_url_empty_allowlist_rejects_everything() {
+        let allow = DownloadAllowlist::new();
+        let err = is_safe_url("https://cdn.example.com/img.jpg", &allow).unwrap_err();
+        assert!(matches!(err, UrlSafetyError::HostNotAllowed(_)));
+    }
+
+    #[test]
+    fn allowlist_matches_case_insensitively() {
+        let allow = DownloadAllowlist::new().allow("CDN.Example.COM");
+        assert!(is_safe_url("https://cdn.example.com/x.jpg", &allow).is_ok());
+        assert!(is_safe_url("https://CDN.EXAMPLE.com/x.jpg", &allow).is_ok());
+    }
+
+    #[tokio::test]
+    async fn accumulate_capped_returns_full_body_under_cap() {
+        let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
+            Ok(bytes::Bytes::from_static(b"hello ")),
+            Ok(bytes::Bytes::from_static(b"world")),
+        ];
+        let s = stream::iter(chunks);
+        let out = accumulate_capped(s, 100).await.unwrap();
+        assert_eq!(out.as_ref(), b"hello world");
+    }
+
+    #[tokio::test]
+    async fn accumulate_capped_bails_past_cap() {
+        let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
+            Ok(bytes::Bytes::from(vec![0u8; 50])),
+            Ok(bytes::Bytes::from(vec![0u8; 60])),
+        ];
+        let s = stream::iter(chunks);
+        let err = accumulate_capped(s, 100).await.unwrap_err();
+        assert!(err.to_string().contains("100-byte cap"));
+    }
+
+    #[tokio::test]
+    async fn accumulate_capped_surfaces_stream_errors() {
+        let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
+            Ok(bytes::Bytes::from_static(b"ok")),
+            Err(std::io::Error::other("network blip")),
+        ];
+        let s = stream::iter(chunks);
+        let err = accumulate_capped(s, 100).await.unwrap_err();
+        assert!(err.to_string().contains("network blip"));
+    }
+
+    #[test]
+    fn looks_like_image_accepts_jpeg() {
+        // JPEG SOI + APP0 segment.
+        let jpeg = [0xff, 0xd8, 0xff, 0xe0, 0, 0x10, b'J', b'F', b'I', b'F'];
+        assert!(looks_like_image(&jpeg));
+    }
+
+    #[test]
+    fn looks_like_image_accepts_png() {
+        let png = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0];
+        assert!(looks_like_image(&png));
+    }
+
+    #[test]
+    fn looks_like_image_rejects_html_disguised_as_image() {
+        let html = b"<html><body>not an image</body></html>";
+        assert!(!looks_like_image(html));
+    }
+
+    #[test]
+    fn looks_like_image_rejects_empty() {
+        assert!(!looks_like_image(&[]));
+    }
+
+    #[test]
+    fn looks_like_image_rejects_renderable_but_unsupported_formats() {
+        // BMP, TIFF, ICO, PSD are `infer::MatcherType::Image` but the
+        // /files/*key handler doesn't have Content-Type mappings for
+        // them, so they'd be served as application/octet-stream and
+        // download instead of render. Reject at the crawler so we
+        // never land them in storage.
+        // BMP magic: "BM" + 4-byte size.
+        let bmp = [b'B', b'M', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+        assert!(!looks_like_image(&bmp), "BMP must be rejected (not renderable by /files)");
+
+        // TIFF little-endian magic: "II" + 42.
+        let tiff = [0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0];
+        assert!(!looks_like_image(&tiff), "TIFF must be rejected");
+
+        // ICO magic: 0x00,0x00,0x01,0x00.
+        let ico = [0x00, 0x00, 0x01, 0x00, 1, 0, 16, 16, 0, 0, 1, 0, 0x18, 0, 0x40, 0, 0, 0, 0x16, 0, 0, 0];
+        assert!(!looks_like_image(&ico), "ICO must be rejected");
+    }
+
+    #[test]
+    fn looks_like_image_accepts_webp_gif_avif() {
+        // Cover the three remaining whitelisted formats so a future
+        // tightening that drops one would fail noisily.
+        let webp = [
+            b'R', b'I', b'F', b'F',
+            0, 0, 0, 0,
+            b'W', b'E', b'B', b'P',
+            b'V', b'P', b'8', b' ',
+        ];
+        assert!(looks_like_image(&webp));
+
+        let gif = [b'G', b'I', b'F', b'8', b'7', b'a', 0, 0, 0, 0];
+        assert!(looks_like_image(&gif));
+
+        let avif = [
+            0x00, 0x00, 0x00, 0x18,
+            b'f', b't', b'y', b'p',
+            b'a', b'v', b'i', b'f',
+            0x00, 0x00, 0x00, 0x00,
+            b'm', b'i', b'f', b'1',
+            b'a', b'v', b'i', b'f',
+        ];
+        assert!(looks_like_image(&avif));
+    }
+}
--- a/backend/src/crawler/session.rs
+++ b/backend/src/crawler/session.rs
@@ -42,9 +42,36 @@ pub enum SessionProbe {
    Transient,
 }

-/// Re-export so existing callers keep working after the helper moved
-/// to `crawler::url_utils`. The body lives there.
-pub use crate::crawler::url_utils::registrable_domain;
+/// Compute the cookie domain (e.g. `.example.com`) from a start URL.
+/// The leading dot makes the cookie cover every subdomain — the source
+/// often redirects between `www.` and other prefixes mid-crawl, and a
+/// host-only cookie would silently drop on the cross-subdomain hop.
+///
+/// Caveat: this takes the last two dot-labels, which is wrong for
+/// multi-part TLDs (`.co.uk`, `.com.br` would resolve to `.co.uk` and
+/// attach to every site on `.co.uk`). For those, the operator should
+/// override via `CRAWLER_COOKIE_DOMAIN` rather than relying on this
+/// function — pulling in the Public Suffix List for one knob isn't
+/// worth it yet.
+pub fn registrable_domain(url: &str) -> Option<String> {
+    let after_scheme = url.split_once("://")?.1;
+    let host_with_port = after_scheme.split('/').next()?;
+    let host = host_with_port
+        .rsplit_once(':')
+        .map_or(host_with_port, |(h, _)| h)
+        .to_ascii_lowercase();
+    if host.is_empty() {
+        return None;
+    }
+    let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
+    if labels.len() < 2 {
+        // Bare hostname (e.g. `localhost`) — return as-is, no leading
+        // dot. Setting `.localhost` as cookie domain is invalid.
+        return Some(host);
+    }
+    let registrable = &labels[labels.len() - 2..];
+    Some(format!(".{}", registrable.join(".")))
+}

 /// Inject the PHPSESSID cookie into the browser's cookie store for the
 /// catalog domain. Must be called before any navigation that depends on
@@ -100,6 +127,54 @@ pub fn classify_probe(html: &str) -> SessionProbe {
    }
 }

+/// Three-way classification of a chapter page response.
+///
+/// Reader pages don't render `#logo`, so [`classify_probe`] can't be
+/// reused as-is. The chapter-specific marker is `a#pic_container`
+/// (asserted by the reader-page parser at `parse_chapter_pages`).
+///
+/// Order matters: broken-page body wins over selector matches, so a
+/// transient site-wide 5xx that happens to render the avatar widget
+/// elsewhere doesn't falsely reach `Ok`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ChapterProbe {
+    /// `a#pic_container` present — reader rendered. Whether
+    /// `#avatar_menu` is also there is informational; if the reader
+    /// loaded the session is by definition still good.
+    Ok,
+    /// Site rendered a "logged out" or "please log in" page (no
+    /// reader, no broken-page body, and no avatar widget either).
+    /// Distinguishes the genuine expired-session case from a
+    /// transient site hiccup.
+    Unauthenticated,
+    /// Broken-page body, or reader didn't render but the user is
+    /// still logged in (avatar widget present). Caller should retry
+    /// rather than blame the session.
+    Transient,
+}
+
+pub fn classify_chapter_probe(html: &str) -> ChapterProbe {
+    if is_broken_page_body(html) {
+        return ChapterProbe::Transient;
+    }
+    let doc = scraper::Html::parse_document(html);
+    let container = scraper::Selector::parse("a#pic_container").unwrap();
+    if doc.select(&container).next().is_some() {
+        return ChapterProbe::Ok;
+    }
+    let avatar = scraper::Selector::parse("#avatar_menu").unwrap();
+    if doc.select(&avatar).next().is_some() {
+        // Logged-in user, but the reader didn't render — most likely
+        // the layout shifted or the site is serving an interstitial.
+        ChapterProbe::Transient
+    } else {
+        // No reader, no avatar, no broken-body marker — site rendered
+        // the "please log in" page, which is the genuine session-
+        // expired signal on this route.
+        ChapterProbe::Unauthenticated
+    }
+}
+
 /// In-startup retry budget for the session probe. Small but non-zero —
 /// startup hitting a 5-second site hiccup shouldn't fail the operator
 /// with "PHPSESSID expired" when the session is actually fine.
@@ -165,8 +240,44 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
 mod tests {
    use super::*;

-    // registrable_domain tests live in crawler::url_utils now —
-    // it's the canonical home for that helper.
+    #[test]
+    fn registrable_domain_strips_subdomain() {
+        assert_eq!(
+            registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
+            Some(".target-site.com")
+        );
+        assert_eq!(
+            registrable_domain("https://m.example.org").as_deref(),
+            Some(".example.org")
+        );
+    }
+
+    #[test]
+    fn registrable_domain_keeps_two_label_host() {
+        assert_eq!(
+            registrable_domain("https://example.com/").as_deref(),
+            Some(".example.com")
+        );
+    }
+
+    #[test]
+    fn registrable_domain_handles_port() {
+        assert_eq!(
+            registrable_domain("http://www.foo.bar:8080/x").as_deref(),
+            Some(".foo.bar")
+        );
+    }
+
+    #[test]
+    fn registrable_domain_bare_hostname_no_leading_dot() {
+        // .localhost would be invalid as a cookie Domain.
+        assert_eq!(registrable_domain("http://localhost:5173").as_deref(), Some("localhost"));
+    }
+
+    #[test]
+    fn registrable_domain_returns_none_for_garbage() {
+        assert!(registrable_domain("not a url").is_none());
+    }

    #[test]
    fn classify_probe_ok_when_logo_and_avatar_present() {
@@ -210,6 +321,73 @@ mod tests {
        assert_eq!(classify_probe(""), SessionProbe::Transient);
    }

+    #[test]
+    fn classify_chapter_probe_ok_when_reader_rendered() {
+        let html = r#"
+            <html><body>
+              <a id="pic_container">
+                <img id="page1" src="https://cdn/1.jpg">
+              </a>
+            </body></html>
+        "#;
+        assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
+    }
+
+    #[test]
+    fn classify_chapter_probe_unauthenticated_when_no_reader_and_no_avatar() {
+        // What a logged-out hit on a chapter URL renders: a normal
+        // site layout (header etc.) with a "please log in" body, but
+        // no reader and no avatar widget.
+        let html = r#"
+            <html><body>
+              <header><div id="logo">Catalog</div></header>
+              <main>Please log in to read this chapter.</main>
+            </body></html>
+        "#;
+        assert_eq!(
+            classify_chapter_probe(html),
+            ChapterProbe::Unauthenticated
+        );
+    }
+
+    #[test]
+    fn classify_chapter_probe_transient_when_logged_in_but_reader_missing() {
+        // Avatar shows the session is still valid; reader didn't
+        // render — site is serving an interstitial or the layout
+        // momentarily shifted. Retry, don't blame the session.
+        let html = r#"
+            <html><body>
+              <header><div id="logo">Catalog</div><div id="avatar_menu"></div></header>
+              <main>Site maintenance — back in 5 minutes.</main>
+            </body></html>
+        "#;
+        assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
+    }
+
+    #[test]
+    fn classify_chapter_probe_transient_on_broken_page_body() {
+        let html =
+            "<html><body><p>we're sorry, the request file are not found.</p></body></html>";
+        assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
+    }
+
+    #[test]
+    fn classify_chapter_probe_does_not_misfire_on_avatar_alone_without_reader() {
+        // Regression for the original bug: the binary
+        // find_element("#avatar_menu") check treated "no avatar" as
+        // session-expired even when a transient hiccup was the real
+        // cause. classify_chapter_probe must NOT trip on that pattern
+        // when pic_container *is* present.
+        let html = r#"
+            <html><body>
+              <a id="pic_container">
+                <img id="page1" src="https://cdn/1.jpg">
+              </a>
+            </body></html>
+        "#;
+        assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
+    }
+
    #[test]
    fn classify_probe_trusts_broken_body_over_stray_avatar_match() {
        // Defensive: if a broken-page body somehow contains an
--- a/backend/src/crawler/url_utils.rs
+++ b/backend/src/crawler/url_utils.rs
@@ -1,194 +0,0 @@
-//! Centralised URL helpers for the crawler subsystem.
-//!
-//! Three near-identical hand-rolled URL parsers used to live in
-//! `crawler::session`, `crawler::rate_limit`, and `crawler::pipeline`
-//! respectively, each with subtly different edge-case behaviour
-//! around port handling and IPv6 literals. They're consolidated here
-//! so the divergence can't drift again.
-//!
-//! The hand-rolled implementations are kept intentionally — they
-//! preserve the exact semantics every existing test pins. A future
-//! refactor can switch to `reqwest::Url` if it can be done without
-//! changing those semantics.
-
-/// Lowercased host (no port). Returns `None` for inputs without a
-/// `scheme://host` shape — those would never have reached the network
-/// layer anyway. Used by the per-host rate limiter as its bucket key.
-///
-/// IPv6 literals are kept in their `[::1]` bracketed form so the
-/// `rsplit_once(':')` port-stripping logic doesn't split inside the
-/// address (e.g. `https://[::1]/foo` used to return `"[:"` because
-/// the rightmost `:` is inside the literal). Buckets keyed by
-/// `[::1]` vs `::1` are still uniquely-per-host; the brackets are
-/// cosmetic.
-pub fn host_of(url: &str) -> Option<String> {
-    let after_scheme = url.split_once("://")?.1;
-    let host_with_port = after_scheme.split('/').next()?;
-    let host = if host_with_port.starts_with('[') {
-        // IPv6 literal: keep through the closing bracket. There may
-        // be a trailing `:port` after `]`; strip only that.
-        match host_with_port.rfind(']') {
-            Some(end) => &host_with_port[..=end],
-            None => host_with_port,
-        }
-    } else {
-        // Hostnames and IPv4 literals: trailing `:port` (if any) is
-        // after the last `:`.
-        host_with_port
-            .rsplit_once(':')
-            .map_or(host_with_port, |(h, _)| h)
-    };
-    (!host.is_empty()).then(|| host.to_ascii_lowercase())
-}
-
-/// `scheme://host` with no path or port stripping. Used by the metadata
-/// pass to seed `sources.base_url` from `CRAWLER_START_URL`.
-pub fn origin_of(url: &str) -> Option<String> {
-    let (scheme, rest) = url.split_once("://")?;
-    let host = rest.split('/').next()?;
-    Some(format!("{scheme}://{host}"))
-}
-
-/// Approximate registrable-domain calculation: take the last two
-/// dot-labels of the host, prefix with `.`. Used to set a parent-
-/// domain cookie so the catalog's `www.` / `m.` redirects don't drop
-/// the cookie mid-crawl.
-///
-/// Caveat: wrong for multi-part TLDs (`.co.uk`, `.com.br`). The
-/// operator can override via `CRAWLER_COOKIE_DOMAIN`; pulling in the
-/// Public Suffix List for one knob isn't worth it yet.
-///
-/// Bare hostnames (e.g. `localhost`) return the host as-is, with no
-/// leading dot — setting `.localhost` as a cookie domain is invalid.
-/// IPv6 literals (e.g. `[::1]`) are returned bracketed and unchanged;
-/// the browser will reject them as a cookie `Domain` anyway, but the
-/// representation stays sensible. Same `starts_with('[')` branch as
-/// [`host_of`] for consistent IPv6 handling across the module.
-pub fn registrable_domain(url: &str) -> Option<String> {
-    let after_scheme = url.split_once("://")?.1;
-    let host_with_port = after_scheme.split('/').next()?;
-    let host_str = if host_with_port.starts_with('[') {
-        // IPv6 literal: keep through the closing bracket; an optional
-        // `:port` follows `]`.
-        match host_with_port.rfind(']') {
-            Some(end) => &host_with_port[..=end],
-            None => host_with_port,
-        }
-    } else {
-        host_with_port
-            .rsplit_once(':')
-            .map_or(host_with_port, |(h, _)| h)
-    };
-    let host = host_str.to_ascii_lowercase();
-    if host.is_empty() {
-        return None;
-    }
-    let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
-    if labels.len() < 2 {
-        return Some(host);
-    }
-    let registrable = &labels[labels.len() - 2..];
-    Some(format!(".{}", registrable.join(".")))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn host_of_strips_port_and_lowercases() {
-        assert_eq!(
-            host_of("https://CDN.Example.com:443/x").as_deref(),
-            Some("cdn.example.com")
-        );
-        assert_eq!(host_of("http://localhost/").as_deref(), Some("localhost"));
-        assert_eq!(host_of("not a url"), None);
-    }
-
-    #[test]
-    fn host_of_keeps_bracketed_ipv6_literal_intact() {
-        // Regression: the old impl rsplit_once(':')'d the IPv6 address,
-        // returning "[:" instead of "[::1]". A real IPv6 source would
-        // silently get a wrong rate-limit bucket key.
-        assert_eq!(host_of("https://[::1]/").as_deref(), Some("[::1]"));
-        assert_eq!(host_of("https://[::1]:8080/").as_deref(), Some("[::1]"));
-        assert_eq!(
-            host_of("https://[2001:db8::1]/foo").as_deref(),
-            Some("[2001:db8::1]")
-        );
-        assert_eq!(
-            host_of("https://[2001:db8::1]:443/foo").as_deref(),
-            Some("[2001:db8::1]")
-        );
-    }
-
-    #[test]
-    fn origin_of_returns_scheme_and_host() {
-        assert_eq!(
-            origin_of("https://example.com/some/path?q=1").as_deref(),
-            Some("https://example.com")
-        );
-        assert_eq!(origin_of("garbage"), None);
-    }
-
-    #[test]
-    fn registrable_domain_strips_subdomain() {
-        assert_eq!(
-            registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
-            Some(".target-site.com")
-        );
-        assert_eq!(
-            registrable_domain("https://m.example.org").as_deref(),
-            Some(".example.org")
-        );
-    }
-
-    #[test]
-    fn registrable_domain_keeps_two_label_host() {
-        assert_eq!(
-            registrable_domain("https://example.com/").as_deref(),
-            Some(".example.com")
-        );
-    }
-
-    #[test]
-    fn registrable_domain_handles_port() {
-        assert_eq!(
-            registrable_domain("http://www.foo.bar:8080/x").as_deref(),
-            Some(".foo.bar")
-        );
-    }
-
-    #[test]
-    fn registrable_domain_bare_hostname_no_leading_dot() {
-        assert_eq!(
-            registrable_domain("http://localhost:5173").as_deref(),
-            Some("localhost")
-        );
-    }
-
-    #[test]
-    fn registrable_domain_returns_none_for_garbage() {
-        assert!(registrable_domain("not a url").is_none());
-    }
-
-    #[test]
-    fn registrable_domain_keeps_bracketed_ipv6_literal_intact() {
-        // Symmetric with host_of's IPv6 fix. The cookie-domain code
-        // won't accept an IP as a `Domain` value, but the function
-        // should at least return a sensible representation rather
-        // than the truncated `"[:"` the old port-stripper produced.
-        assert_eq!(
-            registrable_domain("https://[::1]/").as_deref(),
-            Some("[::1]")
-        );
-        assert_eq!(
-            registrable_domain("https://[::1]:8080/").as_deref(),
-            Some("[::1]")
-        );
-        assert_eq!(
-            registrable_domain("https://[2001:db8::1]/foo").as_deref(),
-            Some("[2001:db8::1]")
-        );
-    }
-}
--- a/backend/src/repo/author.rs
+++ b/backend/src/repo/author.rs
@@ -99,11 +99,6 @@ pub async fn list(
 /// Atomically replace the set of authors on a manga. Caller passes a
 /// `&mut PgConnection` (`&mut *tx` works) so the delete+upserts run in
 /// one transaction with whatever called us.
-///
-/// Note: `crawler::repo::sync_authors` does a similar replace with the
-/// same semantics on names. The duplication is intentional — handler
-/// callers want the `Vec<AuthorRef>` for the API response; the
-/// crawler doesn't need it and stays inside its own transaction.
 pub async fn set_for_manga(
    conn: &mut PgConnection,
    manga_id: Uuid,
--- a/backend/src/repo/bookmark.rs
+++ b/backend/src/repo/bookmark.rs
@@ -29,9 +29,9 @@ pub async fn create(

    match result {
        Ok(b) => Ok(b),
-        Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => Err(
-            AppError::Conflict("bookmark already exists for this manga/chapter".into()),
-        ),
+        Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(
+            "bookmark already exists for this manga/chapter".into(),
+        )),
        Err(e) => Err(AppError::Database(e)),
    }
 }
@@ -97,3 +97,10 @@ pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> {
    Ok(())
 }

+fn is_unique_violation(err: &sqlx::Error) -> bool {
+    if let sqlx::Error::Database(db_err) = err {
+        db_err.code().as_deref() == Some("23505")
+    } else {
+        false
+    }
+}
--- a/backend/src/repo/chapter.rs
+++ b/backend/src/repo/chapter.rs
@@ -4,7 +4,7 @@ use sqlx::{PgExecutor, PgPool};
 use uuid::Uuid;

 use crate::domain::Chapter;
-use crate::error::AppResult;
+use crate::error::{AppError, AppResult};

 pub async fn list_for_manga(
    pool: &PgPool,
@@ -62,9 +62,10 @@ pub async fn find_by_id_in_manga(
 ///
 /// Chapter identity is the row UUID; the same (manga_id, number)
 /// combination can repeat (multiple translations, re-uploads). The
-/// 0013 migration dropped the (manga_id, number) UNIQUE, so duplicate
-/// inserts succeed by design. If a future migration re-adds any
-/// uniqueness, surface a 409 by adding a unique-violation arm here.
+/// `is_unique_violation` branch below is a defensive holdover from
+/// 0001's (manga_id, number) UNIQUE — it can no longer fire under
+/// normal operation, but we surface a clean 409 if a future migration
+/// re-adds any chapter uniqueness.
 pub async fn create<'e, E: PgExecutor<'e>>(
    executor: E,
    manga_id: Uuid,
@@ -72,7 +73,7 @@ pub async fn create<'e, E: PgExecutor<'e>>(
    title: Option<&str>,
    uploaded_by: Option<Uuid>,
 ) -> AppResult<Chapter> {
-    let row = sqlx::query_as::<_, Chapter>(
+    let result = sqlx::query_as::<_, Chapter>(
        r#"
        INSERT INTO chapters (manga_id, number, title, uploaded_by)
        VALUES ($1, $2, $3, $4)
@@ -84,58 +85,15 @@ pub async fn create<'e, E: PgExecutor<'e>>(
    .bind(title)
    .bind(uploaded_by)
    .fetch_one(executor)
-    .await?;
-    Ok(row)
-}
+    .await;

-/// Cross-link guard for `POST /bookmarks`: the bookmarks FK accepts
-/// any valid chapter id, but a chapter must belong to the bookmark's
-/// manga or the bookmark would dangle on a foreign manga. Handlers
-/// call this before the insert and surface `NotFound` when it
-/// returns `false`.
-pub async fn belongs_to_manga(
-    pool: &PgPool,
-    chapter_id: Uuid,
-    manga_id: Uuid,
-) -> AppResult<bool> {
-    let (exists,): (bool,) = sqlx::query_as(
-        "SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1 AND manga_id = $2)",
-    )
-    .bind(chapter_id)
-    .bind(manga_id)
-    .fetch_one(pool)
-    .await?;
-    Ok(exists)
-}
-
-/// Read just the page_count for a chapter. Used by the crawler
-/// daemon's consumer-side dedup safety net so it can ack-done a job
-/// whose chapter has already been fetched by a racing worker.
-pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result<Option<i32>> {
-    sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
-        .bind(id)
-        .fetch_optional(pool)
-        .await
-}
-
-/// Look up the manga_id + most recent source_url for a chapter. Used
-/// by the daemon's chapter dispatcher to resolve the URL it needs to
-/// hand to `content::sync_chapter_content`. Returns `None` if the
-/// chapter (or its source row) is gone.
-pub async fn dispatch_target(
-    pool: &PgPool,
-    chapter_id: Uuid,
-) -> sqlx::Result<Option<(Uuid, String)>> {
-    sqlx::query_as(
-        "SELECT c.manga_id, cs.source_url \
-           FROM chapters c \
-           JOIN chapter_sources cs ON cs.chapter_id = c.id \
-          WHERE c.id = $1 \
-          LIMIT 1",
-    )
-    .bind(chapter_id)
-    .fetch_optional(pool)
-    .await
+    match result {
+        Ok(c) => Ok(c),
+        Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(format!(
+            "chapter {number} conflicts with an existing chapter for this manga"
+        ))),
+        Err(e) => Err(AppError::Database(e)),
+    }
 }

 pub async fn set_page_count<'e, E: PgExecutor<'e>>(
@@ -151,3 +109,10 @@ pub async fn set_page_count<'e, E: PgExecutor<'e>>(
    Ok(())
 }

+fn is_unique_violation(err: &sqlx::Error) -> bool {
+    if let sqlx::Error::Database(db_err) = err {
+        db_err.code().as_deref() == Some("23505")
+    } else {
+        false
+    }
+}
--- a/backend/src/repo/genre.rs
+++ b/backend/src/repo/genre.rs
@@ -61,11 +61,6 @@ pub async fn load_for_mangas(
 /// FK constraint would reject them, so we filter upstream rather than
 /// surface a 500 here. (The API layer validates the set against
 /// `list_all` first.)
-///
-/// Note: `crawler::repo::sync_genres` does a similar replace, but by
-/// *name* and with auto-create of unseen genres — the crawler can't
-/// validate against the curated vocabulary on its own. Both paths are
-/// intentional; don't merge them without preserving that semantic.
 pub async fn set_for_manga(
    conn: &mut PgConnection,
    manga_id: Uuid,
--- a/backend/src/repo/user.rs
+++ b/backend/src/repo/user.rs
@@ -21,7 +21,7 @@ pub async fn create(pool: &PgPool, username: &str, password_hash: &str) -> AppRe

    match result {
        Ok(user) => Ok(user),
-        Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
+        Err(e) if is_unique_violation(&e) => {
            Err(AppError::Conflict("username is already taken".into()))
        }
        Err(e) => Err(AppError::Database(e)),
@@ -56,3 +56,10 @@ pub async fn find_by_id(pool: &PgPool, id: Uuid) -> AppResult<Option<User>> {
    Ok(row)
 }

+fn is_unique_violation(err: &sqlx::Error) -> bool {
+    if let sqlx::Error::Database(db_err) = err {
+        db_err.code().as_deref() == Some("23505")
+    } else {
+        false
+    }
+}
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
  "name": "mangalord-frontend",
-  "version": "0.34.0",
+  "version": "0.34.1",
  "private": true,
  "type": "module",
  "scripts": {