fix: opt-in CDP re-validation of headless-browser navigations (SSRF)

The reqwest DNS resolver can't see Chromium navigations, so a scraped page
that redirects the browser to an internal target (302 -> 127.0.0.1:5432, or
a rebinding hostname) was still loaded. Add crawler::intercept: an opt-in CDP
Fetch guard that re-validates every Document navigation/redirect through the
same ensure_public_target + resolved-IP check, failing internal targets.

Gated behind CRAWLER_SSRF_INTERCEPT (default OFF): enabling Fetch is a fragile
critical-path hook and the CDP wiring is not CI-verifiable (no Chromium in
CI). When off, open_page is byte-identical to browser.new_page; when on, a
guard-install failure falls back to an unguarded navigation rather than
wedging the crawl. The pure decision logic (verdict/is_blocked) is unit-tested;
an #[ignore] smoke test covers the CDP path. Validate with a manual crawl
before enabling in production.

Bump to 0.124.11.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.env.example

@@ -184,6 +184,12 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
 # exhausted) that trigger an automatic coordinated browser restart.
 # Default 3.
 CRAWLER_BROWSER_RESTART_THRESHOLD=3
+# Opt-in CDP Fetch interception that re-validates every headless-browser
+# navigation/redirect against the SSRF check (blocks a scraped page that
+# redirects the browser to an internal target). Default false — enabling
+# Fetch is a fragile hook in the crawler's critical path and is not
+# CI-verified; validate with a manual crawl before turning on.
+CRAWLER_SSRF_INTERCEPT=false
 # Path to a system Chromium binary. When set, the crawler skips the
 # bundled-fetcher download. Required on platforms without a usable
 # upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On

backend/Cargo.lock

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.124.10"
+version = "0.124.11"
 dependencies = [
  "anyhow",
  "argon2",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.124.10"
+version = "0.124.11"
 edition = "2021"
 default-run = "mangalord"
 

backend/src/app.rs

@@ -564,6 +564,10 @@ async fn spawn_crawler_daemon(
     cfg: &CrawlerConfig,
     analysis_enabled: Arc<AtomicBool>,
 ) -> anyhow::Result<SpawnedDaemon> {
+    // Publish the opt-in browser SSRF-interception toggle so every headless
+    // navigation (via `intercept::open_page`) honours it. Off by default.
+    crate::crawler::intercept::set_enabled(cfg.ssrf_intercept);
+
     // Reqwest client with a shared cookie jar so CDN image fetches include
     // PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
     // runtime session refresh rewrites it in place. Initial value: a

backend/src/bin/crawler.rs

@@ -138,6 +138,13 @@ async fn main() -> anyhow::Result<()> {
     }
     let http = http_builder.build().context("build http client")?;
 
+    // Opt-in browser SSRF interception (default off), mirroring the daemon.
+    mangalord::crawler::intercept::set_enabled(
+        std::env::var("CRAWLER_SSRF_INTERCEPT")
+            .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes"))
+            .unwrap_or(false),
+    );
+
     let mut options = LaunchOptions::from_env();
     if let Some(proxy) = &proxy_url {
         let chromium_proxy = mangalord::crawler::url_utils::chromium_proxy_arg(proxy);

backend/src/config.rs

@@ -511,6 +511,13 @@ pub struct CrawlerConfig {
     /// exhausted) that trigger an automatic coordinated browser restart.
     /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
     pub browser_restart_threshold: u32,
+    /// Opt-in CDP `Fetch` interception that re-validates every headless-browser
+    /// navigation/redirect against the SSRF check (blocks a scraped page that
+    /// redirects the browser to an internal target). Default `false`: enabling
+    /// `Fetch` is a fragile hook in the crawler's critical path and the wiring
+    /// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl
+    /// before turning on. `CRAWLER_SSRF_INTERCEPT`.
+    pub ssrf_intercept: bool,
 }
 
 impl Default for CrawlerConfig {
@@ -543,6 +550,7 @@ impl Default for CrawlerConfig {
             job_timeout: Duration::from_secs(600),
             metadata_max_consecutive_failures: 10,
             browser_restart_threshold: 3,
+            ssrf_intercept: false,
         }
     }
 }
@@ -707,6 +715,7 @@ impl CrawlerConfig {
             ) as u32,
             browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
                 as u32,
+            ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false),
         })
     }
 }

backend/src/crawler/content.rs

@@ -118,8 +118,7 @@ async fn fetch_chapter_html_once(
 ) -> anyhow::Result<String> {
     guard_nav_url(source_url)?;
     rate.wait_for(source_url).await?;
-    let page = browser
+    let page = crate::crawler::intercept::open_page(browser, source_url)
-        .new_page(source_url)
         .await
         .with_context(|| format!("open chapter page {source_url}"))?;
     // Close the tab on every exit path — a `?` on wait_for_nav / content()

backend/src/crawler/intercept.rs

@@ -0,0 +1,257 @@
+//! Optional CDP-level SSRF guard for headless-browser navigations.
+//!
+//! The reqwest clients get a DNS-filtering resolver (see
+//! [`crate::crawler::safety::SafeResolver`]) so an image/API host that
+//! resolves to an internal IP is refused at connect time. Chromium, however,
+//! does its **own** DNS and connection handling, so that resolver can't see
+//! browser navigations. Without a second guard, a scraped chapter page that
+//! `302`-redirects the browser to `http://127.0.0.1:5432/` (or an
+//! attacker-owned hostname that resolves to `169.254.169.254`) would be
+//! loaded and parsed as if it were catalog content.
+//!
+//! This module installs CDP `Fetch` interception on a page so every
+//! **Document** (main-frame) request and redirect is re-validated through the
+//! same [`ensure_public_target`] + resolved-IP check the reqwest paths use,
+//! failing any that target an internal address.
+//!
+//! **Opt-in / default-off.** Enabling `Fetch` means every intercepted request
+//! *must* be resolved by a live handler or the navigation hangs, so this is a
+//! fragile hook in the crawler's critical path. It ships behind
+//! `CRAWLER_SSRF_INTERCEPT` (default `false`) and the wiring has **not** been
+//! exercised against a real Chromium in CI — validate with a manual crawl
+//! before enabling in production. When disabled, [`open_page`] is byte-for-byte
+//! the previous `browser.new_page(url)` behavior. Even when enabled, a failure
+//! to install the guard falls back to an unguarded navigation rather than
+//! failing the crawl.
+
+use std::net::IpAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use chromiumoxide::browser::Browser;
+use chromiumoxide::cdp::browser_protocol::fetch::{
+    ContinueRequestParams, EnableParams, EventRequestPaused, FailRequestParams, RequestPattern,
+    RequestStage,
+};
+use chromiumoxide::cdp::browser_protocol::network::{ErrorReason, ResourceType};
+use chromiumoxide::error::Result as CdpResult;
+use chromiumoxide::Page;
+use futures_util::StreamExt;
+use reqwest::Url;
+
+use crate::crawler::safety::{ensure_public_target, is_private_ip};
+
+/// Process-wide toggle, set once at startup from `CRAWLER_SSRF_INTERCEPT`.
+/// A single boot-time flag (rather than threading a param through every
+/// crawler navigation signature) keeps the off-path a no-op.
+static ENABLED: AtomicBool = AtomicBool::new(false);
+
+pub fn set_enabled(on: bool) {
+    ENABLED.store(on, Ordering::Relaxed);
+}
+
+pub fn is_enabled() -> bool {
+    ENABLED.load(Ordering::Relaxed)
+}
+
+/// What to do with a navigation URL, decided without DNS where possible so the
+/// security-critical branching is unit-testable.
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum Verdict {
+    /// Safe to continue (non-network scheme, or a public IP literal).
+    Allow,
+    /// Refuse — bad scheme handled elsewhere, localhost, or a private IP literal.
+    Block,
+    /// A hostname that must be resolved to decide (DNS-rebinding check).
+    ResolveHost { host: String, port: u16 },
+}
+
+/// Pure decision over a URL string. `data:` / `blob:` / `about:` and other
+/// non-http(s) schemes are allowed (not a network-SSRF vector); http(s) with a
+/// private/loopback/localhost literal is blocked; an http(s) hostname needs
+/// resolution.
+pub(crate) fn verdict(url: &str) -> Verdict {
+    let Ok(parsed) = Url::parse(url) else {
+        // Unparseable — let Chromium reject it; not our call to make.
+        return Verdict::Allow;
+    };
+    match parsed.scheme() {
+        "http" | "https" => {}
+        _ => return Verdict::Allow,
+    }
+    // Literal private IP / localhost / missing host → block (string check).
+    if ensure_public_target(url).is_err() {
+        return Verdict::Block;
+    }
+    match parsed.host_str() {
+        Some(host) if !is_ip_literal(host) => Verdict::ResolveHost {
+            host: host.to_string(),
+            port: parsed.port_or_known_default().unwrap_or(80),
+        },
+        // Public IP literal (ensure_public_target already passed it).
+        _ => Verdict::Allow,
+    }
+}
+
+/// Whether `host` (as reqwest's `host_str()` yields it) is an IP literal.
+/// IPv6 literals arrive bracketed (`[::1]`), which don't parse as `IpAddr`
+/// directly — strip the brackets first.
+fn is_ip_literal(host: &str) -> bool {
+    let unbracketed = host
+        .strip_prefix('[')
+        .and_then(|s| s.strip_suffix(']'))
+        .unwrap_or(host);
+    unbracketed.parse::<IpAddr>().is_ok()
+}
+
+/// Async decision: resolves hostnames and blocks if any resolved address is
+/// private (DNS rebinding). A resolution failure is *not* treated as blocked —
+/// Chromium won't be able to connect either, so there's nothing to exfiltrate.
+pub(crate) async fn is_blocked(url: &str) -> bool {
+    match verdict(url) {
+        Verdict::Allow => false,
+        Verdict::Block => true,
+        Verdict::ResolveHost { host, port } => match tokio::net::lookup_host((host.as_str(), port))
+            .await
+        {
+            Ok(addrs) => addrs.map(|a| a.ip()).any(|ip| is_private_ip(&ip)),
+            Err(_) => false,
+        },
+    }
+}
+
+/// Open a page for navigation. With interception off, this is exactly
+/// `browser.new_page(url)`. With it on, the page is created blank (no network),
+/// the navigation guard is installed, and only then does it navigate — so the
+/// initial request and any redirects pass through the guard.
+pub async fn open_page(browser: &Browser, url: &str) -> CdpResult<Page> {
+    if !is_enabled() {
+        return browser.new_page(url).await;
+    }
+    let page = browser.new_page("about:blank").await?;
+    if let Err(e) = install_navigation_guard(&page).await {
+        // Fail open on install error: navigate unguarded rather than wedge the
+        // crawl. The reqwest-layer resolver still covers image downloads.
+        tracing::warn!(url = %url, error = %e, "SSRF navigation guard failed to install; navigating unguarded");
+    }
+    page.goto(url).await?;
+    Ok(page)
+}
+
+/// Enable `Fetch` for Document requests on `page` and spawn a task that
+/// continues/fails each paused request per [`is_blocked`].
+async fn install_navigation_guard(page: &Page) -> CdpResult<()> {
+    // Intercept only main-frame Document requests at the request stage — a
+    // navigation plus its redirects, nothing else. Keeps the paused-request
+    // volume tiny so the handler can't become a page-load bottleneck.
+    let pattern = RequestPattern::builder()
+        .resource_type(ResourceType::Document)
+        .request_stage(RequestStage::Request)
+        .build();
+    page.execute(EnableParams {
+        patterns: Some(vec![pattern]),
+        handle_auth_requests: None,
+    })
+    .await?;
+
+    let mut paused = page.event_listener::<EventRequestPaused>().await?;
+    let handler_page = page.clone();
+    tokio::spawn(async move {
+        while let Some(ev) = paused.next().await {
+            let request_id = ev.request_id.clone();
+            let outcome = if is_blocked(&ev.request.url).await {
+                tracing::warn!(
+                    url = %ev.request.url,
+                    "SSRF guard blocked browser navigation to an internal target"
+                );
+                handler_page
+                    .execute(FailRequestParams::new(request_id, ErrorReason::BlockedByClient))
+                    .await
+                    .map(|_| ())
+            } else {
+                handler_page
+                    .execute(ContinueRequestParams::new(request_id))
+                    .await
+                    .map(|_| ())
+            };
+            if let Err(e) = outcome {
+                // The page/session is gone (page closed) — the stream will end
+                // too; stop handling so the task exits rather than spins.
+                tracing::debug!(error = %e, "fetch interceptor: continue/fail failed, ending handler");
+                break;
+            }
+        }
+    });
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn verdict_allows_non_network_schemes() {
+        for url in ["about:blank", "data:text/html,hi", "blob:abc", "chrome://version"] {
+            assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
+        }
+    }
+
+    #[test]
+    fn verdict_blocks_private_ip_literals_and_localhost() {
+        for url in [
+            "http://127.0.0.1/",
+            "http://10.0.0.1/",
+            "http://192.168.1.1:5432/",
+            "http://169.254.169.254/latest/meta-data/",
+            "http://[::1]/",
+            "http://[::ffff:127.0.0.1]/",
+            "http://localhost/",
+            "https://[fd00::1]/",
+        ] {
+            assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
+        }
+    }
+
+    #[test]
+    fn verdict_allows_public_ip_literals() {
+        for url in ["http://93.184.216.34/", "https://[2606:4700:4700::1111]/"] {
+            assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
+        }
+    }
+
+    #[test]
+    fn verdict_defers_hostnames_to_resolution() {
+        assert_eq!(
+            verdict("https://cdn.example.com/img.jpg"),
+            Verdict::ResolveHost {
+                host: "cdn.example.com".to_string(),
+                port: 443
+            }
+        );
+        assert_eq!(
+            verdict("http://catalog.test:8080/list"),
+            Verdict::ResolveHost {
+                host: "catalog.test".to_string(),
+                port: 8080
+            }
+        );
+    }
+
+    #[tokio::test]
+    async fn is_blocked_true_for_private_literal_no_dns() {
+        assert!(is_blocked("http://169.254.169.254/").await);
+        assert!(!is_blocked("http://93.184.216.34/").await);
+        // Non-network scheme is always allowed through.
+        assert!(!is_blocked("about:blank").await);
+    }
+
+    #[test]
+    fn enabled_toggle_roundtrips() {
+        // Global; restore afterwards so other tests see the default.
+        let prev = is_enabled();
+        set_enabled(true);
+        assert!(is_enabled());
+        set_enabled(false);
+        assert!(!is_enabled());
+        set_enabled(prev);
+    }
+}

backend/src/crawler/mod.rs

@@ -19,6 +19,7 @@ pub mod content;
 pub mod daemon;
 pub mod detect;
 pub mod diff;
+pub mod intercept;
 pub mod jobs;
 pub mod nav;
 pub mod pipeline;

backend/src/crawler/session.rs

@@ -291,8 +291,7 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
     // the URL ever becomes attacker-influenced.
     crate::crawler::safety::ensure_public_target(probe_url)
         .with_context(|| format!("refuse to navigate unsafe probe URL {probe_url}"))?;
-    let page = browser
+    let page = crate::crawler::intercept::open_page(browser, probe_url)
-        .new_page(probe_url)
         .await
         .with_context(|| format!("open probe page {probe_url}"))?;
     // Close the tab on every exit path — a `?` on wait_for_nav / content()

backend/src/crawler/source/target.rs

@@ -253,9 +253,7 @@ async fn navigate(
 ) -> Result<String, PageError> {
     guard_navigate_url(url)?;
     ctx.rate.wait_for(url).await?;
-    let page = ctx
+    let page = crate::crawler::intercept::open_page(ctx.browser, url)
-        .browser
-        .new_page(url)
         .await
         .map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
     // Close the tab on every exit path — the previous code closed on the two

backend/src/settings.rs

@@ -269,6 +269,7 @@ impl CrawlerSettings {
             tor_control_cookie_path: base.tor_control_cookie_path.clone(),
             tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
             browser: base.browser.clone(),
+            ssrf_intercept: base.ssrf_intercept,
         })
     }
 }

backend/tests/crawler_browser_smoke.rs

@@ -62,6 +62,43 @@ async fn headless_browser_can_navigate_and_read_title() {
     handle.close().await.expect("close cleanly");
 }
 
+/// Smoke-test the opt-in SSRF navigation guard (`CRAWLER_SSRF_INTERCEPT`).
+/// With interception ON, a normal (allowed) navigation must still complete —
+/// i.e. enabling CDP `Fetch` and installing the request handler must NOT wedge
+/// page loads. This is the regression the interceptor's fragility risks; it
+/// can only be exercised with a real Chromium, hence `#[ignore]`.
+///
+/// (The private-target *blocking* path is covered by the pure-logic unit tests
+/// in `crawler::intercept` — `verdict` / `is_blocked` — which don't need a
+/// browser.)
+#[tokio::test]
+#[ignore = "downloads Chromium; run with --ignored"]
+async fn ssrf_interception_does_not_wedge_allowed_navigation() {
+    use mangalord::crawler::intercept;
+
+    const PAGE: &str =
+        "data:text/html,<html><head><title>Guarded%20OK</title></head><body></body></html>";
+
+    intercept::set_enabled(true);
+    let handle = browser::launch(LaunchOptions::headless())
+        .await
+        .expect("launch headless chromium");
+
+    // Route through the guarded opener (blank page -> Fetch.enable -> handler
+    // -> goto). If the handler failed to continue the navigation, this would
+    // hang until the test harness times out.
+    let page = intercept::open_page(handle.browser(), PAGE)
+        .await
+        .expect("guarded open_page");
+    page.wait_for_navigation().await.expect("wait for navigation");
+
+    let title = page.get_title().await.expect("get title");
+    assert_eq!(title.as_deref(), Some("Guarded OK"));
+
+    handle.close().await.expect("close cleanly");
+    intercept::set_enabled(false);
+}
+
 /// Live end-to-end: navigate to a real page, get the rendered HTML, and
 /// parse it with `scraper`. ipify.org renders the visitor's public IP
 /// into the page DOM, so a successful run proves browser → render →

docker-compose.yml

@@ -108,6 +108,7 @@ services:
       CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600}
       CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10}
       CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3}
+      CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false}
       # Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps
       # the in-process scheduler off; the dashboard force-resync still works.
       CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true}

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.124.10",
+  "version": "0.124.11",
   "private": true,
   "type": "module",
   "scripts": {