fix: opt-in CDP re-validation of headless-browser navigations (SSRF)

The reqwest DNS resolver can't see Chromium navigations, so a scraped page
that redirects the browser to an internal target (302 -> 127.0.0.1:5432, or
a rebinding hostname) was still loaded. Add crawler::intercept: an opt-in CDP
Fetch guard that re-validates every Document navigation/redirect through the
same ensure_public_target + resolved-IP check, failing internal targets.

Gated behind CRAWLER_SSRF_INTERCEPT (default OFF): enabling Fetch is a fragile
critical-path hook and the CDP wiring is not CI-verifiable (no Chromium in
CI). When off, open_page is byte-identical to browser.new_page; when on, a
guard-install failure falls back to an unguarded navigation rather than
wedging the crawl. The pure decision logic (verdict/is_blocked) is unit-tested;
an #[ignore] smoke test covers the CDP path. Validate with a manual crawl
before enabling in production.

Bump to 0.124.11.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/src/crawler/content.rs

@@ -118,8 +118,7 @@ async fn fetch_chapter_html_once(
 ) -> anyhow::Result<String> {
     guard_nav_url(source_url)?;
     rate.wait_for(source_url).await?;
-    let page = browser
-        .new_page(source_url)
+    let page = crate::crawler::intercept::open_page(browser, source_url)
         .await
         .with_context(|| format!("open chapter page {source_url}"))?;
     // Close the tab on every exit path — a `?` on wait_for_nav / content()

backend/src/crawler/intercept.rs

@@ -0,0 +1,257 @@
+//! Optional CDP-level SSRF guard for headless-browser navigations.
+//!
+//! The reqwest clients get a DNS-filtering resolver (see
+//! [`crate::crawler::safety::SafeResolver`]) so an image/API host that
+//! resolves to an internal IP is refused at connect time. Chromium, however,
+//! does its **own** DNS and connection handling, so that resolver can't see
+//! browser navigations. Without a second guard, a scraped chapter page that
+//! `302`-redirects the browser to `http://127.0.0.1:5432/` (or an
+//! attacker-owned hostname that resolves to `169.254.169.254`) would be
+//! loaded and parsed as if it were catalog content.
+//!
+//! This module installs CDP `Fetch` interception on a page so every
+//! **Document** (main-frame) request and redirect is re-validated through the
+//! same [`ensure_public_target`] + resolved-IP check the reqwest paths use,
+//! failing any that target an internal address.
+//!
+//! **Opt-in / default-off.** Enabling `Fetch` means every intercepted request
+//! *must* be resolved by a live handler or the navigation hangs, so this is a
+//! fragile hook in the crawler's critical path. It ships behind
+//! `CRAWLER_SSRF_INTERCEPT` (default `false`) and the wiring has **not** been
+//! exercised against a real Chromium in CI — validate with a manual crawl
+//! before enabling in production. When disabled, [`open_page`] is byte-for-byte
+//! the previous `browser.new_page(url)` behavior. Even when enabled, a failure
+//! to install the guard falls back to an unguarded navigation rather than
+//! failing the crawl.
+
+use std::net::IpAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use chromiumoxide::browser::Browser;
+use chromiumoxide::cdp::browser_protocol::fetch::{
+    ContinueRequestParams, EnableParams, EventRequestPaused, FailRequestParams, RequestPattern,
+    RequestStage,
+};
+use chromiumoxide::cdp::browser_protocol::network::{ErrorReason, ResourceType};
+use chromiumoxide::error::Result as CdpResult;
+use chromiumoxide::Page;
+use futures_util::StreamExt;
+use reqwest::Url;
+
+use crate::crawler::safety::{ensure_public_target, is_private_ip};
+
+/// Process-wide toggle, set once at startup from `CRAWLER_SSRF_INTERCEPT`.
+/// A single boot-time flag (rather than threading a param through every
+/// crawler navigation signature) keeps the off-path a no-op.
+static ENABLED: AtomicBool = AtomicBool::new(false);
+
+pub fn set_enabled(on: bool) {
+    ENABLED.store(on, Ordering::Relaxed);
+}
+
+pub fn is_enabled() -> bool {
+    ENABLED.load(Ordering::Relaxed)
+}
+
+/// What to do with a navigation URL, decided without DNS where possible so the
+/// security-critical branching is unit-testable.
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum Verdict {
+    /// Safe to continue (non-network scheme, or a public IP literal).
+    Allow,
+    /// Refuse — bad scheme handled elsewhere, localhost, or a private IP literal.
+    Block,
+    /// A hostname that must be resolved to decide (DNS-rebinding check).
+    ResolveHost { host: String, port: u16 },
+}
+
+/// Pure decision over a URL string. `data:` / `blob:` / `about:` and other
+/// non-http(s) schemes are allowed (not a network-SSRF vector); http(s) with a
+/// private/loopback/localhost literal is blocked; an http(s) hostname needs
+/// resolution.
+pub(crate) fn verdict(url: &str) -> Verdict {
+    let Ok(parsed) = Url::parse(url) else {
+        // Unparseable — let Chromium reject it; not our call to make.
+        return Verdict::Allow;
+    };
+    match parsed.scheme() {
+        "http" | "https" => {}
+        _ => return Verdict::Allow,
+    }
+    // Literal private IP / localhost / missing host → block (string check).
+    if ensure_public_target(url).is_err() {
+        return Verdict::Block;
+    }
+    match parsed.host_str() {
+        Some(host) if !is_ip_literal(host) => Verdict::ResolveHost {
+            host: host.to_string(),
+            port: parsed.port_or_known_default().unwrap_or(80),
+        },
+        // Public IP literal (ensure_public_target already passed it).
+        _ => Verdict::Allow,
+    }
+}
+
+/// Whether `host` (as reqwest's `host_str()` yields it) is an IP literal.
+/// IPv6 literals arrive bracketed (`[::1]`), which don't parse as `IpAddr`
+/// directly — strip the brackets first.
+fn is_ip_literal(host: &str) -> bool {
+    let unbracketed = host
+        .strip_prefix('[')
+        .and_then(|s| s.strip_suffix(']'))
+        .unwrap_or(host);
+    unbracketed.parse::<IpAddr>().is_ok()
+}
+
+/// Async decision: resolves hostnames and blocks if any resolved address is
+/// private (DNS rebinding). A resolution failure is *not* treated as blocked —
+/// Chromium won't be able to connect either, so there's nothing to exfiltrate.
+pub(crate) async fn is_blocked(url: &str) -> bool {
+    match verdict(url) {
+        Verdict::Allow => false,
+        Verdict::Block => true,
+        Verdict::ResolveHost { host, port } => match tokio::net::lookup_host((host.as_str(), port))
+            .await
+        {
+            Ok(addrs) => addrs.map(|a| a.ip()).any(|ip| is_private_ip(&ip)),
+            Err(_) => false,
+        },
+    }
+}
+
+/// Open a page for navigation. With interception off, this is exactly
+/// `browser.new_page(url)`. With it on, the page is created blank (no network),
+/// the navigation guard is installed, and only then does it navigate — so the
+/// initial request and any redirects pass through the guard.
+pub async fn open_page(browser: &Browser, url: &str) -> CdpResult<Page> {
+    if !is_enabled() {
+        return browser.new_page(url).await;
+    }
+    let page = browser.new_page("about:blank").await?;
+    if let Err(e) = install_navigation_guard(&page).await {
+        // Fail open on install error: navigate unguarded rather than wedge the
+        // crawl. The reqwest-layer resolver still covers image downloads.
+        tracing::warn!(url = %url, error = %e, "SSRF navigation guard failed to install; navigating unguarded");
+    }
+    page.goto(url).await?;
+    Ok(page)
+}
+
+/// Enable `Fetch` for Document requests on `page` and spawn a task that
+/// continues/fails each paused request per [`is_blocked`].
+async fn install_navigation_guard(page: &Page) -> CdpResult<()> {
+    // Intercept only main-frame Document requests at the request stage — a
+    // navigation plus its redirects, nothing else. Keeps the paused-request
+    // volume tiny so the handler can't become a page-load bottleneck.
+    let pattern = RequestPattern::builder()
+        .resource_type(ResourceType::Document)
+        .request_stage(RequestStage::Request)
+        .build();
+    page.execute(EnableParams {
+        patterns: Some(vec![pattern]),
+        handle_auth_requests: None,
+    })
+    .await?;
+
+    let mut paused = page.event_listener::<EventRequestPaused>().await?;
+    let handler_page = page.clone();
+    tokio::spawn(async move {
+        while let Some(ev) = paused.next().await {
+            let request_id = ev.request_id.clone();
+            let outcome = if is_blocked(&ev.request.url).await {
+                tracing::warn!(
+                    url = %ev.request.url,
+                    "SSRF guard blocked browser navigation to an internal target"
+                );
+                handler_page
+                    .execute(FailRequestParams::new(request_id, ErrorReason::BlockedByClient))
+                    .await
+                    .map(|_| ())
+            } else {
+                handler_page
+                    .execute(ContinueRequestParams::new(request_id))
+                    .await
+                    .map(|_| ())
+            };
+            if let Err(e) = outcome {
+                // The page/session is gone (page closed) — the stream will end
+                // too; stop handling so the task exits rather than spins.
+                tracing::debug!(error = %e, "fetch interceptor: continue/fail failed, ending handler");
+                break;
+            }
+        }
+    });
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn verdict_allows_non_network_schemes() {
+        for url in ["about:blank", "data:text/html,hi", "blob:abc", "chrome://version"] {
+            assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
+        }
+    }
+
+    #[test]
+    fn verdict_blocks_private_ip_literals_and_localhost() {
+        for url in [
+            "http://127.0.0.1/",
+            "http://10.0.0.1/",
+            "http://192.168.1.1:5432/",
+            "http://169.254.169.254/latest/meta-data/",
+            "http://[::1]/",
+            "http://[::ffff:127.0.0.1]/",
+            "http://localhost/",
+            "https://[fd00::1]/",
+        ] {
+            assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
+        }
+    }
+
+    #[test]
+    fn verdict_allows_public_ip_literals() {
+        for url in ["http://93.184.216.34/", "https://[2606:4700:4700::1111]/"] {
+            assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
+        }
+    }
+
+    #[test]
+    fn verdict_defers_hostnames_to_resolution() {
+        assert_eq!(
+            verdict("https://cdn.example.com/img.jpg"),
+            Verdict::ResolveHost {
+                host: "cdn.example.com".to_string(),
+                port: 443
+            }
+        );
+        assert_eq!(
+            verdict("http://catalog.test:8080/list"),
+            Verdict::ResolveHost {
+                host: "catalog.test".to_string(),
+                port: 8080
+            }
+        );
+    }
+
+    #[tokio::test]
+    async fn is_blocked_true_for_private_literal_no_dns() {
+        assert!(is_blocked("http://169.254.169.254/").await);
+        assert!(!is_blocked("http://93.184.216.34/").await);
+        // Non-network scheme is always allowed through.
+        assert!(!is_blocked("about:blank").await);
+    }
+
+    #[test]
+    fn enabled_toggle_roundtrips() {
+        // Global; restore afterwards so other tests see the default.
+        let prev = is_enabled();
+        set_enabled(true);
+        assert!(is_enabled());
+        set_enabled(false);
+        assert!(!is_enabled());
+        set_enabled(prev);
+    }
+}

backend/src/crawler/mod.rs

@@ -19,6 +19,7 @@ pub mod content;
 pub mod daemon;
 pub mod detect;
 pub mod diff;
+pub mod intercept;
 pub mod jobs;
 pub mod nav;
 pub mod pipeline;

backend/src/crawler/session.rs

@@ -291,8 +291,7 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
     // the URL ever becomes attacker-influenced.
     crate::crawler::safety::ensure_public_target(probe_url)
         .with_context(|| format!("refuse to navigate unsafe probe URL {probe_url}"))?;
-    let page = browser
-        .new_page(probe_url)
+    let page = crate::crawler::intercept::open_page(browser, probe_url)
         .await
         .with_context(|| format!("open probe page {probe_url}"))?;
     // Close the tab on every exit path — a `?` on wait_for_nav / content()

backend/src/crawler/source/target.rs

@@ -253,9 +253,7 @@ async fn navigate(
 ) -> Result<String, PageError> {
     guard_navigate_url(url)?;
     ctx.rate.wait_for(url).await?;
-    let page = ctx
-        .browser
-        .new_page(url)
+    let page = crate::crawler::intercept::open_page(ctx.browser, url)
         .await
         .map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
     // Close the tab on every exit path — the previous code closed on the two