diff --git a/.env.example b/.env.example index 66700a6..63e12ef 100644 --- a/.env.example +++ b/.env.example @@ -184,6 +184,12 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10 # exhausted) that trigger an automatic coordinated browser restart. # Default 3. CRAWLER_BROWSER_RESTART_THRESHOLD=3 +# Opt-in CDP Fetch interception that re-validates every headless-browser +# navigation/redirect against the SSRF check (blocks a scraped page that +# redirects the browser to an internal target). Default false — enabling +# Fetch is a fragile hook in the crawler's critical path and is not +# CI-verified; validate with a manual crawl before turning on. +CRAWLER_SSRF_INTERCEPT=false # Path to a system Chromium binary. When set, the crawler skips the # bundled-fetcher download. Required on platforms without a usable # upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7d67fa3..e7d0c52 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.124.10" +version = "0.124.11" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 35b1a9f..6e5d191 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.124.10" +version = "0.124.11" edition = "2021" default-run = "mangalord" diff --git a/backend/src/app.rs b/backend/src/app.rs index 2cc620c..472bc11 100644 --- a/backend/src/app.rs +++ b/backend/src/app.rs @@ -564,6 +564,10 @@ async fn spawn_crawler_daemon( cfg: &CrawlerConfig, analysis_enabled: Arc, ) -> anyhow::Result { + // Publish the opt-in browser SSRF-interception toggle so every headless + // navigation (via `intercept::open_page`) honours it. Off by default. + crate::crawler::intercept::set_enabled(cfg.ssrf_intercept); + // Reqwest client with a shared cookie jar so CDN image fetches include // PHPSESSID. The same `Arc` is held by the SessionController, so a // runtime session refresh rewrites it in place. Initial value: a diff --git a/backend/src/bin/crawler.rs b/backend/src/bin/crawler.rs index b2f8f99..1d4cb41 100644 --- a/backend/src/bin/crawler.rs +++ b/backend/src/bin/crawler.rs @@ -138,6 +138,13 @@ async fn main() -> anyhow::Result<()> { } let http = http_builder.build().context("build http client")?; + // Opt-in browser SSRF interception (default off), mirroring the daemon. + mangalord::crawler::intercept::set_enabled( + std::env::var("CRAWLER_SSRF_INTERCEPT") + .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes")) + .unwrap_or(false), + ); + let mut options = LaunchOptions::from_env(); if let Some(proxy) = &proxy_url { let chromium_proxy = mangalord::crawler::url_utils::chromium_proxy_arg(proxy); diff --git a/backend/src/config.rs b/backend/src/config.rs index 06da544..552cd3b 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -511,6 +511,13 @@ pub struct CrawlerConfig { /// exhausted) that trigger an automatic coordinated browser restart. /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`. pub browser_restart_threshold: u32, + /// Opt-in CDP `Fetch` interception that re-validates every headless-browser + /// navigation/redirect against the SSRF check (blocks a scraped page that + /// redirects the browser to an internal target). Default `false`: enabling + /// `Fetch` is a fragile hook in the crawler's critical path and the wiring + /// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl + /// before turning on. `CRAWLER_SSRF_INTERCEPT`. + pub ssrf_intercept: bool, } impl Default for CrawlerConfig { @@ -543,6 +550,7 @@ impl Default for CrawlerConfig { job_timeout: Duration::from_secs(600), metadata_max_consecutive_failures: 10, browser_restart_threshold: 3, + ssrf_intercept: false, } } } @@ -707,6 +715,7 @@ impl CrawlerConfig { ) as u32, browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1) as u32, + ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false), }) } } diff --git a/backend/src/crawler/content.rs b/backend/src/crawler/content.rs index f89d0d6..d813a9f 100644 --- a/backend/src/crawler/content.rs +++ b/backend/src/crawler/content.rs @@ -118,8 +118,7 @@ async fn fetch_chapter_html_once( ) -> anyhow::Result { guard_nav_url(source_url)?; rate.wait_for(source_url).await?; - let page = browser - .new_page(source_url) + let page = crate::crawler::intercept::open_page(browser, source_url) .await .with_context(|| format!("open chapter page {source_url}"))?; // Close the tab on every exit path — a `?` on wait_for_nav / content() diff --git a/backend/src/crawler/intercept.rs b/backend/src/crawler/intercept.rs new file mode 100644 index 0000000..6db78ee --- /dev/null +++ b/backend/src/crawler/intercept.rs @@ -0,0 +1,257 @@ +//! Optional CDP-level SSRF guard for headless-browser navigations. +//! +//! The reqwest clients get a DNS-filtering resolver (see +//! [`crate::crawler::safety::SafeResolver`]) so an image/API host that +//! resolves to an internal IP is refused at connect time. Chromium, however, +//! does its **own** DNS and connection handling, so that resolver can't see +//! browser navigations. Without a second guard, a scraped chapter page that +//! `302`-redirects the browser to `http://127.0.0.1:5432/` (or an +//! attacker-owned hostname that resolves to `169.254.169.254`) would be +//! loaded and parsed as if it were catalog content. +//! +//! This module installs CDP `Fetch` interception on a page so every +//! **Document** (main-frame) request and redirect is re-validated through the +//! same [`ensure_public_target`] + resolved-IP check the reqwest paths use, +//! failing any that target an internal address. +//! +//! **Opt-in / default-off.** Enabling `Fetch` means every intercepted request +//! *must* be resolved by a live handler or the navigation hangs, so this is a +//! fragile hook in the crawler's critical path. It ships behind +//! `CRAWLER_SSRF_INTERCEPT` (default `false`) and the wiring has **not** been +//! exercised against a real Chromium in CI — validate with a manual crawl +//! before enabling in production. When disabled, [`open_page`] is byte-for-byte +//! the previous `browser.new_page(url)` behavior. Even when enabled, a failure +//! to install the guard falls back to an unguarded navigation rather than +//! failing the crawl. + +use std::net::IpAddr; +use std::sync::atomic::{AtomicBool, Ordering}; + +use chromiumoxide::browser::Browser; +use chromiumoxide::cdp::browser_protocol::fetch::{ + ContinueRequestParams, EnableParams, EventRequestPaused, FailRequestParams, RequestPattern, + RequestStage, +}; +use chromiumoxide::cdp::browser_protocol::network::{ErrorReason, ResourceType}; +use chromiumoxide::error::Result as CdpResult; +use chromiumoxide::Page; +use futures_util::StreamExt; +use reqwest::Url; + +use crate::crawler::safety::{ensure_public_target, is_private_ip}; + +/// Process-wide toggle, set once at startup from `CRAWLER_SSRF_INTERCEPT`. +/// A single boot-time flag (rather than threading a param through every +/// crawler navigation signature) keeps the off-path a no-op. +static ENABLED: AtomicBool = AtomicBool::new(false); + +pub fn set_enabled(on: bool) { + ENABLED.store(on, Ordering::Relaxed); +} + +pub fn is_enabled() -> bool { + ENABLED.load(Ordering::Relaxed) +} + +/// What to do with a navigation URL, decided without DNS where possible so the +/// security-critical branching is unit-testable. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + /// Safe to continue (non-network scheme, or a public IP literal). + Allow, + /// Refuse — bad scheme handled elsewhere, localhost, or a private IP literal. + Block, + /// A hostname that must be resolved to decide (DNS-rebinding check). + ResolveHost { host: String, port: u16 }, +} + +/// Pure decision over a URL string. `data:` / `blob:` / `about:` and other +/// non-http(s) schemes are allowed (not a network-SSRF vector); http(s) with a +/// private/loopback/localhost literal is blocked; an http(s) hostname needs +/// resolution. +pub(crate) fn verdict(url: &str) -> Verdict { + let Ok(parsed) = Url::parse(url) else { + // Unparseable — let Chromium reject it; not our call to make. + return Verdict::Allow; + }; + match parsed.scheme() { + "http" | "https" => {} + _ => return Verdict::Allow, + } + // Literal private IP / localhost / missing host → block (string check). + if ensure_public_target(url).is_err() { + return Verdict::Block; + } + match parsed.host_str() { + Some(host) if !is_ip_literal(host) => Verdict::ResolveHost { + host: host.to_string(), + port: parsed.port_or_known_default().unwrap_or(80), + }, + // Public IP literal (ensure_public_target already passed it). + _ => Verdict::Allow, + } +} + +/// Whether `host` (as reqwest's `host_str()` yields it) is an IP literal. +/// IPv6 literals arrive bracketed (`[::1]`), which don't parse as `IpAddr` +/// directly — strip the brackets first. +fn is_ip_literal(host: &str) -> bool { + let unbracketed = host + .strip_prefix('[') + .and_then(|s| s.strip_suffix(']')) + .unwrap_or(host); + unbracketed.parse::().is_ok() +} + +/// Async decision: resolves hostnames and blocks if any resolved address is +/// private (DNS rebinding). A resolution failure is *not* treated as blocked — +/// Chromium won't be able to connect either, so there's nothing to exfiltrate. +pub(crate) async fn is_blocked(url: &str) -> bool { + match verdict(url) { + Verdict::Allow => false, + Verdict::Block => true, + Verdict::ResolveHost { host, port } => match tokio::net::lookup_host((host.as_str(), port)) + .await + { + Ok(addrs) => addrs.map(|a| a.ip()).any(|ip| is_private_ip(&ip)), + Err(_) => false, + }, + } +} + +/// Open a page for navigation. With interception off, this is exactly +/// `browser.new_page(url)`. With it on, the page is created blank (no network), +/// the navigation guard is installed, and only then does it navigate — so the +/// initial request and any redirects pass through the guard. +pub async fn open_page(browser: &Browser, url: &str) -> CdpResult { + if !is_enabled() { + return browser.new_page(url).await; + } + let page = browser.new_page("about:blank").await?; + if let Err(e) = install_navigation_guard(&page).await { + // Fail open on install error: navigate unguarded rather than wedge the + // crawl. The reqwest-layer resolver still covers image downloads. + tracing::warn!(url = %url, error = %e, "SSRF navigation guard failed to install; navigating unguarded"); + } + page.goto(url).await?; + Ok(page) +} + +/// Enable `Fetch` for Document requests on `page` and spawn a task that +/// continues/fails each paused request per [`is_blocked`]. +async fn install_navigation_guard(page: &Page) -> CdpResult<()> { + // Intercept only main-frame Document requests at the request stage — a + // navigation plus its redirects, nothing else. Keeps the paused-request + // volume tiny so the handler can't become a page-load bottleneck. + let pattern = RequestPattern::builder() + .resource_type(ResourceType::Document) + .request_stage(RequestStage::Request) + .build(); + page.execute(EnableParams { + patterns: Some(vec![pattern]), + handle_auth_requests: None, + }) + .await?; + + let mut paused = page.event_listener::().await?; + let handler_page = page.clone(); + tokio::spawn(async move { + while let Some(ev) = paused.next().await { + let request_id = ev.request_id.clone(); + let outcome = if is_blocked(&ev.request.url).await { + tracing::warn!( + url = %ev.request.url, + "SSRF guard blocked browser navigation to an internal target" + ); + handler_page + .execute(FailRequestParams::new(request_id, ErrorReason::BlockedByClient)) + .await + .map(|_| ()) + } else { + handler_page + .execute(ContinueRequestParams::new(request_id)) + .await + .map(|_| ()) + }; + if let Err(e) = outcome { + // The page/session is gone (page closed) — the stream will end + // too; stop handling so the task exits rather than spins. + tracing::debug!(error = %e, "fetch interceptor: continue/fail failed, ending handler"); + break; + } + } + }); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verdict_allows_non_network_schemes() { + for url in ["about:blank", "data:text/html,hi", "blob:abc", "chrome://version"] { + assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed"); + } + } + + #[test] + fn verdict_blocks_private_ip_literals_and_localhost() { + for url in [ + "http://127.0.0.1/", + "http://10.0.0.1/", + "http://192.168.1.1:5432/", + "http://169.254.169.254/latest/meta-data/", + "http://[::1]/", + "http://[::ffff:127.0.0.1]/", + "http://localhost/", + "https://[fd00::1]/", + ] { + assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked"); + } + } + + #[test] + fn verdict_allows_public_ip_literals() { + for url in ["http://93.184.216.34/", "https://[2606:4700:4700::1111]/"] { + assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed"); + } + } + + #[test] + fn verdict_defers_hostnames_to_resolution() { + assert_eq!( + verdict("https://cdn.example.com/img.jpg"), + Verdict::ResolveHost { + host: "cdn.example.com".to_string(), + port: 443 + } + ); + assert_eq!( + verdict("http://catalog.test:8080/list"), + Verdict::ResolveHost { + host: "catalog.test".to_string(), + port: 8080 + } + ); + } + + #[tokio::test] + async fn is_blocked_true_for_private_literal_no_dns() { + assert!(is_blocked("http://169.254.169.254/").await); + assert!(!is_blocked("http://93.184.216.34/").await); + // Non-network scheme is always allowed through. + assert!(!is_blocked("about:blank").await); + } + + #[test] + fn enabled_toggle_roundtrips() { + // Global; restore afterwards so other tests see the default. + let prev = is_enabled(); + set_enabled(true); + assert!(is_enabled()); + set_enabled(false); + assert!(!is_enabled()); + set_enabled(prev); + } +} diff --git a/backend/src/crawler/mod.rs b/backend/src/crawler/mod.rs index 81167ba..fe20b65 100644 --- a/backend/src/crawler/mod.rs +++ b/backend/src/crawler/mod.rs @@ -19,6 +19,7 @@ pub mod content; pub mod daemon; pub mod detect; pub mod diff; +pub mod intercept; pub mod jobs; pub mod nav; pub mod pipeline; diff --git a/backend/src/crawler/session.rs b/backend/src/crawler/session.rs index b86465a..049da99 100644 --- a/backend/src/crawler/session.rs +++ b/backend/src/crawler/session.rs @@ -291,8 +291,7 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result< // the URL ever becomes attacker-influenced. crate::crawler::safety::ensure_public_target(probe_url) .with_context(|| format!("refuse to navigate unsafe probe URL {probe_url}"))?; - let page = browser - .new_page(probe_url) + let page = crate::crawler::intercept::open_page(browser, probe_url) .await .with_context(|| format!("open probe page {probe_url}"))?; // Close the tab on every exit path — a `?` on wait_for_nav / content() diff --git a/backend/src/crawler/source/target.rs b/backend/src/crawler/source/target.rs index 671cbdc..271ef5b 100644 --- a/backend/src/crawler/source/target.rs +++ b/backend/src/crawler/source/target.rs @@ -253,9 +253,7 @@ async fn navigate( ) -> Result { guard_navigate_url(url)?; ctx.rate.wait_for(url).await?; - let page = ctx - .browser - .new_page(url) + let page = crate::crawler::intercept::open_page(ctx.browser, url) .await .map_err(|e| PageError::Other(anyhow::Error::from(e)))?; // Close the tab on every exit path — the previous code closed on the two diff --git a/backend/src/settings.rs b/backend/src/settings.rs index a8ebe7f..0d9bace 100644 --- a/backend/src/settings.rs +++ b/backend/src/settings.rs @@ -269,6 +269,7 @@ impl CrawlerSettings { tor_control_cookie_path: base.tor_control_cookie_path.clone(), tor_recircuit_max_attempts: base.tor_recircuit_max_attempts, browser: base.browser.clone(), + ssrf_intercept: base.ssrf_intercept, }) } } diff --git a/backend/tests/crawler_browser_smoke.rs b/backend/tests/crawler_browser_smoke.rs index ac74f60..dace5cb 100644 --- a/backend/tests/crawler_browser_smoke.rs +++ b/backend/tests/crawler_browser_smoke.rs @@ -62,6 +62,43 @@ async fn headless_browser_can_navigate_and_read_title() { handle.close().await.expect("close cleanly"); } +/// Smoke-test the opt-in SSRF navigation guard (`CRAWLER_SSRF_INTERCEPT`). +/// With interception ON, a normal (allowed) navigation must still complete — +/// i.e. enabling CDP `Fetch` and installing the request handler must NOT wedge +/// page loads. This is the regression the interceptor's fragility risks; it +/// can only be exercised with a real Chromium, hence `#[ignore]`. +/// +/// (The private-target *blocking* path is covered by the pure-logic unit tests +/// in `crawler::intercept` — `verdict` / `is_blocked` — which don't need a +/// browser.) +#[tokio::test] +#[ignore = "downloads Chromium; run with --ignored"] +async fn ssrf_interception_does_not_wedge_allowed_navigation() { + use mangalord::crawler::intercept; + + const PAGE: &str = + "data:text/html,Guarded%20OK"; + + intercept::set_enabled(true); + let handle = browser::launch(LaunchOptions::headless()) + .await + .expect("launch headless chromium"); + + // Route through the guarded opener (blank page -> Fetch.enable -> handler + // -> goto). If the handler failed to continue the navigation, this would + // hang until the test harness times out. + let page = intercept::open_page(handle.browser(), PAGE) + .await + .expect("guarded open_page"); + page.wait_for_navigation().await.expect("wait for navigation"); + + let title = page.get_title().await.expect("get title"); + assert_eq!(title.as_deref(), Some("Guarded OK")); + + handle.close().await.expect("close cleanly"); + intercept::set_enabled(false); +} + /// Live end-to-end: navigate to a real page, get the rendered HTML, and /// parse it with `scraper`. ipify.org renders the visitor's public IP /// into the page DOM, so a successful run proves browser → render → diff --git a/docker-compose.yml b/docker-compose.yml index 29a275e..eed7c01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,6 +108,7 @@ services: CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600} CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10} CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3} + CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false} # Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps # the in-process scheduler off; the dashboard force-resync still works. CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true} diff --git a/frontend/package.json b/frontend/package.json index 3ff0392..8fd9062 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.124.10", + "version": "0.124.11", "private": true, "type": "module", "scripts": {