fix: enable browser SSRF interception by default (H1)

Off-by-default left headless-browser page JS/subresources/redirects able to reach
internal IPs (Chromium bypasses the reqwest SafeResolver). The crawler runs in the
backend container so it can't be network-isolated without breaking Postgres — the
CDP Fetch interceptor is the control. Flip CRAWLER_SSRF_INTERCEPT default to true
(config + .env.example + compose); off-path unchanged as break-glass. The
#[ignore]'d smoke test validates against real Chromium (command documented).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -191,12 +191,19 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
 # exhausted) that trigger an automatic coordinated browser restart.
 # Default 3.
 CRAWLER_BROWSER_RESTART_THRESHOLD=3
-# Opt-in CDP Fetch interception that re-validates every headless-browser
+# CDP Fetch interception that re-validates every headless-browser
-# navigation/redirect against the SSRF check (blocks a scraped page that
+# navigation/redirect/subresource against the SSRF check (blocks a scraped page
-# redirects the browser to an internal target). Default false — enabling
+# that drives the browser — via a redirect OR page JS/subresource — to an
-# Fetch is a fragile hook in the crawler's critical path and is not
+# internal target such as the cloud metadata service or postgres). Default TRUE:
-# CI-verified; validate with a manual crawl before turning on.
+# with it off, only the top-level URL string is checked and page JS/subresources
-CRAWLER_SSRF_INTERCEPT=false
+# reach internal IPs (the reqwest SafeResolver does not cover Chromium's own
+# stack). The CDP Fetch hook can't be exercised in CI (no Chromium); before
+# relying on a fresh deploy, validate it doesn't wedge navigation:
+#   CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium \
+#     cargo test -p mangalord --test crawler_browser_smoke -- --ignored \
+#     ssrf_interception_does_not_wedge_allowed_navigation
+# Set to false only as a break-glass if the hook destabilizes a deployment.
+CRAWLER_SSRF_INTERCEPT=true
 # Path to a system Chromium binary. When set, the crawler skips the
 # bundled-fetcher download. Required on platforms without a usable
 # upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On

backend/Cargo.lock

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.128.21"
+version = "0.128.22"
 dependencies = [
  "anyhow",
  "argon2",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.128.21"
+version = "0.128.22"
 edition = "2021"
 default-run = "mangalord"
 

backend/src/config.rs

@@ -520,12 +520,15 @@ pub struct CrawlerConfig {
     /// exhausted) that trigger an automatic coordinated browser restart.
     /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
     pub browser_restart_threshold: u32,
-    /// Opt-in CDP `Fetch` interception that re-validates every headless-browser
+    /// CDP `Fetch` interception that re-validates every headless-browser
-    /// navigation/redirect against the SSRF check (blocks a scraped page that
+    /// navigation/redirect/subresource against the SSRF check. Default `true`:
-    /// redirects the browser to an internal target). Default `false`: enabling
+    /// with it off, only the top-level URL string is validated, so a scraped
-    /// `Fetch` is a fragile hook in the crawler's critical path and the wiring
+    /// page's JS/subresources (which use Chromium's own network stack, not the
-    /// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl
+    /// reqwest `SafeResolver`) can reach internal targets like the cloud
-    /// before turning on. `CRAWLER_SSRF_INTERCEPT`.
+    /// metadata service or postgres. The Fetch hook can't be exercised in CI (no
+    /// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation`
+    /// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`;
+    /// set `false` only as a break-glass if the hook destabilizes a deployment.
     pub ssrf_intercept: bool,
 }
 
@@ -559,7 +562,7 @@ impl Default for CrawlerConfig {
             job_timeout: Duration::from_secs(600),
             metadata_max_consecutive_failures: 10,
             browser_restart_threshold: 3,
-            ssrf_intercept: false,
+            ssrf_intercept: true,
         }
     }
 }
@@ -725,7 +728,7 @@ impl CrawlerConfig {
             ) as u32,
             browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
                 as u32,
-            ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false),
+            ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true),
         })
     }
 }

docker-compose.yml

@@ -112,7 +112,7 @@ services:
       CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600}
       CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10}
       CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3}
-      CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false}
+      CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-true}
       # Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps
       # the in-process scheduler off; the dashboard force-resync still works.
       CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true}

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.128.21",
+  "version": "0.128.22",
   "private": true,
   "type": "module",
   "scripts": {