diff --git a/.env.example b/.env.example index a2a5579..3747b61 100644 --- a/.env.example +++ b/.env.example @@ -191,12 +191,19 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10 # exhausted) that trigger an automatic coordinated browser restart. # Default 3. CRAWLER_BROWSER_RESTART_THRESHOLD=3 -# Opt-in CDP Fetch interception that re-validates every headless-browser -# navigation/redirect against the SSRF check (blocks a scraped page that -# redirects the browser to an internal target). Default false — enabling -# Fetch is a fragile hook in the crawler's critical path and is not -# CI-verified; validate with a manual crawl before turning on. -CRAWLER_SSRF_INTERCEPT=false +# CDP Fetch interception that re-validates every headless-browser +# navigation/redirect/subresource against the SSRF check (blocks a scraped page +# that drives the browser — via a redirect OR page JS/subresource — to an +# internal target such as the cloud metadata service or postgres). Default TRUE: +# with it off, only the top-level URL string is checked and page JS/subresources +# reach internal IPs (the reqwest SafeResolver does not cover Chromium's own +# stack). The CDP Fetch hook can't be exercised in CI (no Chromium); before +# relying on a fresh deploy, validate it doesn't wedge navigation: +# CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium \ +# cargo test -p mangalord --test crawler_browser_smoke -- --ignored \ +# ssrf_interception_does_not_wedge_allowed_navigation +# Set to false only as a break-glass if the hook destabilizes a deployment. +CRAWLER_SSRF_INTERCEPT=true # Path to a system Chromium binary. When set, the crawler skips the # bundled-fetcher download. Required on platforms without a usable # upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ce12a07..5bf0041 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.128.21" +version = "0.128.22" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c12a69b..15e7a59 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.128.21" +version = "0.128.22" edition = "2021" default-run = "mangalord" diff --git a/backend/src/config.rs b/backend/src/config.rs index 25e9915..d9bcd9a 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -520,12 +520,15 @@ pub struct CrawlerConfig { /// exhausted) that trigger an automatic coordinated browser restart. /// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`. pub browser_restart_threshold: u32, - /// Opt-in CDP `Fetch` interception that re-validates every headless-browser - /// navigation/redirect against the SSRF check (blocks a scraped page that - /// redirects the browser to an internal target). Default `false`: enabling - /// `Fetch` is a fragile hook in the crawler's critical path and the wiring - /// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl - /// before turning on. `CRAWLER_SSRF_INTERCEPT`. + /// CDP `Fetch` interception that re-validates every headless-browser + /// navigation/redirect/subresource against the SSRF check. Default `true`: + /// with it off, only the top-level URL string is validated, so a scraped + /// page's JS/subresources (which use Chromium's own network stack, not the + /// reqwest `SafeResolver`) can reach internal targets like the cloud + /// metadata service or postgres. The Fetch hook can't be exercised in CI (no + /// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation` + /// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`; + /// set `false` only as a break-glass if the hook destabilizes a deployment. pub ssrf_intercept: bool, } @@ -559,7 +562,7 @@ impl Default for CrawlerConfig { job_timeout: Duration::from_secs(600), metadata_max_consecutive_failures: 10, browser_restart_threshold: 3, - ssrf_intercept: false, + ssrf_intercept: true, } } } @@ -725,7 +728,7 @@ impl CrawlerConfig { ) as u32, browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1) as u32, - ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false), + ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true), }) } } diff --git a/docker-compose.yml b/docker-compose.yml index fb9291e..4f08c3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,7 +112,7 @@ services: CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600} CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10} CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3} - CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false} + CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-true} # Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps # the in-process scheduler off; the dashboard force-resync still works. CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true} diff --git a/frontend/package.json b/frontend/package.json index 17fc6fa..1b2458d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.128.21", + "version": "0.128.22", "private": true, "type": "module", "scripts": {