From 91d058c4262bbb1e0a454611c60317c7d92e14be Mon Sep 17 00:00:00 2001 From: fabi Date: Sun, 14 Jun 2026 21:41:00 +0200 Subject: [PATCH] fix(crawler): send a realistic browser User-Agent (Cloudflare evasion) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawler's headless Chromium advertised the default HeadlessChrome UA, which Cloudflare's bot detection challenges on sight — over Tor that meant an unsolvable 'Just a moment' page on every fetch, so session verification kept classifying pages as transient and the metadata pass failed at 'acquire browser lease'. (Confirmed empirically: same Tor exit + a normal Chrome UA returns the real page with the #logo sentinel; the HeadlessChrome UA returns the challenge.) The existing user_agent setting only fed reqwest, never the browser. Override the launcher's UA with a realistic non-headless default (CRAWLER_USER_AGENT to customize); the UA may contain spaces, unlike CRAWLER_BROWSER_ARGS. Co-Authored-By: Claude Opus 4.8 --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- backend/src/crawler/browser.rs | 63 +++++++++++++++++++++++++++++++--- frontend/package.json | 2 +- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 957dbcf..49eae86 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.81.1" +version = "0.81.2" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index dcf03bd..cae91cd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.81.1" +version = "0.81.2" edition = "2021" default-run = "mangalord" diff --git a/backend/src/crawler/browser.rs b/backend/src/crawler/browser.rs index c9981ea..127e799 100644 --- a/backend/src/crawler/browser.rs +++ b/backend/src/crawler/browser.rs @@ -54,13 +54,26 @@ pub struct LaunchOptions { /// defaults. Example: `vec!["--lang=de-DE".into(), /// "--window-size=1280,800".into()]`. pub extra_args: Vec, + /// User-Agent for the browser. `None` uses [`DEFAULT_USER_AGENT`]. + /// Critically, this is NEVER Chromium's built-in headless UA (which + /// contains the `HeadlessChrome` token that Cloudflare's bot detection + /// flags — over Tor that means an unsolvable "Just a moment" challenge + /// on every page). Sourced from `CRAWLER_USER_AGENT`. + pub user_agent: Option, } +/// A realistic, non-headless Chrome UA matching the bundled engine major. +/// Replaces Chromium's default `HeadlessChrome/` UA, which anti-bot +/// services (Cloudflare) challenge on sight. +pub const DEFAULT_USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) \ +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"; + impl LaunchOptions { pub fn headed() -> Self { Self { mode: BrowserMode::Headed, extra_args: Vec::new(), + user_agent: None, } } @@ -68,13 +81,17 @@ impl LaunchOptions { Self { mode: BrowserMode::Headless, extra_args: Vec::new(), + user_agent: None, } } /// Reads `CRAWLER_BROWSER_MODE` (`headless`|`headed`, default - /// `headless`) and `CRAWLER_BROWSER_ARGS` (whitespace-separated - /// Chromium flags). Flags containing whitespace aren't supported - /// through the env var — use the programmatic API for those. + /// `headless`), `CRAWLER_BROWSER_ARGS` (whitespace-separated + /// Chromium flags), and `CRAWLER_USER_AGENT` (the browser UA; + /// blank/unset → [`DEFAULT_USER_AGENT`]). Flags containing whitespace + /// aren't supported through `CRAWLER_BROWSER_ARGS` — use the + /// programmatic API (or `CRAWLER_USER_AGENT` for the UA, which may + /// contain spaces) for those. pub fn from_env() -> Self { let mode = match std::env::var("CRAWLER_BROWSER_MODE").as_deref() { Ok("headed") => BrowserMode::Headed, @@ -83,7 +100,16 @@ impl LaunchOptions { let extra_args = std::env::var("CRAWLER_BROWSER_ARGS") .map(|s| parse_args(&s)) .unwrap_or_default(); - Self { mode, extra_args } + let user_agent = std::env::var("CRAWLER_USER_AGENT") + .ok() + .filter(|s| !s.trim().is_empty()); + Self { mode, extra_args, user_agent } + } + + /// The effective browser UA: the configured override, else the + /// non-headless [`DEFAULT_USER_AGENT`]. + pub fn effective_user_agent(&self) -> &str { + self.user_agent.as_deref().unwrap_or(DEFAULT_USER_AGENT) } } @@ -211,7 +237,11 @@ pub async fn launch(options: LaunchOptions) -> anyhow::Result { // Chromium's sandbox wants. Disable it; the crawler runs in its // own container anyway. .arg("--no-sandbox") - .arg("--disable-dev-shm-usage"); + .arg("--disable-dev-shm-usage") + // Override Chromium's default headless UA (contains `HeadlessChrome`, + // which Cloudflare challenges) with a realistic one. Placed before + // extra_args so an explicit override there still wins. + .arg(format!("--user-agent={}", options.effective_user_agent())); for arg in &options.extra_args { builder = builder.arg(arg); } @@ -345,6 +375,29 @@ mod tests { assert_eq!(LaunchOptions::headed().mode, BrowserMode::Headed); } + #[test] + fn default_user_agent_is_not_headless() { + // The whole point: never advertise `HeadlessChrome`, which Cloudflare + // challenges (especially over Tor) — that was the crawler's silent + // failure mode. A bare LaunchOptions must still yield a realistic UA. + assert!( + !DEFAULT_USER_AGENT.contains("Headless"), + "default UA must not contain the Headless token" + ); + assert_eq!(LaunchOptions::headless().effective_user_agent(), DEFAULT_USER_AGENT); + assert!(!LaunchOptions::default().effective_user_agent().contains("Headless")); + } + + #[test] + fn configured_user_agent_overrides_default() { + let opts = LaunchOptions { + mode: BrowserMode::Headless, + extra_args: Vec::new(), + user_agent: Some("Custom/1.0".to_string()), + }; + assert_eq!(opts.effective_user_agent(), "Custom/1.0"); + } + // Regression: if another Arc outlives `Handle::close`, the // old code awaited the driver task forever because the chromiumoxide // handler stream doesn't return None on its own. Aborting the driver diff --git a/frontend/package.json b/frontend/package.json index 8960c35..e22b036 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.81.1", + "version": "0.81.2", "private": true, "type": "module", "scripts": {