fix(crawler): send a realistic browser User-Agent (Cloudflare evasion)

The crawler's headless Chromium advertised the default HeadlessChrome UA,
which Cloudflare's bot detection challenges on sight — over Tor that meant an
unsolvable 'Just a moment' page on every fetch, so session verification kept
classifying pages as transient and the metadata pass failed at
'acquire browser lease'. (Confirmed empirically: same Tor exit + a normal
Chrome UA returns the real page with the #logo sentinel; the HeadlessChrome
UA returns the challenge.) The existing user_agent setting only fed reqwest,
never the browser. Override the launcher's UA with a realistic non-headless
default (CRAWLER_USER_AGENT to customize); the UA may contain spaces, unlike
CRAWLER_BROWSER_ARGS.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/Cargo.lock

@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "mangalord"
-version = "0.81.1"
+version = "0.81.2"
 dependencies = [
  "anyhow",
  "argon2",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mangalord"
-version = "0.81.1"
+version = "0.81.2"
 edition = "2021"
 default-run = "mangalord"
 

backend/src/crawler/browser.rs

@@ -54,13 +54,26 @@ pub struct LaunchOptions {
     /// defaults. Example: `vec!["--lang=de-DE".into(),
     /// "--window-size=1280,800".into()]`.
     pub extra_args: Vec<String>,
+    /// User-Agent for the browser. `None` uses [`DEFAULT_USER_AGENT`].
+    /// Critically, this is NEVER Chromium's built-in headless UA (which
+    /// contains the `HeadlessChrome` token that Cloudflare's bot detection
+    /// flags — over Tor that means an unsolvable "Just a moment" challenge
+    /// on every page). Sourced from `CRAWLER_USER_AGENT`.
+    pub user_agent: Option<String>,
 }
 
+/// A realistic, non-headless Chrome UA matching the bundled engine major.
+/// Replaces Chromium's default `HeadlessChrome/<v>` UA, which anti-bot
+/// services (Cloudflare) challenge on sight.
+pub const DEFAULT_USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) \
+AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
+
 impl LaunchOptions {
     pub fn headed() -> Self {
         Self {
             mode: BrowserMode::Headed,
             extra_args: Vec::new(),
+            user_agent: None,
         }
     }
 
@@ -68,13 +81,17 @@ impl LaunchOptions {
         Self {
             mode: BrowserMode::Headless,
             extra_args: Vec::new(),
+            user_agent: None,
         }
     }
 
     /// Reads `CRAWLER_BROWSER_MODE` (`headless`|`headed`, default
-    /// `headless`) and `CRAWLER_BROWSER_ARGS` (whitespace-separated
-    /// Chromium flags). Flags containing whitespace aren't supported
-    /// through the env var — use the programmatic API for those.
+    /// `headless`), `CRAWLER_BROWSER_ARGS` (whitespace-separated
+    /// Chromium flags), and `CRAWLER_USER_AGENT` (the browser UA;
+    /// blank/unset → [`DEFAULT_USER_AGENT`]). Flags containing whitespace
+    /// aren't supported through `CRAWLER_BROWSER_ARGS` — use the
+    /// programmatic API (or `CRAWLER_USER_AGENT` for the UA, which may
+    /// contain spaces) for those.
     pub fn from_env() -> Self {
         let mode = match std::env::var("CRAWLER_BROWSER_MODE").as_deref() {
             Ok("headed") => BrowserMode::Headed,
@@ -83,7 +100,16 @@ impl LaunchOptions {
         let extra_args = std::env::var("CRAWLER_BROWSER_ARGS")
             .map(|s| parse_args(&s))
             .unwrap_or_default();
-        Self { mode, extra_args }
+        let user_agent = std::env::var("CRAWLER_USER_AGENT")
+            .ok()
+            .filter(|s| !s.trim().is_empty());
+        Self { mode, extra_args, user_agent }
+    }
+
+    /// The effective browser UA: the configured override, else the
+    /// non-headless [`DEFAULT_USER_AGENT`].
+    pub fn effective_user_agent(&self) -> &str {
+        self.user_agent.as_deref().unwrap_or(DEFAULT_USER_AGENT)
     }
 }
 
@@ -211,7 +237,11 @@ pub async fn launch(options: LaunchOptions) -> anyhow::Result<Handle> {
         // Chromium's sandbox wants. Disable it; the crawler runs in its
         // own container anyway.
         .arg("--no-sandbox")
-        .arg("--disable-dev-shm-usage");
+        .arg("--disable-dev-shm-usage")
+        // Override Chromium's default headless UA (contains `HeadlessChrome`,
+        // which Cloudflare challenges) with a realistic one. Placed before
+        // extra_args so an explicit override there still wins.
+        .arg(format!("--user-agent={}", options.effective_user_agent()));
     for arg in &options.extra_args {
         builder = builder.arg(arg);
     }
@@ -345,6 +375,29 @@ mod tests {
         assert_eq!(LaunchOptions::headed().mode, BrowserMode::Headed);
     }
 
+    #[test]
+    fn default_user_agent_is_not_headless() {
+        // The whole point: never advertise `HeadlessChrome`, which Cloudflare
+        // challenges (especially over Tor) — that was the crawler's silent
+        // failure mode. A bare LaunchOptions must still yield a realistic UA.
+        assert!(
+            !DEFAULT_USER_AGENT.contains("Headless"),
+            "default UA must not contain the Headless token"
+        );
+        assert_eq!(LaunchOptions::headless().effective_user_agent(), DEFAULT_USER_AGENT);
+        assert!(!LaunchOptions::default().effective_user_agent().contains("Headless"));
+    }
+
+    #[test]
+    fn configured_user_agent_overrides_default() {
+        let opts = LaunchOptions {
+            mode: BrowserMode::Headless,
+            extra_args: Vec::new(),
+            user_agent: Some("Custom/1.0".to_string()),
+        };
+        assert_eq!(opts.effective_user_agent(), "Custom/1.0");
+    }
+
     // Regression: if another Arc<Browser> outlives `Handle::close`, the
     // old code awaited the driver task forever because the chromiumoxide
     // handler stream doesn't return None on its own. Aborting the driver

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mangalord-frontend",
-  "version": "0.81.1",
+  "version": "0.81.2",
   "private": true,
   "type": "module",
   "scripts": {