feat(crawler): CRAWLER_ALLOW_ANY_HOST bypasses the host allowlist (0.44.0)

Operators whose sources shard images across numbered CDN subdomains
can't pre-enumerate every host in CRAWLER_DOWNLOAD_ALLOWLIST. The new
flag short-circuits the host check in DownloadAllowlist::contains
while leaving scheme, localhost, and private-IP defenses in
is_safe_url untouched — scraped URLs pointing at 10.x /
169.254.169.254 / file:// stay refused. Default is false; fail-closed
posture is preserved unless the operator opts in. Wired into both the
server (config::build_download_allowlist) and the bin/crawler.rs
one-shot.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/src/bin/crawler.rs

@@ -226,24 +226,30 @@ async fn run(
     // SSRF defence: only download from the catalog host + CDN host
     // (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
     // single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
-    let mut allowlist =
-        mangalord::crawler::safety::DownloadAllowlist::new();
-    if let Ok(parsed) = reqwest::Url::parse(start_url) {
-        if let Some(h) = parsed.host_str() {
-            allowlist = allowlist.allow(h);
-        }
-    }
-    if let Some(host) = cdn_host {
-        allowlist = allowlist.allow(host);
-    }
-    if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
-        for piece in extras.split(',') {
-            let trimmed = piece.trim();
-            if !trimmed.is_empty() {
-                allowlist = allowlist.allow(trimmed);
+    // CRAWLER_ALLOW_ANY_HOST=true short-circuits the host check for
+    // sharded-CDN sources; private-IP and scheme guards still apply.
+    let allowlist = if env_bool("CRAWLER_ALLOW_ANY_HOST", false) {
+        mangalord::crawler::safety::DownloadAllowlist::allow_any()
+    } else {
+        let mut allow = mangalord::crawler::safety::DownloadAllowlist::new();
+        if let Ok(parsed) = reqwest::Url::parse(start_url) {
+            if let Some(h) = parsed.host_str() {
+                allow = allow.allow(h);
             }
         }
-    }
+        if let Some(host) = cdn_host {
+            allow = allow.allow(host);
+        }
+        if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
+            for piece in extras.split(',') {
+                let trimmed = piece.trim();
+                if !trimmed.is_empty() {
+                    allow = allow.allow(trimmed);
+                }
+            }
+        }
+        allow
+    };
     let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
         .ok()
         .and_then(|s| s.parse().ok())