fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): re-validate redirect hops to close SSRF via 3xx

Browse Source 
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-01 07:21:15 +02:00
parent cbbc626768
commit 2ed42f7b9e
6 changed files with 181 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
backend/src/bin/crawler.rs
View File

@@ -112,9 +112,20 @@ async fn main() -> anyhow::Result<()> {
cookie_jar.add_cookie_str(&cookie_str, &seed_url);
tracing::info!(domain, "seeded PHPSESSID into reqwest cookie jar");
}
// SSRF defence: only download from the catalog host + CDN host (plus
// optional CRAWLER_DOWNLOAD_ALLOWLIST extras). Built here so the same
// allowlist guards both the redirect policy and the per-image check.
let allowlist = Arc::new(build_download_allowlist(&start_url, cdn_host.as_deref()));
let mut http_builder = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.no_proxy()
// Re-validate every redirect hop against the allowlist — reqwest's
// default follows up to 10 redirects and `is_safe_url` only guards
// the initial URL, so a 302 to a private IP would otherwise pivot
// inside the deployment (SSRF).
.redirect(mangalord::crawler::safety::safe_redirect_policy(
(*allowlist).clone(),
))
.cookie_provider(cookie_jar);
if let Some(ua) = &user_agent {
http_builder = http_builder.user_agent(ua);
@@ -216,6 +227,7 @@ async fn main() -> anyhow::Result<()> {
rate_ms,
cdn_host.as_deref(),
cdn_rate_ms,
Arc::clone(&allowlist),
limit,
skip_chapters,
skip_chapter_content || !session_ready,
@@ -246,6 +258,7 @@ async fn run(
rate_ms: u64,
cdn_host: Option<&str>,
cdn_rate_ms: u64,
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
limit: usize,
skip_chapters: bool,
skip_chapter_content: bool,
@@ -259,38 +272,12 @@ async fn run(
}
let rate = Arc::new(rate);
// SSRF defence: only download from the catalog host + CDN host
// (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
// single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
// CRAWLER_ALLOW_ANY_HOST=true short-circuits the host check for
// sharded-CDN sources; private-IP and scheme guards still apply.
let allowlist = if env_bool("CRAWLER_ALLOW_ANY_HOST", false) {
mangalord::crawler::safety::DownloadAllowlist::allow_any()
} else {
let mut allow = mangalord::crawler::safety::DownloadAllowlist::new();
if let Ok(parsed) = reqwest::Url::parse(start_url) {
if let Some(h) = parsed.host_str() {
allow = allow.allow(h);
}
}
if let Some(host) = cdn_host {
allow = allow.allow(host);
}
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
for piece in extras.split(',') {
let trimmed = piece.trim();
if !trimmed.is_empty() {
allow = allow.allow(trimmed);
}
}
}
allow
};
// Per-image download cap (the allowlist is built in `main` and passed in
// so the HTTP client's redirect policy and this check share one source).
let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
let allowlist = Arc::new(allowlist);
let stats = pipeline::run_metadata_pass(
manager.as_ref(),
@@ -497,3 +484,37 @@ fn env_bool(name: &str, default: bool) -> bool {
}
}
/// Build the crawler download allowlist from env + the catalog/CDN hosts.
/// Shared by the HTTP client's redirect policy and the per-image safety
/// check so both agree on which hosts are reachable.
///
/// `CRAWLER_ALLOW_ANY_HOST=true` short-circuits the host check for
/// sharded-CDN sources; private-IP and scheme guards still apply.
fn build_download_allowlist(
start_url: &str,
cdn_host: Option<&str>,
) -> mangalord::crawler::safety::DownloadAllowlist {
use mangalord::crawler::safety::DownloadAllowlist;
if env_bool("CRAWLER_ALLOW_ANY_HOST", false) {
return DownloadAllowlist::allow_any();
}
let mut allow = DownloadAllowlist::new();
if let Ok(parsed) = reqwest::Url::parse(start_url) {
if let Some(h) = parsed.host_str() {
allow = allow.allow(h);
}
}
if let Some(host) = cdn_host {
allow = allow.allow(host);
}
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
for piece in extras.split(',') {
let trimmed = piece.trim();
if !trimmed.is_empty() {
allow = allow.allow(trimmed);
}
}
}
allow
}