feat(crawler): re-validate redirect hops to close SSRF via 3xx

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-01 07:21:15 +02:00
parent cbbc626768
commit 2ed42f7b9e
6 changed files with 181 additions and 31 deletions

View File

@@ -469,6 +469,12 @@ async fn spawn_analysis_daemon(
// both. Mirrors the crawler client's `.no_proxy()` (see
// `spawn_crawler_daemon`).
.no_proxy()
// Re-validate redirect hops so a hostile/compromised vision
// endpoint can't 302 the bearer token + page bytes into the
// deployment's internal network. No allowlist here (the
// endpoint is a single admin-configured URL), so the policy
// enforces scheme + private-IP only.
.redirect(crate::crawler::safety::public_redirect_policy())
.build()
.context("build analysis http client")?;
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
@@ -494,6 +500,7 @@ async fn spawn_analysis_daemon(
// still gets a useful side-channel on backend
// uptime + vision health.
.no_proxy()
.redirect(crate::crawler::safety::public_redirect_policy())
.build()
.context("build vision readiness http client")?;
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
@@ -571,6 +578,13 @@ async fn spawn_crawler_daemon(
let mut http_builder = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.no_proxy()
// Re-validate every redirect hop against the download allowlist:
// reqwest's default policy follows up to 10 redirects, and
// `is_safe_url` only guards the initial URL, so an allowlisted CDN
// 302ing to a private IP would otherwise be followed (SSRF).
.redirect(crate::crawler::safety::safe_redirect_policy(
cfg.download_allowlist.clone(),
))
.cookie_provider(Arc::clone(&cookie_jar));
if let Some(ua) = &cfg.user_agent {
http_builder = http_builder.user_agent(ua);