bf425cf8e6
The reqwest DNS resolver can't see Chromium navigations, so a scraped page that redirects the browser to an internal target (302 -> 127.0.0.1:5432, or a rebinding hostname) was still loaded. Add crawler::intercept: an opt-in CDP Fetch guard that re-validates every Document navigation/redirect through the same ensure_public_target + resolved-IP check, failing internal targets. Gated behind CRAWLER_SSRF_INTERCEPT (default OFF): enabling Fetch is a fragile critical-path hook and the CDP wiring is not CI-verifiable (no Chromium in CI). When off, open_page is byte-identical to browser.new_page; when on, a guard-install failure falls back to an unguarded navigation rather than wedging the crawl. The pure decision logic (verdict/is_blocked) is unit-tested; an #[ignore] smoke test covers the CDP path. Validate with a manual crawl before enabling in production. Bump to 0.124.11. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
201 lines
7.2 KiB
Rust
201 lines
7.2 KiB
Rust
//! Smoke test for the Chromium launcher.
|
|
//!
|
|
//! Marked `#[ignore]` because it (a) downloads ~150 MB of Chromium on
|
|
//! first run via the `fetcher` feature and (b) requires a real `$DISPLAY`
|
|
//! for the headed path. Run it explicitly:
|
|
//!
|
|
//! ```sh
|
|
//! cargo test --test crawler_browser_smoke -- --ignored --nocapture
|
|
//! ```
|
|
//!
|
|
//! Override the cache location with `CRAWLER_CHROMIUM_DIR=/some/path` if
|
|
//! `$HOME/.cache/mangalord/chromium` isn't writable.
|
|
//!
|
|
//! Set `CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium-headless-shell` (or
|
|
//! another system chromium path) to exercise the system-chromium
|
|
//! launch path instead of the fetcher download — this is the path the
|
|
//! Raspberry Pi deployment takes.
|
|
|
|
use mangalord::crawler::browser::{self, LaunchOptions};
|
|
|
|
#[tokio::test]
|
|
#[ignore = "downloads Chromium and needs a display; run with --ignored"]
|
|
async fn headed_browser_can_navigate_and_read_title() {
|
|
// A data URL avoids any network dependency — we're testing the
|
|
// browser launcher, not connectivity.
|
|
const PAGE: &str = "data:text/html,<html><head><title>Mangalord%20Smoke</title></head><body>OK</body></html>";
|
|
|
|
let handle = browser::launch(LaunchOptions::headed())
|
|
.await
|
|
.expect("launch headed chromium");
|
|
|
|
let page = handle
|
|
.browser()
|
|
.new_page(PAGE)
|
|
.await
|
|
.expect("open new page");
|
|
page.wait_for_navigation()
|
|
.await
|
|
.expect("wait for navigation");
|
|
|
|
let title = page.get_title().await.expect("get title");
|
|
assert_eq!(title.as_deref(), Some("Mangalord Smoke"));
|
|
|
|
handle.close().await.expect("close cleanly");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "downloads Chromium; run with --ignored"]
|
|
async fn headless_browser_can_navigate_and_read_title() {
|
|
const PAGE: &str = "data:text/html,<html><head><title>Headless%20OK</title></head><body></body></html>";
|
|
|
|
let handle = browser::launch(LaunchOptions::headless())
|
|
.await
|
|
.expect("launch headless chromium");
|
|
|
|
let page = handle.browser().new_page(PAGE).await.expect("open new page");
|
|
page.wait_for_navigation().await.expect("wait for navigation");
|
|
|
|
let title = page.get_title().await.expect("get title");
|
|
assert_eq!(title.as_deref(), Some("Headless OK"));
|
|
|
|
handle.close().await.expect("close cleanly");
|
|
}
|
|
|
|
/// Smoke-test the opt-in SSRF navigation guard (`CRAWLER_SSRF_INTERCEPT`).
|
|
/// With interception ON, a normal (allowed) navigation must still complete —
|
|
/// i.e. enabling CDP `Fetch` and installing the request handler must NOT wedge
|
|
/// page loads. This is the regression the interceptor's fragility risks; it
|
|
/// can only be exercised with a real Chromium, hence `#[ignore]`.
|
|
///
|
|
/// (The private-target *blocking* path is covered by the pure-logic unit tests
|
|
/// in `crawler::intercept` — `verdict` / `is_blocked` — which don't need a
|
|
/// browser.)
|
|
#[tokio::test]
|
|
#[ignore = "downloads Chromium; run with --ignored"]
|
|
async fn ssrf_interception_does_not_wedge_allowed_navigation() {
|
|
use mangalord::crawler::intercept;
|
|
|
|
const PAGE: &str =
|
|
"data:text/html,<html><head><title>Guarded%20OK</title></head><body></body></html>";
|
|
|
|
intercept::set_enabled(true);
|
|
let handle = browser::launch(LaunchOptions::headless())
|
|
.await
|
|
.expect("launch headless chromium");
|
|
|
|
// Route through the guarded opener (blank page -> Fetch.enable -> handler
|
|
// -> goto). If the handler failed to continue the navigation, this would
|
|
// hang until the test harness times out.
|
|
let page = intercept::open_page(handle.browser(), PAGE)
|
|
.await
|
|
.expect("guarded open_page");
|
|
page.wait_for_navigation().await.expect("wait for navigation");
|
|
|
|
let title = page.get_title().await.expect("get title");
|
|
assert_eq!(title.as_deref(), Some("Guarded OK"));
|
|
|
|
handle.close().await.expect("close cleanly");
|
|
intercept::set_enabled(false);
|
|
}
|
|
|
|
/// Live end-to-end: navigate to a real page, get the rendered HTML, and
|
|
/// parse it with `scraper`. ipify.org renders the visitor's public IP
|
|
/// into the page DOM, so a successful run proves browser → render →
|
|
/// `Html::parse_document` → selector → text extraction all work
|
|
/// against a real site. This is the same path each future `Source`
|
|
/// impl will take.
|
|
#[tokio::test]
|
|
#[ignore = "needs network; run with --ignored"]
|
|
async fn fetches_public_ip_from_ipify() {
|
|
use std::time::Duration;
|
|
|
|
let handle = browser::launch(LaunchOptions::headless())
|
|
.await
|
|
.expect("launch headless chromium");
|
|
|
|
let page = handle
|
|
.browser()
|
|
.new_page("https://www.ipify.org")
|
|
.await
|
|
.expect("open ipify");
|
|
page.wait_for_navigation().await.expect("wait for navigation");
|
|
// ipify injects the IP via JS after load, so the navigation event
|
|
// alone isn't enough — give the script a beat to run.
|
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
|
|
let html = page.content().await.expect("get rendered html");
|
|
let doc = scraper::Html::parse_document(&html);
|
|
let body_sel = scraper::Selector::parse("body").unwrap();
|
|
let body_text: String = doc
|
|
.select(&body_sel)
|
|
.next()
|
|
.map(|n| n.text().collect::<Vec<_>>().join(" "))
|
|
.unwrap_or_default();
|
|
|
|
let ip = extract_ipv4(&body_text)
|
|
.unwrap_or_else(|| panic!("no IPv4 found in ipify body: {body_text}"));
|
|
eprintln!("ipify says our public IP is: {ip}");
|
|
|
|
handle.close().await.expect("close cleanly");
|
|
}
|
|
|
|
/// Proves that `LaunchOptions::extra_args` actually reach Chromium and
|
|
/// influence its runtime. `--user-agent=...` overrides `navigator.userAgent`,
|
|
/// observable from JS — read it back via `page.evaluate`.
|
|
#[tokio::test]
|
|
#[ignore = "downloads Chromium; run with --ignored"]
|
|
async fn extra_args_reach_chromium() {
|
|
const UA: &str = "MangalordCrawlerTest/1.0";
|
|
let options = LaunchOptions {
|
|
mode: browser::BrowserMode::Headless,
|
|
extra_args: vec![format!("--user-agent={UA}")],
|
|
user_agent: None,
|
|
};
|
|
let handle = browser::launch(options).await.expect("launch with extra args");
|
|
|
|
let page = handle
|
|
.browser()
|
|
.new_page("about:blank")
|
|
.await
|
|
.expect("open page");
|
|
page.wait_for_navigation().await.expect("wait");
|
|
|
|
let ua: String = page
|
|
.evaluate("navigator.userAgent")
|
|
.await
|
|
.expect("evaluate navigator.userAgent")
|
|
.into_value()
|
|
.expect("string value");
|
|
assert_eq!(
|
|
ua, UA,
|
|
"extra --user-agent flag should override navigator.userAgent"
|
|
);
|
|
|
|
handle.close().await.expect("close cleanly");
|
|
}
|
|
|
|
/// Tiny dotted-quad finder — avoids pulling `regex` in just for one
|
|
/// test. Scans the first valid IPv4 substring (four 0..=255 octets
|
|
/// separated by dots).
|
|
fn extract_ipv4(s: &str) -> Option<String> {
|
|
let bytes = s.as_bytes();
|
|
let mut i = 0;
|
|
while i < bytes.len() {
|
|
if !bytes[i].is_ascii_digit() {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
let start = i;
|
|
while i < bytes.len() && (bytes[i].is_ascii_digit() || bytes[i] == b'.') {
|
|
i += 1;
|
|
}
|
|
let candidate = &s[start..i];
|
|
let parts: Vec<&str> = candidate.split('.').collect();
|
|
if parts.len() == 4 && parts.iter().all(|p| p.parse::<u8>().is_ok()) {
|
|
return Some(candidate.to_string());
|
|
}
|
|
}
|
|
None
|
|
}
|