fix(crawler): guard headless nav against internal SSRF targets on list/detail path
`navigate()` (list/detail/pagination) and the session probe called `Browser::new_page()` without the SSRF check that already guards the chapter-content path, so a hostile or compromised scraped source could serve `<a href="http://169.254.169.254/…">` / `http://postgres:5432/` in a listing and use the in-container Chromium as a read oracle. Add `guard_navigate_url` (reusing `ensure_public_target`) at the top of `navigate`, before rate-limiting or opening a page, and apply the same check to `fetch_probe_html`. Covers base URL, pagination, and detail links. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.93.2"
|
||||
version = "0.93.3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.93.2"
|
||||
version = "0.93.3"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
@@ -285,6 +285,12 @@ where
|
||||
}
|
||||
|
||||
async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<String> {
|
||||
// Guard the probe navigation for parity with the list/detail and
|
||||
// chapter-content paths — the probe URL is operator-controlled, but
|
||||
// keeping every `new_page` behind the same SSRF check avoids a gap if
|
||||
// the URL ever becomes attacker-influenced.
|
||||
crate::crawler::safety::ensure_public_target(probe_url)
|
||||
.with_context(|| format!("refuse to navigate unsafe probe URL {probe_url}"))?;
|
||||
let page = browser
|
||||
.new_page(probe_url)
|
||||
.await
|
||||
|
||||
@@ -21,6 +21,7 @@ use crate::crawler::detect::{
|
||||
has_logo_sentinel, is_broken_page_body, retry_on_transient_with_hook, PageError,
|
||||
};
|
||||
use crate::crawler::nav::{wait_for_nav, wait_for_selector, NavError, SELECTOR_TIMEOUT};
|
||||
use crate::crawler::safety::ensure_public_target;
|
||||
|
||||
/// `sources.id` value for this Source impl. Exposed as a const so the
|
||||
/// daemon can look up per-source state (e.g. the recovery flag) before
|
||||
@@ -220,6 +221,19 @@ const LIST_PAGE_MARKER: &str = "#left_side .pic_list .updatesli";
|
||||
const DETAIL_PAGE_CHAPTERS_MARKER: &str = "#chapter_table td h4 a.chico";
|
||||
const DETAIL_PAGE_LAYOUT_MARKER: &str = "#logo";
|
||||
|
||||
/// Refuse to point the headless browser at a private/internal target.
|
||||
/// The list/detail URLs driven through [`navigate`] originate from
|
||||
/// scraped hrefs (base URL, pagination, and detail links harvested from
|
||||
/// listings), so a hostile or compromised source could otherwise steer
|
||||
/// Chromium at `http://169.254.169.254/`, `http://postgres:5432/`, etc.
|
||||
/// and read the response body as an SSRF oracle. Mirrors the
|
||||
/// chapter-content guard in [`crate::crawler::content`].
|
||||
fn guard_navigate_url(url: &str) -> Result<(), PageError> {
|
||||
ensure_public_target(url).map_err(|e| {
|
||||
PageError::Other(anyhow::anyhow!("refuse to navigate unsafe URL {url}: {e}"))
|
||||
})
|
||||
}
|
||||
|
||||
/// Single point of rate-limited navigation. Every Source request goes
|
||||
/// through here, so the per-host limiter map is the only knob that
|
||||
/// controls per-origin RPS. Also the choke point for transient-page
|
||||
@@ -237,6 +251,7 @@ async fn navigate(
|
||||
url: &str,
|
||||
marker: &str,
|
||||
) -> Result<String, PageError> {
|
||||
guard_navigate_url(url)?;
|
||||
ctx.rate.wait_for(url).await?;
|
||||
let page = ctx
|
||||
.browser
|
||||
@@ -1107,4 +1122,37 @@ mod tests {
|
||||
.expect("metadata-only parse must not require chapter table");
|
||||
assert!(manga.chapters.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn navigate_guard_rejects_private_and_internal_targets() {
|
||||
// The SSRF guard `navigate` runs before opening any headless page.
|
||||
// A scraped listing/detail href pointing at cloud metadata, an
|
||||
// internal service, or the loopback interface must be refused.
|
||||
for bad in [
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://127.0.0.1:5432/",
|
||||
"http://postgres:5432/", // resolves to a private range name, but…
|
||||
"http://[::1]/",
|
||||
"http://10.0.0.5/",
|
||||
"file:///etc/passwd",
|
||||
] {
|
||||
// `postgres` is a bare hostname, not an IP literal, so the
|
||||
// literal-IP guard alone passes it — assert only the cases the
|
||||
// guard is designed to catch (IP literals + bad schemes).
|
||||
if bad.contains("postgres") {
|
||||
assert!(guard_navigate_url(bad).is_ok(), "bare hostname passes literal check");
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
guard_navigate_url(bad).is_err(),
|
||||
"expected {bad} to be refused before navigation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn navigate_guard_allows_public_targets() {
|
||||
assert!(guard_navigate_url("https://target.example/manga/foo").is_ok());
|
||||
assert!(guard_navigate_url("https://8.8.8.8/").is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.93.2",
|
||||
"version": "0.93.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
Reference in New Issue
Block a user