feat(crawler): guard Chromium chapter navigation against internal targets

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-01 07:22:10 +02:00
parent 2ed42f7b9e
commit ed18d95bb0
4 changed files with 39 additions and 4 deletions

2
backend/Cargo.lock generated
View File

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mangalord"
version = "0.91.0"
version = "0.92.0"
dependencies = [
"anyhow",
"argon2",

View File

@@ -1,6 +1,6 @@
[package]
name = "mangalord"
version = "0.91.0"
version = "0.92.0"
edition = "2021"
default-run = "mangalord"

View File

@@ -18,7 +18,7 @@ use uuid::Uuid;
use crate::crawler::detect::PageError;
use crate::crawler::rate_limit::HostRateLimiters;
use crate::crawler::safety::{fetch_stream, looks_like_image, DownloadAllowlist};
use crate::crawler::safety::{ensure_public_target, fetch_stream, looks_like_image, DownloadAllowlist};
use crate::crawler::session::{self, ChapterProbe};
use crate::storage::{Storage, StorageError};
@@ -95,6 +95,19 @@ enum ChapterFetchOutcome {
PersistentTransient,
}
/// Refuse to navigate Chromium at a chapter URL that points inside the
/// deployment. The image-download path goes through `is_safe_url`, but
/// `new_page` is a second network surface: a `chapter_sources` row whose
/// host resolves to (or was crafted to be) a private IP would otherwise let
/// the headless browser probe `postgres:5432`, the cloud metadata service,
/// etc. Reuses the allowlist-free `ensure_public_target` (scheme +
/// private-IP literal check) since there is no per-host allowlist for the
/// scraped catalog itself.
fn guard_nav_url(source_url: &str) -> anyhow::Result<()> {
ensure_public_target(source_url)
.map_err(|e| anyhow::anyhow!("refuse to navigate unsafe chapter URL {source_url}: {e}"))
}
/// Single rate-limited Chromium navigation to the chapter URL,
/// returning the page HTML. Extracted from `sync_chapter_content` so
/// the recircuit loop can call it once per attempt.
@@ -103,6 +116,7 @@ async fn fetch_chapter_html_once(
rate: &HostRateLimiters,
source_url: &str,
) -> anyhow::Result<String> {
guard_nav_url(source_url)?;
rate.wait_for(source_url).await?;
let page = browser
.new_page(source_url)
@@ -609,6 +623,27 @@ mod tests {
use super::*;
use crate::storage::LocalStorage;
#[test]
fn guard_nav_url_rejects_private_and_loopback_targets() {
// A chapter_sources row that resolves to / was crafted as an
// internal target must be refused before Chromium navigates.
for url in [
"http://127.0.0.1:5432/",
"http://169.254.169.254/latest/meta-data/",
"http://10.0.0.1/chapter/1",
"http://localhost:8080/",
"file:///etc/passwd",
] {
assert!(guard_nav_url(url).is_err(), "must reject {url}");
}
}
#[test]
fn guard_nav_url_allows_public_chapter_urls() {
assert!(guard_nav_url("https://reader.example.com/chapter/42").is_ok());
assert!(guard_nav_url("http://manga-host.test/c/1/p/2").is_ok());
}
#[tokio::test]
async fn cleanup_orphans_deletes_written_keys() {
let dir = tempfile::tempdir().unwrap();

View File

@@ -1,6 +1,6 @@
{
"name": "mangalord-frontend",
"version": "0.91.0",
"version": "0.92.0",
"private": true,
"type": "module",
"scripts": {