feat(crawler): guard Chromium chapter navigation against internal targets
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.91.0"
|
version = "0.92.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.91.0"
|
version = "0.92.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
default-run = "mangalord"
|
default-run = "mangalord"
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use uuid::Uuid;
|
|||||||
|
|
||||||
use crate::crawler::detect::PageError;
|
use crate::crawler::detect::PageError;
|
||||||
use crate::crawler::rate_limit::HostRateLimiters;
|
use crate::crawler::rate_limit::HostRateLimiters;
|
||||||
use crate::crawler::safety::{fetch_stream, looks_like_image, DownloadAllowlist};
|
use crate::crawler::safety::{ensure_public_target, fetch_stream, looks_like_image, DownloadAllowlist};
|
||||||
use crate::crawler::session::{self, ChapterProbe};
|
use crate::crawler::session::{self, ChapterProbe};
|
||||||
use crate::storage::{Storage, StorageError};
|
use crate::storage::{Storage, StorageError};
|
||||||
|
|
||||||
@@ -95,6 +95,19 @@ enum ChapterFetchOutcome {
|
|||||||
PersistentTransient,
|
PersistentTransient,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Refuse to navigate Chromium at a chapter URL that points inside the
|
||||||
|
/// deployment. The image-download path goes through `is_safe_url`, but
|
||||||
|
/// `new_page` is a second network surface: a `chapter_sources` row whose
|
||||||
|
/// host resolves to (or was crafted to be) a private IP would otherwise let
|
||||||
|
/// the headless browser probe `postgres:5432`, the cloud metadata service,
|
||||||
|
/// etc. Reuses the allowlist-free `ensure_public_target` (scheme +
|
||||||
|
/// private-IP literal check) since there is no per-host allowlist for the
|
||||||
|
/// scraped catalog itself.
|
||||||
|
fn guard_nav_url(source_url: &str) -> anyhow::Result<()> {
|
||||||
|
ensure_public_target(source_url)
|
||||||
|
.map_err(|e| anyhow::anyhow!("refuse to navigate unsafe chapter URL {source_url}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
/// Single rate-limited Chromium navigation to the chapter URL,
|
/// Single rate-limited Chromium navigation to the chapter URL,
|
||||||
/// returning the page HTML. Extracted from `sync_chapter_content` so
|
/// returning the page HTML. Extracted from `sync_chapter_content` so
|
||||||
/// the recircuit loop can call it once per attempt.
|
/// the recircuit loop can call it once per attempt.
|
||||||
@@ -103,6 +116,7 @@ async fn fetch_chapter_html_once(
|
|||||||
rate: &HostRateLimiters,
|
rate: &HostRateLimiters,
|
||||||
source_url: &str,
|
source_url: &str,
|
||||||
) -> anyhow::Result<String> {
|
) -> anyhow::Result<String> {
|
||||||
|
guard_nav_url(source_url)?;
|
||||||
rate.wait_for(source_url).await?;
|
rate.wait_for(source_url).await?;
|
||||||
let page = browser
|
let page = browser
|
||||||
.new_page(source_url)
|
.new_page(source_url)
|
||||||
@@ -609,6 +623,27 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::storage::LocalStorage;
|
use crate::storage::LocalStorage;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn guard_nav_url_rejects_private_and_loopback_targets() {
|
||||||
|
// A chapter_sources row that resolves to / was crafted as an
|
||||||
|
// internal target must be refused before Chromium navigates.
|
||||||
|
for url in [
|
||||||
|
"http://127.0.0.1:5432/",
|
||||||
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
|
"http://10.0.0.1/chapter/1",
|
||||||
|
"http://localhost:8080/",
|
||||||
|
"file:///etc/passwd",
|
||||||
|
] {
|
||||||
|
assert!(guard_nav_url(url).is_err(), "must reject {url}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn guard_nav_url_allows_public_chapter_urls() {
|
||||||
|
assert!(guard_nav_url("https://reader.example.com/chapter/42").is_ok());
|
||||||
|
assert!(guard_nav_url("http://manga-host.test/c/1/p/2").is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cleanup_orphans_deletes_written_keys() {
|
async fn cleanup_orphans_deletes_written_keys() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "mangalord-frontend",
|
"name": "mangalord-frontend",
|
||||||
"version": "0.91.0",
|
"version": "0.92.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
Reference in New Issue
Block a user