fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(crawler): cap the number of page images per chapter

Browse Source 
The per-image byte cap bounds each download but not the count, so a
hostile or compromised reader page listing thousands of <img> tags could
drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER`
(CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and
reject an over-cap chapter with a failed ack (exponential backoff) rather
than downloading it.

Threaded through sync_chapter_content and its three call sites (daemon
dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right
after parse. The cap is an env-only safety knob, preserved across settings
reloads.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-03 21:05:28 +02:00
parent 592747f1e0
commit 5dc93bfb84
9 changed files with 72 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/Cargo.lock generated
View File

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mangalord"
version = "0.93.4"
version = "0.93.5"
dependencies = [
"anyhow",
"argon2",

2
backend/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "mangalord"
version = "0.93.4"
version = "0.93.5"
edition = "2021"
default-run = "mangalord"

5
backend/src/app.rs
View File

@@ -716,6 +716,7 @@ async fn spawn_crawler_daemon(
rate: Arc::clone(&rate),
download_allowlist: cfg.download_allowlist.clone(),
max_image_bytes: cfg.max_image_bytes,
max_images_per_chapter: cfg.max_images_per_chapter,
analysis_enabled,
transient_failures: Arc::new(AtomicU32::new(0)),
restart_threshold: cfg.browser_restart_threshold,
@@ -732,6 +733,7 @@ async fn spawn_crawler_daemon(
rate: Arc::clone(&rate),
download_allowlist: cfg.download_allowlist.clone(),
max_image_bytes: cfg.max_image_bytes,
max_images_per_chapter: cfg.max_images_per_chapter,
tor: tor.as_ref().map(Arc::clone),
});
@@ -918,6 +920,8 @@ struct RealChapterDispatcher {
rate: Arc<HostRateLimiters>,
download_allowlist: DownloadAllowlist,
max_image_bytes: usize,
/// Per-chapter image-count cap (see `CrawlerConfig::max_images_per_chapter`).
max_images_per_chapter: usize,
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate
/// (read live) so toggling analysis at runtime takes effect without a
/// crawler respawn. Mirrors the analysis enable setting.
@@ -975,6 +979,7 @@ impl ChapterDispatcher for RealChapterDispatcher {
false,
&self.download_allowlist,
self.max_image_bytes,
self.max_images_per_chapter,
self.tor.as_deref(),
Some(&self.status),
self.analysis_enabled.load(Ordering::Relaxed),

9
backend/src/bin/crawler.rs
View File

@@ -278,6 +278,12 @@ async fn run(
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
// Per-chapter image *count* cap — bounds total disk against a hostile
// reader page listing thousands of <img> tags. `0` disables it.
let max_images_per_chapter: usize = std::env::var("CRAWLER_MAX_IMAGES_PER_CHAPTER")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(2000);
let stats = pipeline::run_metadata_pass(
manager.as_ref(),
@@ -312,6 +318,7 @@ async fn run(
force_refetch_chapters,
Arc::clone(&allowlist),
max_image_bytes,
max_images_per_chapter,
tor.clone(),
)
.await?;
@@ -338,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
force_refetch: bool,
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<Arc<mangalord::crawler::tor::TorController>>,
) -> anyhow::Result<()> {
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
@@ -403,6 +411,7 @@ async fn sync_bookmarked_chapter_content(
force_refetch,
allowlist.as_ref(),
max_image_bytes,
max_images_per_chapter,
tor.as_deref(),
// CLI one-shot — no live status surface.
None,

9
backend/src/config.rs
View File

@@ -441,6 +441,13 @@ pub struct CrawlerConfig {
pub download_allowlist: DownloadAllowlist,
/// Hard upper bound on a single image download. Defaults to 32 MiB.
pub max_image_bytes: usize,
/// Hard upper bound on the number of page images in one chapter. A
/// hostile reader page could otherwise list thousands of `<img>`
/// tags; `max_image_bytes` caps each one but not the count, so the
/// product is an unbounded disk-fill. A chapter exceeding this is
/// acked failed rather than downloaded. `0` disables the cap.
/// Defaults to 2000. `CRAWLER_MAX_IMAGES_PER_CHAPTER`.
pub max_images_per_chapter: usize,
/// Max manga detail fetches per metadata pass. `0` means no cap
/// (full sweep up to the source's own bound). Sourced from
/// `CRAWLER_LIMIT`, mirroring the CLI binary.
@@ -485,6 +492,7 @@ impl Default for CrawlerConfig {
browser: LaunchOptions::headless(),
download_allowlist: DownloadAllowlist::new(),
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
max_images_per_chapter: 2000,
manga_limit: 0,
job_timeout: Duration::from_secs(600),
metadata_max_consecutive_failures: 10,
@@ -642,6 +650,7 @@ impl CrawlerConfig {
browser: LaunchOptions::from_env(),
download_allowlist,
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
max_images_per_chapter: env_usize("CRAWLER_MAX_IMAGES_PER_CHAPTER", 2000),
manga_limit: env_usize("CRAWLER_LIMIT", 0),
job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)),
metadata_max_consecutive_failures: env_u64(

41
backend/src/crawler/content.rs
View File

@@ -228,6 +228,7 @@ pub async fn sync_chapter_content(
force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<&crate::crawler::tor::TorController>,
progress: Option<&crate::crawler::status::StatusHandle>,
enqueue_analysis: bool,
@@ -235,7 +236,8 @@ pub async fn sync_chapter_content(
let started = std::time::Instant::now();
let result = sync_chapter_content_inner(
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
force_refetch, allowlist, max_image_bytes, max_images_per_chapter, tor, progress,
enqueue_analysis,
)
.await;
let duration_ms = started.elapsed().as_millis() as i64;
@@ -285,6 +287,7 @@ async fn sync_chapter_content_inner(
force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<&crate::crawler::tor::TorController>,
// Optional live-status sink for the realtime page counter. The daemon
// dispatcher passes the shared handle (the chapter has already been
@@ -345,6 +348,16 @@ async fn sync_chapter_content_inner(
if images.is_empty() {
anyhow::bail!("no page images parsed from {source_url}");
}
// Bound total disk per chapter: the per-image byte cap doesn't stop a
// hostile reader page from listing thousands of <img> tags. Ack failed
// (the caller records it and backs off) rather than downloading them.
if let Some(over) = image_count_over_cap(images.len(), max_images_per_chapter) {
anyhow::bail!(
"chapter at {source_url} lists {} page images, over the {} cap",
over.count,
over.cap
);
}
// Resolve image URLs against the chapter URL (they may be relative).
let base = reqwest::Url::parse(source_url).context("parse chapter URL")?;
@@ -430,6 +443,19 @@ fn remaining_after_prefix(max_image_bytes: usize, prefix_len: usize) -> usize {
max_image_bytes.saturating_sub(prefix_len)
}
/// A rejected over-cap image count, carrying the numbers for the error.
struct ImageCountOverCap {
count: usize,
cap: usize,
}
/// `Some(..)` when `count` exceeds the per-chapter image cap. A `cap` of
/// `0` disables the check (unbounded), matching the config's "0 = no cap"
/// contract.
fn image_count_over_cap(count: usize, cap: usize) -> Option<ImageCountOverCap> {
(cap != 0 && count > cap).then_some(ImageCountOverCap { count, cap })
}
/// Download a single page image, validate it's really an image, and
/// stream it to storage. Returns the storage key + content type. Does
/// not touch the DB — persistence is batched into one short transaction
@@ -676,6 +702,19 @@ mod tests {
assert!(remaining < cap - SNIFF_PREFIX_BYTES);
}
#[test]
fn image_count_cap_rejects_only_over_cap_and_respects_disable() {
// Under / at the cap: accepted.
assert!(image_count_over_cap(0, 2000).is_none());
assert!(image_count_over_cap(2000, 2000).is_none());
// Over the cap: rejected, carrying the numbers for the error.
let over = image_count_over_cap(2001, 2000).expect("over cap");
assert_eq!(over.count, 2001);
assert_eq!(over.cap, 2000);
// `0` disables the cap entirely (unbounded), matching config contract.
assert!(image_count_over_cap(1_000_000, 0).is_none());
}
#[test]
fn tail_budget_saturates_when_prefix_hits_cap() {
// Body shorter than the sniff window: prefix drained == cap, tail

3
backend/src/crawler/resync.rs
View File

@@ -81,6 +81,8 @@ pub struct RealResyncService {
pub rate: Arc<HostRateLimiters>,
pub download_allowlist: DownloadAllowlist,
pub max_image_bytes: usize,
/// Per-chapter image-count cap (see `CrawlerConfig::max_images_per_chapter`).
pub max_images_per_chapter: usize,
pub tor: Option<Arc<TorController>>,
}
@@ -256,6 +258,7 @@ impl ResyncService for RealResyncService {
true,
&self.download_allowlist,
self.max_image_bytes,
self.max_images_per_chapter,
self.tor.as_deref(),
// Admin resync isn't a daemon worker slot — no live status.
None,

3
backend/src/settings.rs
View File

@@ -254,6 +254,9 @@ impl CrawlerSettings {
.map(str::to_string),
download_allowlist,
max_image_bytes: self.max_image_bytes as usize,
// Env-only safety cap, not a runtime-editable setting — preserve
// it from the base so a settings reload keeps the boot value.
max_images_per_chapter: base.max_images_per_chapter,
manga_limit: self.manga_limit as usize,
job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,