fix(crawler): cap the number of page images per chapter

The per-image byte cap bounds each download but not the count, so a
hostile or compromised reader page listing thousands of <img> tags could
drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER`
(CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and
reject an over-cap chapter with a failed ack (exponential backoff) rather
than downloading it.

Threaded through sync_chapter_content and its three call sites (daemon
dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right
after parse. The cap is an env-only safety knob, preserved across settings
reloads.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-03 21:05:28 +02:00
parent 592747f1e0
commit 5dc93bfb84
9 changed files with 72 additions and 4 deletions

View File

@@ -228,6 +228,7 @@ pub async fn sync_chapter_content(
force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<&crate::crawler::tor::TorController>,
progress: Option<&crate::crawler::status::StatusHandle>,
enqueue_analysis: bool,
@@ -235,7 +236,8 @@ pub async fn sync_chapter_content(
let started = std::time::Instant::now();
let result = sync_chapter_content_inner(
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
force_refetch, allowlist, max_image_bytes, max_images_per_chapter, tor, progress,
enqueue_analysis,
)
.await;
let duration_ms = started.elapsed().as_millis() as i64;
@@ -285,6 +287,7 @@ async fn sync_chapter_content_inner(
force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<&crate::crawler::tor::TorController>,
// Optional live-status sink for the realtime page counter. The daemon
// dispatcher passes the shared handle (the chapter has already been
@@ -345,6 +348,16 @@ async fn sync_chapter_content_inner(
if images.is_empty() {
anyhow::bail!("no page images parsed from {source_url}");
}
// Bound total disk per chapter: the per-image byte cap doesn't stop a
// hostile reader page from listing thousands of <img> tags. Ack failed
// (the caller records it and backs off) rather than downloading them.
if let Some(over) = image_count_over_cap(images.len(), max_images_per_chapter) {
anyhow::bail!(
"chapter at {source_url} lists {} page images, over the {} cap",
over.count,
over.cap
);
}
// Resolve image URLs against the chapter URL (they may be relative).
let base = reqwest::Url::parse(source_url).context("parse chapter URL")?;
@@ -430,6 +443,19 @@ fn remaining_after_prefix(max_image_bytes: usize, prefix_len: usize) -> usize {
max_image_bytes.saturating_sub(prefix_len)
}
/// A rejected over-cap image count, carrying the numbers for the error.
struct ImageCountOverCap {
count: usize,
cap: usize,
}
/// `Some(..)` when `count` exceeds the per-chapter image cap. A `cap` of
/// `0` disables the check (unbounded), matching the config's "0 = no cap"
/// contract.
fn image_count_over_cap(count: usize, cap: usize) -> Option<ImageCountOverCap> {
(cap != 0 && count > cap).then_some(ImageCountOverCap { count, cap })
}
/// Download a single page image, validate it's really an image, and
/// stream it to storage. Returns the storage key + content type. Does
/// not touch the DB — persistence is batched into one short transaction
@@ -676,6 +702,19 @@ mod tests {
assert!(remaining < cap - SNIFF_PREFIX_BYTES);
}
#[test]
fn image_count_cap_rejects_only_over_cap_and_respects_disable() {
// Under / at the cap: accepted.
assert!(image_count_over_cap(0, 2000).is_none());
assert!(image_count_over_cap(2000, 2000).is_none());
// Over the cap: rejected, carrying the numbers for the error.
let over = image_count_over_cap(2001, 2000).expect("over cap");
assert_eq!(over.count, 2001);
assert_eq!(over.cap, 2000);
// `0` disables the cap entirely (unbounded), matching config contract.
assert!(image_count_over_cap(1_000_000, 0).is_none());
}
#[test]
fn tail_budget_saturates_when_prefix_hits_cap() {
// Body shorter than the sniff window: prefix drained == cap, tail