fix(crawler): cap the number of page images per chapter
The per-image byte cap bounds each download but not the count, so a hostile or compromised reader page listing thousands of <img> tags could drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER` (CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and reject an over-cap chapter with a failed ack (exponential backoff) rather than downloading it. Threaded through sync_chapter_content and its three call sites (daemon dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right after parse. The cap is an env-only safety knob, preserved across settings reloads. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -228,6 +228,7 @@ pub async fn sync_chapter_content(
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
max_images_per_chapter: usize,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
progress: Option<&crate::crawler::status::StatusHandle>,
|
||||
enqueue_analysis: bool,
|
||||
@@ -235,7 +236,8 @@ pub async fn sync_chapter_content(
|
||||
let started = std::time::Instant::now();
|
||||
let result = sync_chapter_content_inner(
|
||||
browser, db, storage, http, rate, chapter_id, manga_id, source_url,
|
||||
force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis,
|
||||
force_refetch, allowlist, max_image_bytes, max_images_per_chapter, tor, progress,
|
||||
enqueue_analysis,
|
||||
)
|
||||
.await;
|
||||
let duration_ms = started.elapsed().as_millis() as i64;
|
||||
@@ -285,6 +287,7 @@ async fn sync_chapter_content_inner(
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
max_images_per_chapter: usize,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
// Optional live-status sink for the realtime page counter. The daemon
|
||||
// dispatcher passes the shared handle (the chapter has already been
|
||||
@@ -345,6 +348,16 @@ async fn sync_chapter_content_inner(
|
||||
if images.is_empty() {
|
||||
anyhow::bail!("no page images parsed from {source_url}");
|
||||
}
|
||||
// Bound total disk per chapter: the per-image byte cap doesn't stop a
|
||||
// hostile reader page from listing thousands of <img> tags. Ack failed
|
||||
// (the caller records it and backs off) rather than downloading them.
|
||||
if let Some(over) = image_count_over_cap(images.len(), max_images_per_chapter) {
|
||||
anyhow::bail!(
|
||||
"chapter at {source_url} lists {} page images, over the {} cap",
|
||||
over.count,
|
||||
over.cap
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve image URLs against the chapter URL (they may be relative).
|
||||
let base = reqwest::Url::parse(source_url).context("parse chapter URL")?;
|
||||
@@ -430,6 +443,19 @@ fn remaining_after_prefix(max_image_bytes: usize, prefix_len: usize) -> usize {
|
||||
max_image_bytes.saturating_sub(prefix_len)
|
||||
}
|
||||
|
||||
/// A rejected over-cap image count, carrying the numbers for the error.
|
||||
struct ImageCountOverCap {
|
||||
count: usize,
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
/// `Some(..)` when `count` exceeds the per-chapter image cap. A `cap` of
|
||||
/// `0` disables the check (unbounded), matching the config's "0 = no cap"
|
||||
/// contract.
|
||||
fn image_count_over_cap(count: usize, cap: usize) -> Option<ImageCountOverCap> {
|
||||
(cap != 0 && count > cap).then_some(ImageCountOverCap { count, cap })
|
||||
}
|
||||
|
||||
/// Download a single page image, validate it's really an image, and
|
||||
/// stream it to storage. Returns the storage key + content type. Does
|
||||
/// not touch the DB — persistence is batched into one short transaction
|
||||
@@ -676,6 +702,19 @@ mod tests {
|
||||
assert!(remaining < cap - SNIFF_PREFIX_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_count_cap_rejects_only_over_cap_and_respects_disable() {
|
||||
// Under / at the cap: accepted.
|
||||
assert!(image_count_over_cap(0, 2000).is_none());
|
||||
assert!(image_count_over_cap(2000, 2000).is_none());
|
||||
// Over the cap: rejected, carrying the numbers for the error.
|
||||
let over = image_count_over_cap(2001, 2000).expect("over cap");
|
||||
assert_eq!(over.count, 2001);
|
||||
assert_eq!(over.cap, 2000);
|
||||
// `0` disables the cap entirely (unbounded), matching config contract.
|
||||
assert!(image_count_over_cap(1_000_000, 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tail_budget_saturates_when_prefix_hits_cap() {
|
||||
// Body shorter than the sniff window: prefix drained == cap, tail
|
||||
|
||||
Reference in New Issue
Block a user