fix(crawler): cap the number of page images per chapter
The per-image byte cap bounds each download but not the count, so a hostile or compromised reader page listing thousands of <img> tags could drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER` (CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and reject an over-cap chapter with a failed ack (exponential backoff) rather than downloading it. Threaded through sync_chapter_content and its three call sites (daemon dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right after parse. The cap is an env-only safety knob, preserved across settings reloads. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -716,6 +716,7 @@ async fn spawn_crawler_daemon(
|
||||
rate: Arc::clone(&rate),
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
max_images_per_chapter: cfg.max_images_per_chapter,
|
||||
analysis_enabled,
|
||||
transient_failures: Arc::new(AtomicU32::new(0)),
|
||||
restart_threshold: cfg.browser_restart_threshold,
|
||||
@@ -732,6 +733,7 @@ async fn spawn_crawler_daemon(
|
||||
rate: Arc::clone(&rate),
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
max_images_per_chapter: cfg.max_images_per_chapter,
|
||||
tor: tor.as_ref().map(Arc::clone),
|
||||
});
|
||||
|
||||
@@ -918,6 +920,8 @@ struct RealChapterDispatcher {
|
||||
rate: Arc<HostRateLimiters>,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
/// Per-chapter image-count cap (see `CrawlerConfig::max_images_per_chapter`).
|
||||
max_images_per_chapter: usize,
|
||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate
|
||||
/// (read live) so toggling analysis at runtime takes effect without a
|
||||
/// crawler respawn. Mirrors the analysis enable setting.
|
||||
@@ -975,6 +979,7 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
false,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
self.max_images_per_chapter,
|
||||
self.tor.as_deref(),
|
||||
Some(&self.status),
|
||||
self.analysis_enabled.load(Ordering::Relaxed),
|
||||
|
||||
Reference in New Issue
Block a user