fix(crawler): cap the number of page images per chapter
The per-image byte cap bounds each download but not the count, so a hostile or compromised reader page listing thousands of <img> tags could drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER` (CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and reject an over-cap chapter with a failed ack (exponential backoff) rather than downloading it. Threaded through sync_chapter_content and its three call sites (daemon dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right after parse. The cap is an env-only safety knob, preserved across settings reloads. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -278,6 +278,12 @@ async fn run(
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
|
||||
// Per-chapter image *count* cap — bounds total disk against a hostile
|
||||
// reader page listing thousands of <img> tags. `0` disables it.
|
||||
let max_images_per_chapter: usize = std::env::var("CRAWLER_MAX_IMAGES_PER_CHAPTER")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(2000);
|
||||
|
||||
let stats = pipeline::run_metadata_pass(
|
||||
manager.as_ref(),
|
||||
@@ -312,6 +318,7 @@ async fn run(
|
||||
force_refetch_chapters,
|
||||
Arc::clone(&allowlist),
|
||||
max_image_bytes,
|
||||
max_images_per_chapter,
|
||||
tor.clone(),
|
||||
)
|
||||
.await?;
|
||||
@@ -338,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
|
||||
force_refetch: bool,
|
||||
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
|
||||
max_image_bytes: usize,
|
||||
max_images_per_chapter: usize,
|
||||
tor: Option<Arc<mangalord::crawler::tor::TorController>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
|
||||
@@ -403,6 +411,7 @@ async fn sync_bookmarked_chapter_content(
|
||||
force_refetch,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
max_images_per_chapter,
|
||||
tor.as_deref(),
|
||||
// CLI one-shot — no live status surface.
|
||||
None,
|
||||
|
||||
Reference in New Issue
Block a user