fix(crawler): cap the number of page images per chapter

The per-image byte cap bounds each download but not the count, so a
hostile or compromised reader page listing thousands of <img> tags could
drive an unbounded disk fill. Add `CRAWLER_MAX_IMAGES_PER_CHAPTER`
(CrawlerConfig::max_images_per_chapter, default 2000, 0 = disabled) and
reject an over-cap chapter with a failed ack (exponential backoff) rather
than downloading it.

Threaded through sync_chapter_content and its three call sites (daemon
dispatcher, admin resync, CLI); enforced via `image_count_over_cap` right
after parse. The cap is an env-only safety knob, preserved across settings
reloads.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-03 21:05:28 +02:00
parent 592747f1e0
commit 5dc93bfb84
9 changed files with 72 additions and 4 deletions

View File

@@ -278,6 +278,12 @@ async fn run(
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
// Per-chapter image *count* cap — bounds total disk against a hostile
// reader page listing thousands of <img> tags. `0` disables it.
let max_images_per_chapter: usize = std::env::var("CRAWLER_MAX_IMAGES_PER_CHAPTER")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(2000);
let stats = pipeline::run_metadata_pass(
manager.as_ref(),
@@ -312,6 +318,7 @@ async fn run(
force_refetch_chapters,
Arc::clone(&allowlist),
max_image_bytes,
max_images_per_chapter,
tor.clone(),
)
.await?;
@@ -338,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
force_refetch: bool,
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
max_image_bytes: usize,
max_images_per_chapter: usize,
tor: Option<Arc<mangalord::crawler::tor::TorController>>,
) -> anyhow::Result<()> {
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
@@ -403,6 +411,7 @@ async fn sync_bookmarked_chapter_content(
force_refetch,
allowlist.as_ref(),
max_image_bytes,
max_images_per_chapter,
tor.as_deref(),
// CLI one-shot — no live status surface.
None,