diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2f8e4b2..60991c0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.93.4" +version = "0.93.5" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7d64e2e..bdf5927 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.93.4" +version = "0.93.5" edition = "2021" default-run = "mangalord" diff --git a/backend/src/app.rs b/backend/src/app.rs index 86a3a52..5e2e8ce 100644 --- a/backend/src/app.rs +++ b/backend/src/app.rs @@ -716,6 +716,7 @@ async fn spawn_crawler_daemon( rate: Arc::clone(&rate), download_allowlist: cfg.download_allowlist.clone(), max_image_bytes: cfg.max_image_bytes, + max_images_per_chapter: cfg.max_images_per_chapter, analysis_enabled, transient_failures: Arc::new(AtomicU32::new(0)), restart_threshold: cfg.browser_restart_threshold, @@ -732,6 +733,7 @@ async fn spawn_crawler_daemon( rate: Arc::clone(&rate), download_allowlist: cfg.download_allowlist.clone(), max_image_bytes: cfg.max_image_bytes, + max_images_per_chapter: cfg.max_images_per_chapter, tor: tor.as_ref().map(Arc::clone), }); @@ -918,6 +920,8 @@ struct RealChapterDispatcher { rate: Arc, download_allowlist: DownloadAllowlist, max_image_bytes: usize, + /// Per-chapter image-count cap (see `CrawlerConfig::max_images_per_chapter`). + max_images_per_chapter: usize, /// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate /// (read live) so toggling analysis at runtime takes effect without a /// crawler respawn. Mirrors the analysis enable setting. @@ -975,6 +979,7 @@ impl ChapterDispatcher for RealChapterDispatcher { false, &self.download_allowlist, self.max_image_bytes, + self.max_images_per_chapter, self.tor.as_deref(), Some(&self.status), self.analysis_enabled.load(Ordering::Relaxed), diff --git a/backend/src/bin/crawler.rs b/backend/src/bin/crawler.rs index e9a15e9..808dd39 100644 --- a/backend/src/bin/crawler.rs +++ b/backend/src/bin/crawler.rs @@ -278,6 +278,12 @@ async fn run( .ok() .and_then(|s| s.parse().ok()) .unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES); + // Per-chapter image *count* cap — bounds total disk against a hostile + // reader page listing thousands of tags. `0` disables it. + let max_images_per_chapter: usize = std::env::var("CRAWLER_MAX_IMAGES_PER_CHAPTER") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(2000); let stats = pipeline::run_metadata_pass( manager.as_ref(), @@ -312,6 +318,7 @@ async fn run( force_refetch_chapters, Arc::clone(&allowlist), max_image_bytes, + max_images_per_chapter, tor.clone(), ) .await?; @@ -338,6 +345,7 @@ async fn sync_bookmarked_chapter_content( force_refetch: bool, allowlist: Arc, max_image_bytes: usize, + max_images_per_chapter: usize, tor: Option>, ) -> anyhow::Result<()> { let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as( @@ -403,6 +411,7 @@ async fn sync_bookmarked_chapter_content( force_refetch, allowlist.as_ref(), max_image_bytes, + max_images_per_chapter, tor.as_deref(), // CLI one-shot — no live status surface. None, diff --git a/backend/src/config.rs b/backend/src/config.rs index f626b27..fccfbb9 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -441,6 +441,13 @@ pub struct CrawlerConfig { pub download_allowlist: DownloadAllowlist, /// Hard upper bound on a single image download. Defaults to 32 MiB. pub max_image_bytes: usize, + /// Hard upper bound on the number of page images in one chapter. A + /// hostile reader page could otherwise list thousands of `` + /// tags; `max_image_bytes` caps each one but not the count, so the + /// product is an unbounded disk-fill. A chapter exceeding this is + /// acked failed rather than downloaded. `0` disables the cap. + /// Defaults to 2000. `CRAWLER_MAX_IMAGES_PER_CHAPTER`. + pub max_images_per_chapter: usize, /// Max manga detail fetches per metadata pass. `0` means no cap /// (full sweep up to the source's own bound). Sourced from /// `CRAWLER_LIMIT`, mirroring the CLI binary. @@ -485,6 +492,7 @@ impl Default for CrawlerConfig { browser: LaunchOptions::headless(), download_allowlist: DownloadAllowlist::new(), max_image_bytes: DEFAULT_MAX_IMAGE_BYTES, + max_images_per_chapter: 2000, manga_limit: 0, job_timeout: Duration::from_secs(600), metadata_max_consecutive_failures: 10, @@ -642,6 +650,7 @@ impl CrawlerConfig { browser: LaunchOptions::from_env(), download_allowlist, max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES), + max_images_per_chapter: env_usize("CRAWLER_MAX_IMAGES_PER_CHAPTER", 2000), manga_limit: env_usize("CRAWLER_LIMIT", 0), job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)), metadata_max_consecutive_failures: env_u64( diff --git a/backend/src/crawler/content.rs b/backend/src/crawler/content.rs index e6f831b..34068ca 100644 --- a/backend/src/crawler/content.rs +++ b/backend/src/crawler/content.rs @@ -228,6 +228,7 @@ pub async fn sync_chapter_content( force_refetch: bool, allowlist: &DownloadAllowlist, max_image_bytes: usize, + max_images_per_chapter: usize, tor: Option<&crate::crawler::tor::TorController>, progress: Option<&crate::crawler::status::StatusHandle>, enqueue_analysis: bool, @@ -235,7 +236,8 @@ pub async fn sync_chapter_content( let started = std::time::Instant::now(); let result = sync_chapter_content_inner( browser, db, storage, http, rate, chapter_id, manga_id, source_url, - force_refetch, allowlist, max_image_bytes, tor, progress, enqueue_analysis, + force_refetch, allowlist, max_image_bytes, max_images_per_chapter, tor, progress, + enqueue_analysis, ) .await; let duration_ms = started.elapsed().as_millis() as i64; @@ -285,6 +287,7 @@ async fn sync_chapter_content_inner( force_refetch: bool, allowlist: &DownloadAllowlist, max_image_bytes: usize, + max_images_per_chapter: usize, tor: Option<&crate::crawler::tor::TorController>, // Optional live-status sink for the realtime page counter. The daemon // dispatcher passes the shared handle (the chapter has already been @@ -345,6 +348,16 @@ async fn sync_chapter_content_inner( if images.is_empty() { anyhow::bail!("no page images parsed from {source_url}"); } + // Bound total disk per chapter: the per-image byte cap doesn't stop a + // hostile reader page from listing thousands of tags. Ack failed + // (the caller records it and backs off) rather than downloading them. + if let Some(over) = image_count_over_cap(images.len(), max_images_per_chapter) { + anyhow::bail!( + "chapter at {source_url} lists {} page images, over the {} cap", + over.count, + over.cap + ); + } // Resolve image URLs against the chapter URL (they may be relative). let base = reqwest::Url::parse(source_url).context("parse chapter URL")?; @@ -430,6 +443,19 @@ fn remaining_after_prefix(max_image_bytes: usize, prefix_len: usize) -> usize { max_image_bytes.saturating_sub(prefix_len) } +/// A rejected over-cap image count, carrying the numbers for the error. +struct ImageCountOverCap { + count: usize, + cap: usize, +} + +/// `Some(..)` when `count` exceeds the per-chapter image cap. A `cap` of +/// `0` disables the check (unbounded), matching the config's "0 = no cap" +/// contract. +fn image_count_over_cap(count: usize, cap: usize) -> Option { + (cap != 0 && count > cap).then_some(ImageCountOverCap { count, cap }) +} + /// Download a single page image, validate it's really an image, and /// stream it to storage. Returns the storage key + content type. Does /// not touch the DB — persistence is batched into one short transaction @@ -676,6 +702,19 @@ mod tests { assert!(remaining < cap - SNIFF_PREFIX_BYTES); } + #[test] + fn image_count_cap_rejects_only_over_cap_and_respects_disable() { + // Under / at the cap: accepted. + assert!(image_count_over_cap(0, 2000).is_none()); + assert!(image_count_over_cap(2000, 2000).is_none()); + // Over the cap: rejected, carrying the numbers for the error. + let over = image_count_over_cap(2001, 2000).expect("over cap"); + assert_eq!(over.count, 2001); + assert_eq!(over.cap, 2000); + // `0` disables the cap entirely (unbounded), matching config contract. + assert!(image_count_over_cap(1_000_000, 0).is_none()); + } + #[test] fn tail_budget_saturates_when_prefix_hits_cap() { // Body shorter than the sniff window: prefix drained == cap, tail diff --git a/backend/src/crawler/resync.rs b/backend/src/crawler/resync.rs index 2b22740..e9334b5 100644 --- a/backend/src/crawler/resync.rs +++ b/backend/src/crawler/resync.rs @@ -81,6 +81,8 @@ pub struct RealResyncService { pub rate: Arc, pub download_allowlist: DownloadAllowlist, pub max_image_bytes: usize, + /// Per-chapter image-count cap (see `CrawlerConfig::max_images_per_chapter`). + pub max_images_per_chapter: usize, pub tor: Option>, } @@ -256,6 +258,7 @@ impl ResyncService for RealResyncService { true, &self.download_allowlist, self.max_image_bytes, + self.max_images_per_chapter, self.tor.as_deref(), // Admin resync isn't a daemon worker slot — no live status. None, diff --git a/backend/src/settings.rs b/backend/src/settings.rs index 322be38..a8ebe7f 100644 --- a/backend/src/settings.rs +++ b/backend/src/settings.rs @@ -254,6 +254,9 @@ impl CrawlerSettings { .map(str::to_string), download_allowlist, max_image_bytes: self.max_image_bytes as usize, + // Env-only safety cap, not a runtime-editable setting — preserve + // it from the base so a settings reload keeps the boot value. + max_images_per_chapter: base.max_images_per_chapter, manga_limit: self.manga_limit as usize, job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)), metadata_max_consecutive_failures: self.metadata_max_consecutive_failures, diff --git a/frontend/package.json b/frontend/package.json index 6fb3510..61cbe69 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.93.4", + "version": "0.93.5", "private": true, "type": "module", "scripts": {