From ce9a727c73a874254ef8b3b52a3bd1a4469fdbd2 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Tue, 7 Jul 2026 21:10:29 +0200 Subject: [PATCH] fix: enforce the analysis image cap while reading, not after The OCR and vision dispatchers read the whole page image into a Vec via storage.get() and only then checked max_image_bytes, so the cap couldn't bound the read. Stream via get_stream() + safety::accumulate_capped so the read bails as soon as the running total exceeds the cap. Bump to 0.124.6. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- backend/src/analysis/daemon.rs | 21 +++++++++++---------- backend/src/analysis/ocr.rs | 22 ++++++++++++---------- backend/tests/analysis_ocr.rs | 30 ++++++++++++++++++++++++++++++ frontend/package.json | 2 +- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 80d9aa6..6a04be0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.124.5" +version = "0.124.6" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5e75240..6f122a0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.124.5" +version = "0.124.6" edition = "2021" default-run = "mangalord" diff --git a/backend/src/analysis/daemon.rs b/backend/src/analysis/daemon.rs index b4dc0f6..f889bef 100644 --- a/backend/src/analysis/daemon.rs +++ b/backend/src/analysis/daemon.rs @@ -393,19 +393,20 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher { // Page was deleted between enqueue and dispatch — nothing to do. return Ok(()); }; - let bytes = self + // Stream through the byte cap so an oversized blob is rejected as it's + // read rather than after it's fully buffered into memory (mirrors the + // OCR dispatcher). + let file = self .storage - .get(&page.storage_key) + .get_stream(&page.storage_key) .await .map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?; - if bytes.len() > self.max_image_bytes { - anyhow::bail!( - "page image {} is {} bytes, over the {} cap", - page.storage_key, - bytes.len(), - self.max_image_bytes - ); - } + let bytes = + crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes) + .await + .map_err(|e| { + anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key) + })?; let analysis = self.vision.analyze(&bytes, &page.content_type).await?; repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?; Ok(()) diff --git a/backend/src/analysis/ocr.rs b/backend/src/analysis/ocr.rs index a5f8aff..c89455f 100644 --- a/backend/src/analysis/ocr.rs +++ b/backend/src/analysis/ocr.rs @@ -207,19 +207,21 @@ impl AnalyzeDispatcher for OcrAnalyzeDispatcher { // Page was deleted between enqueue and dispatch — nothing to do. return Ok(()); }; - let bytes = self + // Stream the blob through the byte cap so the read itself bails once + // the running total exceeds `max_image_bytes` — a plain `get()` would + // buffer the whole (possibly huge) image into memory before the cap + // could reject it. + let file = self .storage - .get(&page.storage_key) + .get_stream(&page.storage_key) .await .map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?; - if bytes.len() > self.max_image_bytes { - anyhow::bail!( - "page image {} is {} bytes, over the {} cap", - page.storage_key, - bytes.len(), - self.max_image_bytes - ); - } + let bytes = + crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes) + .await + .map_err(|e| { + anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key) + })?; // OCR inference is CPU-bound and synchronous — keep it off the async // worker's runtime thread, and gate it behind the shared permit pool so // ANALYSIS_WORKERS > cores can't oversubscribe the blocking pool. diff --git a/backend/tests/analysis_ocr.rs b/backend/tests/analysis_ocr.rs index 6a11dfb..5619f51 100644 --- a/backend/tests/analysis_ocr.rs +++ b/backend/tests/analysis_ocr.rs @@ -99,6 +99,36 @@ async fn dispatch_persists_ocr_lines_and_search_doc(pool: PgPool) { assert!(has_doc, "search_doc must be populated from OCR text"); } +#[sqlx::test(migrations = "./migrations")] +async fn dispatch_rejects_page_image_over_the_byte_cap(pool: PgPool) { + let dir = TempDir::new().unwrap(); + let storage: Arc = Arc::new(LocalStorage::new(dir.path())); + let key = "mangas/x/big.png"; + // 4 KiB on disk, 1 KiB cap — the streamed read must bail on the cap + // rather than buffering the whole blob and OCR-ing it. + storage.put(key, &vec![0u8; 4096]).await.unwrap(); + let page_id = seed_page(&pool, key).await; + + let dispatcher = OcrAnalyzeDispatcher { + db: pool.clone(), + storage: Arc::clone(&storage), + engine: StubOcrEngine::new(&["should not run"]), + max_image_bytes: 1024, + ocr_permits: Arc::new(tokio::sync::Semaphore::new(1)), + }; + let err = dispatcher.dispatch(page_id).await.unwrap_err(); + assert!( + err.chain().any(|c| c.to_string().contains("cap")), + "expected an over-cap error, got: {err:#}" + ); + + // A rejected page must not land an analysis row. + assert!(repo::page_analysis::load(&pool, page_id) + .await + .unwrap() + .is_none()); +} + #[sqlx::test(migrations = "./migrations")] async fn dispatch_missing_page_is_noop(pool: PgPool) { let dir = TempDir::new().unwrap(); diff --git a/frontend/package.json b/frontend/package.json index 2fc23cf..9a7d7ca 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.124.5", + "version": "0.124.6", "private": true, "type": "module", "scripts": {