fix: enforce the analysis image cap while reading, not after

The OCR and vision dispatchers read the whole page image into a Vec via
storage.get() and only then checked max_image_bytes, so the cap couldn't
bound the read. Stream via get_stream() + safety::accumulate_capped so the
read bails as soon as the running total exceeds the cap.

Bump to 0.124.6.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-07 21:10:29 +02:00
parent 46134c8760
commit ce9a727c73
6 changed files with 56 additions and 23 deletions

2
backend/Cargo.lock generated
View File

@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]] [[package]]
name = "mangalord" name = "mangalord"
version = "0.124.5" version = "0.124.6"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"argon2", "argon2",

View File

@@ -1,6 +1,6 @@
[package] [package]
name = "mangalord" name = "mangalord"
version = "0.124.5" version = "0.124.6"
edition = "2021" edition = "2021"
default-run = "mangalord" default-run = "mangalord"

View File

@@ -393,19 +393,20 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher {
// Page was deleted between enqueue and dispatch — nothing to do. // Page was deleted between enqueue and dispatch — nothing to do.
return Ok(()); return Ok(());
}; };
let bytes = self // Stream through the byte cap so an oversized blob is rejected as it's
// read rather than after it's fully buffered into memory (mirrors the
// OCR dispatcher).
let file = self
.storage .storage
.get(&page.storage_key) .get_stream(&page.storage_key)
.await .await
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?; .map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
if bytes.len() > self.max_image_bytes { let bytes =
anyhow::bail!( crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
"page image {} is {} bytes, over the {} cap", .await
page.storage_key, .map_err(|e| {
bytes.len(), anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
self.max_image_bytes })?;
);
}
let analysis = self.vision.analyze(&bytes, &page.content_type).await?; let analysis = self.vision.analyze(&bytes, &page.content_type).await?;
repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?; repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?;
Ok(()) Ok(())

View File

@@ -207,19 +207,21 @@ impl AnalyzeDispatcher for OcrAnalyzeDispatcher {
// Page was deleted between enqueue and dispatch — nothing to do. // Page was deleted between enqueue and dispatch — nothing to do.
return Ok(()); return Ok(());
}; };
let bytes = self // Stream the blob through the byte cap so the read itself bails once
// the running total exceeds `max_image_bytes` — a plain `get()` would
// buffer the whole (possibly huge) image into memory before the cap
// could reject it.
let file = self
.storage .storage
.get(&page.storage_key) .get_stream(&page.storage_key)
.await .await
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?; .map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
if bytes.len() > self.max_image_bytes { let bytes =
anyhow::bail!( crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
"page image {} is {} bytes, over the {} cap", .await
page.storage_key, .map_err(|e| {
bytes.len(), anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
self.max_image_bytes })?;
);
}
// OCR inference is CPU-bound and synchronous — keep it off the async // OCR inference is CPU-bound and synchronous — keep it off the async
// worker's runtime thread, and gate it behind the shared permit pool so // worker's runtime thread, and gate it behind the shared permit pool so
// ANALYSIS_WORKERS > cores can't oversubscribe the blocking pool. // ANALYSIS_WORKERS > cores can't oversubscribe the blocking pool.

View File

@@ -99,6 +99,36 @@ async fn dispatch_persists_ocr_lines_and_search_doc(pool: PgPool) {
assert!(has_doc, "search_doc must be populated from OCR text"); assert!(has_doc, "search_doc must be populated from OCR text");
} }
#[sqlx::test(migrations = "./migrations")]
async fn dispatch_rejects_page_image_over_the_byte_cap(pool: PgPool) {
let dir = TempDir::new().unwrap();
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(dir.path()));
let key = "mangas/x/big.png";
// 4 KiB on disk, 1 KiB cap — the streamed read must bail on the cap
// rather than buffering the whole blob and OCR-ing it.
storage.put(key, &vec![0u8; 4096]).await.unwrap();
let page_id = seed_page(&pool, key).await;
let dispatcher = OcrAnalyzeDispatcher {
db: pool.clone(),
storage: Arc::clone(&storage),
engine: StubOcrEngine::new(&["should not run"]),
max_image_bytes: 1024,
ocr_permits: Arc::new(tokio::sync::Semaphore::new(1)),
};
let err = dispatcher.dispatch(page_id).await.unwrap_err();
assert!(
err.chain().any(|c| c.to_string().contains("cap")),
"expected an over-cap error, got: {err:#}"
);
// A rejected page must not land an analysis row.
assert!(repo::page_analysis::load(&pool, page_id)
.await
.unwrap()
.is_none());
}
#[sqlx::test(migrations = "./migrations")] #[sqlx::test(migrations = "./migrations")]
async fn dispatch_missing_page_is_noop(pool: PgPool) { async fn dispatch_missing_page_is_noop(pool: PgPool) {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();

View File

@@ -1,6 +1,6 @@
{ {
"name": "mangalord-frontend", "name": "mangalord-frontend",
"version": "0.124.5", "version": "0.124.6",
"private": true, "private": true,
"type": "module", "type": "module",
"scripts": { "scripts": {