fix: enforce the analysis image cap while reading, not after

The OCR and vision dispatchers read the whole page image into a Vec via
storage.get() and only then checked max_image_bytes, so the cap couldn't
bound the read. Stream via get_stream() + safety::accumulate_capped so the
read bails as soon as the running total exceeds the cap.

Bump to 0.124.6.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-07 21:10:29 +02:00
parent 46134c8760
commit ce9a727c73
6 changed files with 56 additions and 23 deletions

View File

@@ -393,19 +393,20 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher {
// Page was deleted between enqueue and dispatch — nothing to do.
return Ok(());
};
let bytes = self
// Stream through the byte cap so an oversized blob is rejected as it's
// read rather than after it's fully buffered into memory (mirrors the
// OCR dispatcher).
let file = self
.storage
.get(&page.storage_key)
.get_stream(&page.storage_key)
.await
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
if bytes.len() > self.max_image_bytes {
anyhow::bail!(
"page image {} is {} bytes, over the {} cap",
page.storage_key,
bytes.len(),
self.max_image_bytes
);
}
let bytes =
crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
.await
.map_err(|e| {
anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
})?;
let analysis = self.vision.analyze(&bytes, &page.content_type).await?;
repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?;
Ok(())

View File

@@ -207,19 +207,21 @@ impl AnalyzeDispatcher for OcrAnalyzeDispatcher {
// Page was deleted between enqueue and dispatch — nothing to do.
return Ok(());
};
let bytes = self
// Stream the blob through the byte cap so the read itself bails once
// the running total exceeds `max_image_bytes` — a plain `get()` would
// buffer the whole (possibly huge) image into memory before the cap
// could reject it.
let file = self
.storage
.get(&page.storage_key)
.get_stream(&page.storage_key)
.await
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
if bytes.len() > self.max_image_bytes {
anyhow::bail!(
"page image {} is {} bytes, over the {} cap",
page.storage_key,
bytes.len(),
self.max_image_bytes
);
}
let bytes =
crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
.await
.map_err(|e| {
anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
})?;
// OCR inference is CPU-bound and synchronous — keep it off the async
// worker's runtime thread, and gate it behind the shared permit pool so
// ANALYSIS_WORKERS > cores can't oversubscribe the blocking pool.