diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1c5dce5..9b9f403 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.87.4" +version = "0.87.5" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9c62fad..f5b9909 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.87.4" +version = "0.87.5" edition = "2021" default-run = "mangalord" diff --git a/backend/src/analysis/vision.rs b/backend/src/analysis/vision.rs index cf2dd87..bda7c09 100644 --- a/backend/src/analysis/vision.rs +++ b/backend/src/analysis/vision.rs @@ -57,6 +57,59 @@ struct SliceParams { max_slices: usize, } +/// Result of the (blocking-pool) prep pass: a self-contained set of byte +/// payloads the async HTTP loop can dispatch without further CPU work. +/// The enum mirrors [`Plan`] but carries actual JPEGs instead of geometry. +enum PreparedAnalysis { + /// `image::load_from_memory` failed — fall back to sending the raw bytes. + Undecodable, + /// One vision call against `jpeg` (the encoded whole page). + Single { jpeg: Vec }, + /// Per-band OCR calls plus a final whole-image grounding call. Each + /// slice tuple is `(y0, y1, encoded_jpeg)`. + Sliced { + slices: Vec<(u32, u32, Vec)>, + whole: Vec, + }, +} + +/// CPU-only image work: decode → plan → optional width-reduce → slice → +/// JPEG encode. Returns `Ok(Undecodable)` (never `Err`) when the bytes +/// don't parse, so the caller can fall through to the raw-pass-through +/// behaviour. +fn prepare_analysis(image: &[u8], params: SliceParams) -> PreparedAnalysis { + let Some(img) = image::load_from_memory(image).ok() else { + return PreparedAnalysis::Undecodable; + }; + match plan_slices(img.width(), img.height(), ¶ms) { + Plan::Single { .. } => match render_whole(&img, params.max_pixels) { + Some(jpeg) => PreparedAnalysis::Single { jpeg }, + None => PreparedAnalysis::Undecodable, + }, + Plan::Sliced { width, height, bands } => { + let work = if width == img.width() && height == img.height() { + img.clone() + } else { + img.resize_exact(width, height, FilterType::Triangle) + }; + let mut slices = Vec::with_capacity(bands.len()); + for (y0, y1) in &bands { + let Some(jpeg) = render_slice(&work, *y0, *y1, params.max_pixels) else { + // A failed slice falls back to the undecodable path — + // the server then gets one combined call instead of N + // broken slice calls. + return PreparedAnalysis::Undecodable; + }; + slices.push((*y0, *y1, jpeg)); + } + let Some(whole) = render_whole(&img, params.max_pixels) else { + return PreparedAnalysis::Undecodable; + }; + PreparedAnalysis::Sliced { slices, whole } + } + } +} + impl VisionClient { pub fn new(http: reqwest::Client, cfg: &AnalysisConfig) -> Self { Self { @@ -84,26 +137,39 @@ impl VisionClient { /// Analyze one page image. `mime` is the stored content type (used only /// for the fallback when the image can't be decoded locally). pub async fn analyze(&self, image: &[u8], mime: &str) -> anyhow::Result { - // Undecodable locally → send the raw bytes in a single combined call - // and let the server cope (preserves the prior behavior). - let Some(img) = image::load_from_memory(image).ok() else { - let url = format!("data:{mime};base64,{}", b64(image)); - let body = build_request_body( - &self.model, - self.max_tokens, - &url, - self.response_format, - self.frequency_penalty, - self.temperature, - &self.system_prompt, - ); - return parse_chat_completion(&self.post_chat(body).await?); + // All the CPU-heavy work — JPEG/PNG decode, optional width reduce, + // per-band slice, JPEG re-encode — runs on the blocking pool so it + // doesn't starve the tokio runtime (axum handlers, SSE streams, + // other daemons share the same threads). A single hop pays the + // overhead once per page; the per-call work stays on the blocking + // worker until the HTTP loop below picks back up. + let prepared = { + let bytes = image.to_vec(); + let params = self.slice; + tokio::task::spawn_blocking(move || prepare_analysis(&bytes, params)) + .await + .map_err(|e| anyhow!("vision prep join: {e}"))? }; - match plan_slices(img.width(), img.height(), &self.slice) { - Plan::Single { .. } => { - let jpeg = render_whole(&img, self.slice.max_pixels) - .ok_or_else(|| anyhow!("failed to encode page image"))?; + match prepared { + PreparedAnalysis::Undecodable => { + // Send raw bytes in a single combined call and let the server + // cope (preserves prior behavior). Base64 encoding still runs + // on the runtime; for an undecodable page this is the + // happy path's exit and stays brief. + let url = format!("data:{mime};base64,{}", b64(image)); + let body = build_request_body( + &self.model, + self.max_tokens, + &url, + self.response_format, + self.frequency_penalty, + self.temperature, + &self.system_prompt, + ); + parse_chat_completion(&self.post_chat(body).await?) + } + PreparedAnalysis::Single { jpeg } => { let body = build_request_body( &self.model, self.max_tokens, @@ -115,25 +181,14 @@ impl VisionClient { ); parse_chat_completion(&self.post_chat(body).await?) } - Plan::Sliced { width, height, bands } => { + PreparedAnalysis::Sliced { slices, whole } => { tracing::debug!( - bands = bands.len(), - width, - height, + bands = slices.len(), "analysis: slicing tall page" ); - // Work image at the (possibly width-reduced) slice space. - let work = if width == img.width() && height == img.height() { - img.clone() - } else { - img.resize_exact(width, height, FilterType::Triangle) - }; - // Pass A: OCR each band (keep its y-range for seam dedup). - let mut slices: Vec = Vec::with_capacity(bands.len()); - for (y0, y1) in &bands { - let jpeg = render_slice(&work, *y0, *y1, self.slice.max_pixels) - .ok_or_else(|| anyhow!("failed to encode page slice"))?; + let mut ocrs: Vec = Vec::with_capacity(slices.len()); + for (y0, y1, jpeg) in &slices { let body = build_ocr_body( &self.model, self.max_tokens, @@ -141,20 +196,18 @@ impl VisionClient { self.frequency_penalty, self.temperature, &self.ocr_prompt, - &data_url(&jpeg), + &data_url(jpeg), ); let parsed = parse_chat_completion(&self.post_chat(body).await?)?; - slices.push(SliceOcr { + ocrs.push(SliceOcr { y0: *y0 as f64, y1: *y1 as f64, pieces: parsed.ocr_results, }); } - let merged = merge_ocr(slices); + let merged = merge_ocr(ocrs); // Pass B: ground tags/scene/safety on the whole image + OCR. - let whole = render_whole(&img, self.slice.max_pixels) - .ok_or_else(|| anyhow!("failed to encode page image"))?; let ocr_text = merged .iter() .map(|o| format!("[{}] {}", o.kind, o.text)) @@ -1043,4 +1096,48 @@ mod tests { let dec = image::load_from_memory(&out).unwrap(); assert_eq!((dec.width(), dec.height()), (100, 100)); } + + fn small_params() -> SliceParams { + SliceParams { + max_pixels: 1_000_000, + min_slice_height: 100, + overlap: 0.05, + tall_threshold: 1.8, + max_slices: 6, + } + } + + #[test] + fn prepare_analysis_single_pages_emit_one_jpeg() { + let jpeg = jpeg_of(200, 200); + match prepare_analysis(&jpeg, small_params()) { + PreparedAnalysis::Single { jpeg } => assert!(!jpeg.is_empty()), + other => panic!("expected Single, got {:?}", std::mem::discriminant(&other)), + } + } + + #[test] + fn prepare_analysis_tall_pages_emit_slice_and_whole_jpegs() { + // Long enough that height > slice_h_budget * tall_threshold — + // with the test params (max_pixels=1M, width=200, threshold=1.8) + // the threshold is 200×5000×1.8 = 9000 px. A 200×10_000 page + // pushes us into Sliced. + let jpeg = jpeg_of(200, 10_000); + match prepare_analysis(&jpeg, small_params()) { + PreparedAnalysis::Sliced { slices, whole } => { + assert!(slices.len() >= 2, "expected at least 2 bands, got {}", slices.len()); + assert!(slices.iter().all(|(_, _, j)| !j.is_empty())); + assert!(!whole.is_empty()); + } + other => panic!("expected Sliced, got {:?}", std::mem::discriminant(&other)), + } + } + + #[test] + fn prepare_analysis_garbage_bytes_yield_undecodable() { + match prepare_analysis(&[0u8, 1, 2, 3], small_params()) { + PreparedAnalysis::Undecodable => {} + other => panic!("expected Undecodable, got {:?}", std::mem::discriminant(&other)), + } + } } diff --git a/frontend/package.json b/frontend/package.json index 6f6893b..6d3764f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.87.4", + "version": "0.87.5", "private": true, "type": "module", "scripts": {