diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b88c55e..bc64197 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.77.1" +version = "0.78.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 263e119..8f004d9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.77.1" +version = "0.78.0" edition = "2021" default-run = "mangalord" diff --git a/backend/src/analysis/prompt.rs b/backend/src/analysis/prompt.rs index 0261172..db8ffc4 100644 --- a/backend/src/analysis/prompt.rs +++ b/backend/src/analysis/prompt.rs @@ -22,8 +22,10 @@ pub const MIN_TAGS: usize = 5; pub const MAX_TAGS: usize = 10; /// Output bounds enforced at sanitize time so one verbose response can't -/// bloat the row or the search document. -pub const MAX_OCR_PIECES: usize = 60; +/// bloat the row or the search document. The OCR cap is generous because a +/// long sliced page legitimately accumulates many text pieces (the tsvector +/// handles it). +pub const MAX_OCR_PIECES: usize = 200; pub const MAX_OCR_TEXT_CHARS: usize = 500; pub const MAX_SCENE_CHARS: usize = 1000; @@ -50,6 +52,81 @@ pub const SYSTEM_PROMPT: &str = concat!( "content. Output must be valid minified JSON and nothing else." ); +// --- Two-pass prompts (long-page slicing) ----------------------------------- + +/// Pass A: OCR-only over a single vertical slice of a tall page. Kept +/// narrow so each slice call is fast and never truncates. +pub const OCR_PROMPT: &str = concat!( + "You are an OCR engine for manga/comic pages. The image is one vertical ", + "slice of a larger page. Transcribe EVERY visible text element verbatim ", + "into ocr_results, each with its kind ", + "(speech|thought|narration|sfx|title|caption), in top-to-bottom order. ", + "If there is no text, return {\"ocr_results\":[]}. Output ONLY one ", + "minified JSON object, no prose, no markdown." +); + +/// Pass B: tags + scene + safety for the whole page, grounded in the merged +/// OCR text (supplied as a separate user message part by the client). +pub const GROUNDING_PROMPT: &str = concat!( + "You are a manga/comic page analyzer. You are given the page image ", + "(possibly downscaled) AND the OCR text already extracted from it. Using ", + "both, return ONLY one minified JSON object with: tagging_results (5-10 ", + "short lowercase content tags — characters, actions, setting, mood, ", + "genre; include explicit/sexual tags when present); scene_description ", + "(one or two sentences on setting, characters, and action, referencing ", + "the dialogue where relevant); safety_flag (is_nsfw boolean + ", + "content_type from sexual|nudity|gore|violence|disturbing, [] if none). ", + "No prose, no markdown." +); + +/// Pass-A schema: `{ ocr_results }` only. +pub fn ocr_json_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "ocr_results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "string" }, + "kind": { "type": "string", "enum": OCR_KINDS } + }, + "required": ["text", "kind"] + } + } + }, + "required": ["ocr_results"] + }) +} + +/// Pass-B schema: `{ tagging_results, scene_description, safety_flag }`. +pub fn grounding_json_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "tagging_results": { "type": "array", "items": { "type": "string" } }, + "scene_description": { "type": "string" }, + "safety_flag": { + "type": "object", + "additionalProperties": false, + "properties": { + "is_nsfw": { "type": "boolean" }, + "content_type": { + "type": "array", + "items": { "type": "string", "enum": CONTENT_WARNINGS } + } + }, + "required": ["is_nsfw", "content_type"] + } + }, + "required": ["tagging_results", "scene_description", "safety_flag"] + }) +} + /// JSON Schema for the analysis output, used in `response_format: /// json_schema` mode (OpenAI structured outputs / LM Studio). Mirrors /// [`crate::domain::page_analysis::VisionAnalysis`]; `additionalProperties: diff --git a/backend/src/analysis/vision.rs b/backend/src/analysis/vision.rs index 8b5fe1d..64420f0 100644 --- a/backend/src/analysis/vision.rs +++ b/backend/src/analysis/vision.rs @@ -1,22 +1,30 @@ -//! The OpenAI-compatible vision HTTP client. +//! The OpenAI-compatible vision HTTP client + the long-page slicing wrapper. //! -//! Flow: downscale the page image to fit the model's context budget → -//! base64 data-URL → chat/completions request with an `image_url` content -//! part → parse `choices[0].message.content` as JSON → sanitize/bound the -//! result. Parsing and sanitization are pure functions so they're tested -//! without a live model; only [`VisionClient::analyze`] does I/O. +//! [`VisionClient::analyze`] decides per page: +//! * **Normal-aspect page** → one combined call (OCR + tags + scene + safety). +//! * **Tall page** → slice it into budget-sized bands at native resolution, +//! OCR each (Pass A), merge with seam-dedup, then one grounding call that +//! gets the whole (downscaled) image **plus** the merged OCR text and +//! returns tags + scene + safety (Pass B). Only OCR is merged. +//! +//! The geometry ([`plan_slices`]), merge ([`merge_ocr`]) and request builders +//! are pure so they're unit-tested without a live model; only the `post_chat` +//! path does I/O. +use std::collections::HashSet; use std::io::Cursor; use anyhow::{anyhow, Context}; use base64::Engine; +use image::imageops::FilterType; +use image::DynamicImage; use serde_json::json; use crate::analysis::prompt::{ self, MAX_OCR_PIECES, MAX_OCR_TEXT_CHARS, MAX_SCENE_CHARS, MAX_TAGS, }; use crate::config::{AnalysisConfig, ResponseFormat}; -use crate::domain::page_analysis::{ContentWarning, VisionAnalysis}; +use crate::domain::page_analysis::{ContentWarning, OcrResult, VisionAnalysis}; /// Vision client built from [`AnalysisConfig`]. Cheap to clone (holds a /// `reqwest::Client`, which is internally `Arc`-backed). @@ -27,8 +35,18 @@ pub struct VisionClient { model: String, api_key: Option, max_tokens: u32, - max_image_dim: u32, response_format: ResponseFormat, + slice: SliceParams, +} + +/// Pure geometry inputs, mirrored from [`AnalysisConfig`]. +#[derive(Clone, Copy, Debug)] +struct SliceParams { + max_pixels: u32, + min_slice_height: u32, + overlap: f64, + tall_threshold: f64, + max_slices: usize, } impl VisionClient { @@ -39,28 +57,102 @@ impl VisionClient { model: cfg.model.clone(), api_key: cfg.api_key.clone(), max_tokens: cfg.max_tokens, - max_image_dim: cfg.max_image_dim, response_format: cfg.response_format, + slice: SliceParams { + max_pixels: cfg.max_pixels, + min_slice_height: cfg.min_slice_height, + overlap: cfg.slice_overlap, + tall_threshold: cfg.tall_aspect_threshold, + max_slices: cfg.max_slices, + }, } } - /// Analyze one page image. `mime` is the stored content type (used for - /// the data-URL when no downscale happens). + /// Analyze one page image. `mime` is the stored content type (used only + /// for the fallback when the image can't be decoded locally). pub async fn analyze(&self, image: &[u8], mime: &str) -> anyhow::Result { - let (bytes, mime) = match downscale(image, self.max_image_dim) { - Some((b, m)) => (b, m.to_string()), - None => (image.to_vec(), mime.to_string()), + // Undecodable locally → send the raw bytes in a single combined call + // and let the server cope (preserves the prior behavior). + let Some(img) = image::load_from_memory(image).ok() else { + let url = format!("data:{mime};base64,{}", b64(image)); + let body = + build_request_body(&self.model, self.max_tokens, &url, self.response_format); + return parse_chat_completion(&self.post_chat(body).await?); }; - let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes); - let data_url = format!("data:{mime};base64,{b64}"); - let body = build_request_body( - &self.model, - self.max_tokens, - &data_url, - self.response_format, - ); + match plan_slices(img.width(), img.height(), &self.slice) { + Plan::Single { .. } => { + let jpeg = render_whole(&img, self.slice.max_pixels) + .ok_or_else(|| anyhow!("failed to encode page image"))?; + let body = build_request_body( + &self.model, + self.max_tokens, + &data_url(&jpeg), + self.response_format, + ); + parse_chat_completion(&self.post_chat(body).await?) + } + Plan::Sliced { width, height, bands } => { + tracing::debug!( + bands = bands.len(), + width, + height, + "analysis: slicing tall page" + ); + // Work image at the (possibly width-reduced) slice space. + let work = if width == img.width() && height == img.height() { + img.clone() + } else { + img.resize_exact(width, height, FilterType::Triangle) + }; + // Pass A: OCR each band. + let mut slices: Vec> = Vec::with_capacity(bands.len()); + for (y0, y1) in &bands { + let jpeg = render_slice(&work, *y0, *y1, self.slice.max_pixels) + .ok_or_else(|| anyhow!("failed to encode page slice"))?; + let body = build_ocr_body( + &self.model, + self.max_tokens, + self.response_format, + &data_url(&jpeg), + ); + let parsed = parse_chat_completion(&self.post_chat(body).await?)?; + slices.push(parsed.ocr_results); + } + let merged = merge_ocr(slices); + + // Pass B: ground tags/scene/safety on the whole image + OCR. + let whole = render_whole(&img, self.slice.max_pixels) + .ok_or_else(|| anyhow!("failed to encode page image"))?; + let ocr_text = merged + .iter() + .map(|o| format!("[{}] {}", o.kind, o.text)) + .collect::>() + .join("\n"); + let body = build_grounding_body( + &self.model, + self.max_tokens, + self.response_format, + &data_url(&whole), + &ocr_text, + ); + let g = parse_chat_completion(&self.post_chat(body).await?)?; + + Ok(sanitize(VisionAnalysis { + ocr_results: merged, + tagging_results: g.tagging_results, + scene_description: g.scene_description, + safety_flag: g.safety_flag, + })) + } + } + } + + /// POST a chat/completions body and return the parsed JSON value, + /// surfacing the server's error body on a non-2xx (different + /// OpenAI-compatible servers reject different fields). + async fn post_chat(&self, body: serde_json::Value) -> anyhow::Result { let mut req = self.http.post(&self.endpoint).json(&body); if let Some(key) = &self.api_key { req = req.bearer_auth(key); @@ -68,39 +160,310 @@ impl VisionClient { let resp = req.send().await.context("vision request failed")?; let status = resp.status(); if !status.is_success() { - // Surface the server's error body — different OpenAI-compatible - // servers reject different fields (e.g. LM Studio rejects - // response_format=json_object), and the body says which. let body = resp.text().await.unwrap_or_default(); anyhow::bail!( "vision endpoint returned {status}: {}", body.trim().chars().take(800).collect::() ); } - let value: serde_json::Value = - resp.json().await.context("vision response was not JSON")?; - parse_chat_completion(&value) + resp.json().await.context("vision response was not JSON") } } -/// Build the chat/completions request body. Pure so the per-format shape is -/// unit-tested without a live server. The image is sent as an `image_url` -/// content part with a base64 data URL (OpenAI vision convention). +// --------------------------------------------------------------------------- +// Geometry +// --------------------------------------------------------------------------- + +#[derive(Debug, PartialEq, Eq)] +enum Plan { + /// Whole-page single call (dims are the working dims, informational). + Single { width: u32, height: u32 }, + /// Slice the page into `bands` (y0,y1) over a `width × height` working + /// image; `width` is reduced from native only for pathologically wide + /// pages. + Sliced { + width: u32, + height: u32, + bands: Vec<(u32, u32)>, + }, +} + +/// Plan how to feed a `w×h` page to the model: one call, or N budget-sized +/// vertical slices at native resolution (portrait *or* landscape). See the +/// module/plan docs for the rules. +fn plan_slices(w: u32, h: u32, p: &SliceParams) -> Plan { + if w == 0 || h == 0 { + return Plan::Single { width: w, height: h }; + } + let budget = p.max_pixels.max(1) as f64; + let overlap = p.overlap.clamp(0.0, 0.9); + + // Aspect guard: a page wider than this would yield hair-thin strips, so + // (and only then) reduce the working width. + let max_slice_width = (budget / p.min_slice_height.max(1) as f64).floor().max(1.0); + let (work_w, work_h) = if (w as f64) > max_slice_width { + let s = max_slice_width / w as f64; + ( + max_slice_width as u32, + ((h as f64) * s).round().max(1.0) as u32, + ) + } else { + (w, h) + }; + + // Budget-sized slice height for this width. + let slice_h_budget = (budget / work_w as f64).floor().clamp(1.0, work_h as f64); + if (work_h as f64) <= slice_h_budget * p.tall_threshold.max(1.0) { + return Plan::Single { + width: work_w, + height: work_h, + }; + } + + // Would the budget-sized slices exceed the cap? If so, grow the slice + // height so exactly `max_slices` bands cover the page (coarser; the + // oversized crop is downscaled to budget by `render_slice`). + let step_for = |sh: f64| (sh * (1.0 - overlap)).max(1.0); + let bands_for = |sh: f64| (((work_h as f64 - sh) / step_for(sh)).ceil() as i64 + 1).max(1); + let slice_h = if bands_for(slice_h_budget) > p.max_slices as i64 { + (work_h as f64 / (1.0 + (p.max_slices as f64 - 1.0) * (1.0 - overlap))) + .ceil() + .clamp(1.0, work_h as f64) + } else { + slice_h_budget + } as u32; + + let step = ((slice_h as f64) * (1.0 - overlap)).round().max(1.0) as u32; + let mut bands: Vec<(u32, u32)> = Vec::new(); + let mut y0 = 0u32; + loop { + let y1 = (y0 + slice_h).min(work_h); + bands.push((y0, y1)); + if y1 >= work_h || bands.len() >= p.max_slices { + break; + } + y0 += step; + } + // Guarantee coverage to the bottom edge. + if let Some(last) = bands.last_mut() { + last.1 = work_h; + } + Plan::Sliced { + width: work_w, + height: work_h, + bands, + } +} + +// --------------------------------------------------------------------------- +// Image rendering +// --------------------------------------------------------------------------- + +/// Resize `img` down to `max_pixels` (preserving aspect) only if it exceeds +/// the budget, then JPEG-encode. An image already within budget is encoded +/// at native resolution. +fn render_whole(img: &DynamicImage, max_pixels: u32) -> Option> { + let pixels = img.width() as u64 * img.height() as u64; + let out = if pixels > max_pixels as u64 && pixels > 0 { + let s = (max_pixels as f64 / pixels as f64).sqrt(); + let nw = ((img.width() as f64 * s).round() as u32).max(1); + let nh = ((img.height() as f64 * s).round() as u32).max(1); + img.resize_exact(nw, nh, FilterType::Triangle) + } else { + img.clone() + }; + encode_jpeg(&out) +} + +/// Crop a vertical band `[y0, y1)` from the working image and encode it, +/// downscaling to budget only if the band is oversized (the coarse fallback). +fn render_slice(work: &DynamicImage, y0: u32, y1: u32, max_pixels: u32) -> Option> { + let h = y1.saturating_sub(y0).max(1).min(work.height().saturating_sub(y0).max(1)); + let crop = work.crop_imm(0, y0, work.width(), h); + render_whole(&crop, max_pixels) +} + +fn encode_jpeg(img: &DynamicImage) -> Option> { + let mut buf = Vec::new(); + img.to_rgb8() + .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Jpeg) + .ok()?; + Some(buf) +} + +fn b64(bytes: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +fn data_url(jpeg: &[u8]) -> String { + format!("data:image/jpeg;base64,{}", b64(jpeg)) +} + +// --------------------------------------------------------------------------- +// OCR merge (seam dedup) +// --------------------------------------------------------------------------- + +/// How many pieces at each seam edge to consider for dedup. +const SEAM: usize = 8; + +/// Merge per-slice OCR (top-to-bottom) into one ordered list, de-duplicating +/// text that appears in both slices' overlap region. Dedup is *seam-scoped* +/// (only the first `SEAM` pieces of a slice vs. the last `SEAM` already +/// emitted), so a genuinely repeated line in non-adjacent slices is kept. +/// When two pieces match, the longer (more complete) transcription wins. +fn merge_ocr(slices: Vec>) -> Vec { + let mut out: Vec = Vec::new(); + // Index in `out` where the *previous* slice's pieces began — dedup + // compares only against that slice's tail, never earlier ones. + let mut prev_slice_start = 0usize; + for (si, slice) in slices.into_iter().enumerate() { + let cur_slice_start = out.len(); + for (pi, piece) in slice.into_iter().enumerate() { + if si > 0 && pi < SEAM { + let win_start = cur_slice_start.saturating_sub(SEAM).max(prev_slice_start); + let np = normalize_text(&piece.text); + if !np.is_empty() { + if let Some(j) = (win_start..cur_slice_start) + .find(|&j| similar(&normalize_text(&out[j].text), &np)) + { + if piece.text.chars().count() > out[j].text.chars().count() { + out[j] = piece; + } + continue; + } + } + } + out.push(piece); + } + prev_slice_start = cur_slice_start; + } + out +} + +/// Lowercased, alphanumeric-only, whitespace-collapsed form for comparison. +fn normalize_text(s: &str) -> String { + let mut t = String::with_capacity(s.len()); + for c in s.chars() { + if c.is_alphanumeric() { + t.extend(c.to_lowercase()); + } else { + t.push(' '); + } + } + t.split_whitespace().collect::>().join(" ") +} + +/// Heuristic similarity over already-normalized strings: equal, containment +/// (one inside the other, ≥3 chars), or word-set Jaccard ≥ 0.6. +fn similar(a: &str, b: &str) -> bool { + if a == b { + return true; + } + if a.is_empty() || b.is_empty() { + return false; + } + let (short, long) = if a.len() <= b.len() { (a, b) } else { (b, a) }; + if short.len() >= 3 && long.contains(short) { + return true; + } + let wa: HashSet<&str> = a.split_whitespace().collect(); + let wb: HashSet<&str> = b.split_whitespace().collect(); + if wa.is_empty() || wb.is_empty() { + return false; + } + let inter = wa.intersection(&wb).count() as f64; + let union = wa.union(&wb).count() as f64; + inter / union >= 0.6 +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +/// Combined single-call body (OCR + tags + scene + safety). Public + this +/// exact signature so the existing tests keep working. pub fn build_request_body( model: &str, max_tokens: u32, data_url: &str, response_format: ResponseFormat, ) -> serde_json::Value { + build_body( + model, + max_tokens, + response_format, + prompt::SYSTEM_PROMPT, + "page_analysis", + prompt::output_json_schema(), + data_url, + None, + ) +} + +/// Pass-A OCR-only body. +fn build_ocr_body( + model: &str, + max_tokens: u32, + response_format: ResponseFormat, + data_url: &str, +) -> serde_json::Value { + build_body( + model, + max_tokens, + response_format, + prompt::OCR_PROMPT, + "page_ocr", + prompt::ocr_json_schema(), + data_url, + None, + ) +} + +/// Pass-B grounding body — whole image + the merged OCR text as context. +fn build_grounding_body( + model: &str, + max_tokens: u32, + response_format: ResponseFormat, + data_url: &str, + ocr_text: &str, +) -> serde_json::Value { + let context = format!("OCR text extracted from this page:\n{ocr_text}"); + build_body( + model, + max_tokens, + response_format, + prompt::GROUNDING_PROMPT, + "page_grounding", + prompt::grounding_json_schema(), + data_url, + Some(&context), + ) +} + +/// Build a chat/completions body with a system prompt, an image part, an +/// optional trailing text part, and the requested `response_format`. +fn build_body( + model: &str, + max_tokens: u32, + response_format: ResponseFormat, + system_prompt: &str, + schema_name: &str, + schema: serde_json::Value, + data_url: &str, + user_text: Option<&str>, +) -> serde_json::Value { + let mut user_content = + vec![json!({ "type": "image_url", "image_url": { "url": data_url } })]; + if let Some(text) = user_text { + user_content.push(json!({ "type": "text", "text": text })); + } let mut body = json!({ "model": model, "temperature": 0, "max_tokens": max_tokens, "messages": [ - { "role": "system", "content": prompt::SYSTEM_PROMPT }, - { "role": "user", "content": [ - { "type": "image_url", "image_url": { "url": data_url } } - ]} + { "role": "system", "content": system_prompt }, + { "role": "user", "content": user_content } ] }); let rf = match response_format { @@ -108,11 +471,7 @@ pub fn build_request_body( ResponseFormat::JsonObject => Some(json!({ "type": "json_object" })), ResponseFormat::JsonSchema => Some(json!({ "type": "json_schema", - "json_schema": { - "name": "page_analysis", - "strict": true, - "schema": prompt::output_json_schema() - } + "json_schema": { "name": schema_name, "strict": true, "schema": schema } })), }; if let Some(rf) = rf { @@ -121,25 +480,13 @@ pub fn build_request_body( body } -/// Downscale `bytes` so its longest edge is `max_dim`, re-encoding as JPEG. -/// Returns `None` (use the original bytes) when the image already fits or -/// can't be decoded — a non-decodable page is passed through untouched and -/// the model server can reject it if it wants. -fn downscale(bytes: &[u8], max_dim: u32) -> Option<(Vec, &'static str)> { - let img = image::load_from_memory(bytes).ok()?; - if img.width().max(img.height()) <= max_dim { - return None; - } - let scaled = img.resize(max_dim, max_dim, image::imageops::FilterType::Triangle); - let mut buf = Vec::new(); - scaled - .to_rgb8() - .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Jpeg) - .ok()?; - Some((buf, "image/jpeg")) -} +// --------------------------------------------------------------------------- +// Response parsing / sanitizing +// --------------------------------------------------------------------------- -/// Extract the model's JSON object from a chat/completions response. +/// Extract and parse the model's JSON object from a chat/completions +/// response into a (sanitized) [`VisionAnalysis`]. Lenient field defaults +/// let the same parser serve the combined, OCR-only and grounding shapes. pub fn parse_chat_completion(value: &serde_json::Value) -> anyhow::Result { let content = value .get("choices") @@ -155,9 +502,7 @@ pub fn parse_chat_completion(value: &serde_json::Value) -> anyhow::Result &str { match (s.find('{'), s.rfind('}')) { (Some(start), Some(end)) if end >= start => &s[start..=end], @@ -166,10 +511,8 @@ fn extract_json_object(s: &str) -> &str { } /// Bound the model output: drop empty OCR text, truncate over-long fields, -/// clamp tags to [`MAX_TAGS`] via the page-tag normalizer, and keep only -/// recognized content-warning values. Persist applies the same rules as a -/// safety net, but doing it here keeps the row tidy and the worker's logs -/// honest about what it stored. +/// clamp tags via the page-tag normalizer, and keep only recognized +/// content-warning values. pub fn sanitize(mut a: VisionAnalysis) -> VisionAnalysis { a.ocr_results.retain(|r| !r.text.trim().is_empty()); a.ocr_results.truncate(MAX_OCR_PIECES); @@ -177,9 +520,6 @@ pub fn sanitize(mut a: VisionAnalysis) -> VisionAnalysis { r.text = truncate_chars(r.text.trim(), MAX_OCR_TEXT_CHARS); } - // Normalize tags through the shared page-tag rules (lowercase, collapse - // whitespace, reject wildcards/control/invisible chars); drop the - // unmappable, dedup, cap. let mut seen = std::collections::HashSet::new(); let mut tags = Vec::new(); for raw in std::mem::take(&mut a.tagging_results) { @@ -196,7 +536,6 @@ pub fn sanitize(mut a: VisionAnalysis) -> VisionAnalysis { a.scene_description = truncate_chars(a.scene_description.trim(), MAX_SCENE_CHARS); - // Keep only recognized warnings (canonical lowercase), deduped. let mut seen_w = std::collections::HashSet::new(); a.safety_flag.content_type = std::mem::take(&mut a.safety_flag.content_type) .into_iter() @@ -207,8 +546,6 @@ pub fn sanitize(mut a: VisionAnalysis) -> VisionAnalysis { a } -/// Truncate to at most `n` chars (not bytes), avoiding a panic on a -/// multi-byte boundary. fn truncate_chars(s: &str, n: usize) -> String { s.chars().take(n).collect() } @@ -221,6 +558,30 @@ mod tests { json!({ "choices": [ { "message": { "content": content } } ] }) } + fn params() -> SliceParams { + SliceParams { + max_pixels: 1_000_000, + min_slice_height: 640, + overlap: 0.12, + tall_threshold: 1.6, + max_slices: 16, + } + } + + fn ocr(text: &str, kind: &str) -> OcrResult { + OcrResult { + text: text.into(), + kind: kind.into(), + } + } + + fn jpeg_of(w: u32, h: u32) -> Vec { + let img = DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb([9, 9, 9]))); + encode_jpeg(&img).unwrap() + } + + // --- parse / sanitize (unchanged behavior) --- + #[test] fn parses_a_clean_json_response() { let body = chat_body( @@ -236,48 +597,23 @@ mod tests { } #[test] - fn strips_markdown_fences_and_prose() { - let body = chat_body( - "Here you go:\n```json\n{\"scene_description\":\"x\"}\n```\nthanks", - ); + fn ocr_only_response_parses_with_defaults() { + let body = chat_body(r#"{"ocr_results":[{"text":"Hi","kind":"speech"}]}"#); let v = parse_chat_completion(&body).unwrap(); - assert_eq!(v.scene_description, "x"); + assert_eq!(v.ocr_results.len(), 1); + assert!(v.tagging_results.is_empty()); + assert_eq!(v.scene_description, ""); + } + + #[test] + fn strips_markdown_fences_and_prose() { + let body = chat_body("Here:\n```json\n{\"scene_description\":\"x\"}\n```\nthanks"); + assert_eq!(parse_chat_completion(&body).unwrap().scene_description, "x"); } #[test] fn rejects_non_json_content() { - let body = chat_body("I cannot analyze this image."); - assert!(parse_chat_completion(&body).is_err()); - } - - #[test] - fn rejects_missing_content() { - assert!(parse_chat_completion(&json!({"choices": []})).is_err()); - } - - #[test] - fn sanitize_clamps_tags_and_drops_invalid() { - let body = chat_body( - r#"{"tagging_results":["A","a","b","b","c","d","e","f","g","h","i","j", - "bad%wild"," "]}"#, - ); - let v = parse_chat_completion(&body).unwrap(); - // "A"/"a" dedup to one; wildcard + blank dropped; capped at 10. - assert!(v.tagging_results.len() <= MAX_TAGS); - assert!(v.tagging_results.contains(&"a".to_string())); - assert!(!v.tagging_results.iter().any(|t| t.contains('%'))); - } - - #[test] - fn sanitize_drops_empty_ocr_and_truncates() { - let long = "x".repeat(MAX_OCR_TEXT_CHARS + 50); - let body = chat_body(&format!( - r#"{{"ocr_results":[{{"text":" ","kind":"speech"}}, - {{"text":"{long}","kind":"narration"}}]}}"# - )); - let v = parse_chat_completion(&body).unwrap(); - assert_eq!(v.ocr_results.len(), 1, "blank OCR dropped"); - assert_eq!(v.ocr_results[0].text.chars().count(), MAX_OCR_TEXT_CHARS); + assert!(parse_chat_completion(&chat_body("nope")).is_err()); } #[test] @@ -289,46 +625,40 @@ mod tests { assert_eq!(v.safety_flag.content_type, vec!["sexual", "gore"]); } - #[test] - fn downscale_passes_through_small_images() { - // 8x8 PNG — well under any sane max_dim, so no re-encode. - let img = image::RgbImage::from_pixel(8, 8, image::Rgb([10, 20, 30])); - let mut buf = Vec::new(); - image::DynamicImage::ImageRgb8(img) - .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Png) - .unwrap(); - assert!(downscale(&buf, 1024).is_none()); - } + // --- request builders --- #[test] - fn downscale_shrinks_large_images_to_max_dim() { - let img = image::RgbImage::from_pixel(2000, 1000, image::Rgb([1, 2, 3])); - let mut buf = Vec::new(); - image::DynamicImage::ImageRgb8(img) - .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Png) - .unwrap(); - let (out, mime) = downscale(&buf, 512).expect("should downscale"); - assert_eq!(mime, "image/jpeg"); - let decoded = image::load_from_memory(&out).unwrap(); - assert_eq!(decoded.width().max(decoded.height()), 512); - } - - #[test] - fn downscale_passes_through_undecodable_bytes() { - assert!(downscale(b"not an image", 512).is_none()); - } - - #[test] - fn request_body_carries_model_image_and_prompt() { - let body = build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::None); - assert_eq!(body["model"], "m"); - assert_eq!(body["max_tokens"], 100); - // System prompt + a user image_url part. + fn combined_body_uses_full_schema_and_image() { + let body = build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::JsonSchema); assert_eq!(body["messages"][0]["role"], "system"); assert_eq!( body["messages"][1]["content"][0]["image_url"]["url"], "data:image/png;base64,AA" ); + assert_eq!(body["response_format"]["json_schema"]["name"], "page_analysis"); + let req = &body["response_format"]["json_schema"]["schema"]["required"]; + assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag")); + } + + #[test] + fn ocr_body_carries_only_the_ocr_schema() { + let body = build_ocr_body("m", 100, ResponseFormat::JsonSchema, "d"); + assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr"); + let req = &body["response_format"]["json_schema"]["schema"]["required"]; + assert_eq!(req, &json!(["ocr_results"])); + // No user text part — just the image. + assert_eq!(body["messages"][1]["content"].as_array().unwrap().len(), 1); + } + + #[test] + fn grounding_body_includes_ocr_context_and_grounding_schema() { + let body = build_grounding_body("m", 100, ResponseFormat::JsonSchema, "d", "[speech] Hi"); + assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding"); + let parts = body["messages"][1]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 2, "image + text parts"); + assert_eq!(parts[0]["type"], "image_url"); + assert_eq!(parts[1]["type"], "text"); + assert!(parts[1]["text"].as_str().unwrap().contains("[speech] Hi")); } #[test] @@ -337,20 +667,142 @@ mod tests { assert!(body.get("response_format").is_none()); } + // --- geometry: plan_slices --- + #[test] - fn response_format_json_object_sets_type() { - let body = build_request_body("m", 100, "d", ResponseFormat::JsonObject); - assert_eq!(body["response_format"]["type"], "json_object"); + fn normal_aspect_page_is_a_single_call() { + assert!(matches!( + plan_slices(800, 1000, ¶ms()), + Plan::Single { .. } + )); } #[test] - fn response_format_json_schema_embeds_the_strict_schema() { - let body = build_request_body("m", 100, "d", ResponseFormat::JsonSchema); - assert_eq!(body["response_format"]["type"], "json_schema"); - assert_eq!(body["response_format"]["json_schema"]["strict"], true); - // The schema is the real analysis schema (top-level required keys). - let required = - &body["response_format"]["json_schema"]["schema"]["required"]; - assert!(required.as_array().unwrap().iter().any(|v| v == "safety_flag")); + fn tall_narrow_page_slices_into_portrait_bands() { + let p = params(); + let Plan::Sliced { width, height, bands } = plan_slices(800, 8000, &p) else { + panic!("expected Sliced"); + }; + assert_eq!((width, height), (800, 8000)); + assert!(bands.len() >= 2 && bands.len() <= p.max_slices); + assert_eq!(bands.first().unwrap().0, 0, "starts at top"); + assert_eq!(bands.last().unwrap().1, 8000, "covers the bottom"); + // Overlapping + advancing bands; each ≈ the pixel budget. + for win in bands.windows(2) { + assert!(win[1].0 < win[0].1, "adjacent bands overlap"); + assert!(win[1].0 > win[0].0, "bands advance"); + } + for (y0, y1) in &bands { + let px = (y1 - y0) as u64 * width as u64; + assert!(px <= 1_000_000 + 1, "band within budget, got {px}"); + } + } + + #[test] + fn wide_page_yields_landscape_bands() { + // 1400 < maxSliceWidth(1562) so width is preserved; budget/1400≈714 + // → bands are wider than tall. + let Plan::Sliced { width, bands, .. } = plan_slices(1400, 4000, ¶ms()) else { + panic!("expected Sliced"); + }; + assert_eq!(width, 1400); + for (y0, y1) in &bands { + assert!(y1 - y0 < width, "landscape band (height < width)"); + } + } + + #[test] + fn pathologically_wide_page_reduces_width() { + let p = params(); + let max_slice_width = (p.max_pixels as f64 / p.min_slice_height as f64).floor() as u32; + let Plan::Sliced { width, .. } = plan_slices(3000, 9000, &p) else { + panic!("expected Sliced"); + }; + assert!(width <= max_slice_width && width < 3000, "width reduced to the guard"); + } + + #[test] + fn very_long_page_is_capped_at_max_slices() { + let p = params(); + let Plan::Sliced { bands, height, .. } = plan_slices(800, 200_000, &p) else { + panic!("expected Sliced"); + }; + assert!(bands.len() <= p.max_slices); + assert_eq!(bands.last().unwrap().1, height, "still covers the bottom"); + } + + // --- merge_ocr --- + + #[test] + fn merge_dedups_a_line_repeated_across_the_seam() { + let merged = merge_ocr(vec![ + vec![ocr("Hello", "speech"), ocr("world", "speech")], + vec![ocr("world", "speech"), ocr("bye", "speech")], + ]); + let texts: Vec<&str> = merged.iter().map(|o| o.text.as_str()).collect(); + assert_eq!(texts, vec!["Hello", "world", "bye"]); + } + + #[test] + fn merge_keeps_the_longer_transcription() { + let merged = merge_ocr(vec![ + vec![ocr("I don", "speech")], + vec![ocr("I don't think so", "speech")], + ]); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "I don't think so"); + } + + #[test] + fn merge_keeps_non_adjacent_repeats() { + // "boom" in slice 0 and slice 2 must both survive (not a seam dup). + let merged = merge_ocr(vec![ + vec![ocr("boom", "sfx")], + vec![ocr("crash", "sfx")], + vec![ocr("boom", "sfx")], + ]); + let booms = merged.iter().filter(|o| o.text == "boom").count(); + assert_eq!(booms, 2); + } + + #[test] + fn merge_preserves_distinct_lines_and_order() { + let merged = merge_ocr(vec![ + vec![ocr("one", "narration")], + vec![ocr("two", "speech")], + ]); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].kind, "narration"); + assert_eq!(merged[1].text, "two"); + } + + // --- rendering --- + + #[test] + fn render_whole_passes_small_images_through() { + let jpeg = jpeg_of(100, 80); + let img = image::load_from_memory(&jpeg).unwrap(); + let out = render_whole(&img, 1_000_000).unwrap(); + let dec = image::load_from_memory(&out).unwrap(); + assert_eq!((dec.width(), dec.height()), (100, 80)); + } + + #[test] + fn render_whole_fits_oversized_to_budget() { + let jpeg = jpeg_of(2000, 1000); // 2.0 MP + let img = image::load_from_memory(&jpeg).unwrap(); + let out = render_whole(&img, 500_000).unwrap(); + let dec = image::load_from_memory(&out).unwrap(); + let px = dec.width() as u64 * dec.height() as u64; + assert!(px <= 520_000, "fit to ~budget, got {px}"); + } + + #[test] + fn render_slice_crops_the_band() { + let jpeg = jpeg_of(100, 400); + let img = image::load_from_memory(&jpeg).unwrap(); + let out = render_slice(&img, 50, 150, 1_000_000).unwrap(); + let dec = image::load_from_memory(&out).unwrap(); + assert_eq!((dec.width(), dec.height()), (100, 100)); } } diff --git a/backend/src/config.rs b/backend/src/config.rs index f90b245..f66e89c 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -118,8 +118,23 @@ pub struct AnalysisConfig { /// length`) and the parse fails. The page image + prompt are only a few /// hundred tokens, so a large output budget still fits an 8k window. pub max_tokens: u32, - /// Longest image edge (px) before downscaling (`ANALYSIS_MAX_IMAGE_DIM`). - pub max_image_dim: u32, + /// Per-slice / per-image pixel budget (`ANALYSIS_MAX_PIXELS`). The model + /// resizes any input to roughly this many pixels anyway, so we slice/fit + /// to it at native resolution rather than pre-squashing by a fixed edge. + pub max_pixels: u32, + /// Minimum slice height (px), the aspect guard for very wide pages + /// (`ANALYSIS_MIN_SLICE_HEIGHT`). Implies `max_slice_width = + /// max_pixels / min_slice_height`. + pub min_slice_height: u32, + /// Vertical overlap between adjacent slices as a fraction of slice height + /// (`ANALYSIS_SLICE_OVERLAP`), so text straddling a cut survives. + pub slice_overlap: f64, + /// Slice only when `height/width` exceeds this (`ANALYSIS_TALL_ASPECT`); + /// normal-aspect pages take a single combined call. + pub tall_aspect_threshold: f64, + /// Hard cap on slices per page (`ANALYSIS_MAX_SLICES`); beyond it slices + /// grow coarser (and get downscaled to budget) rather than multiplying. + pub max_slices: usize, /// Hard cap on a page image's stored size; larger pages are skipped /// (`ANALYSIS_MAX_IMAGE_BYTES`). pub max_image_bytes: usize, @@ -137,9 +152,16 @@ impl Default for AnalysisConfig { model: String::new(), api_key: None, request_timeout: Duration::from_secs(120), - job_timeout: Duration::from_secs(180), + // A sliced long page is N sequential calls under one job, so the + // whole-job budget must be generous (request_timeout stays + // per-call). + job_timeout: Duration::from_secs(600), max_tokens: 4096, - max_image_dim: 1024, + max_pixels: 1_000_000, + min_slice_height: 640, + slice_overlap: 0.12, + tall_aspect_threshold: 1.6, + max_slices: 16, max_image_bytes: 8 * 1024 * 1024, response_format: ResponseFormat::JsonSchema, } @@ -166,7 +188,13 @@ impl AnalysisConfig { d.job_timeout.as_secs(), )), max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32, - max_image_dim: env_u64("ANALYSIS_MAX_IMAGE_DIM", d.max_image_dim as u64) as u32, + max_pixels: env_u64("ANALYSIS_MAX_PIXELS", d.max_pixels as u64) as u32, + min_slice_height: env_u64("ANALYSIS_MIN_SLICE_HEIGHT", d.min_slice_height as u64) + .max(1) as u32, + slice_overlap: env_f64("ANALYSIS_SLICE_OVERLAP", d.slice_overlap).clamp(0.0, 0.9), + tall_aspect_threshold: env_f64("ANALYSIS_TALL_ASPECT", d.tall_aspect_threshold) + .max(1.0), + max_slices: env_usize("ANALYSIS_MAX_SLICES", d.max_slices).max(1), max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes), response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT") .map(|s| ResponseFormat::from_str(&s)) @@ -506,6 +534,13 @@ fn env_i64(name: &str, default: i64) -> i64 { .unwrap_or(default) } +fn env_f64(name: &str, default: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(default) +} + fn env_usize(name: &str, default: usize) -> usize { std::env::var(name) .ok() @@ -578,7 +613,11 @@ mod tests { "ANALYSIS_MODEL", "ANALYSIS_API_KEY", "ANALYSIS_MAX_TOKENS", - "ANALYSIS_MAX_IMAGE_DIM", + "ANALYSIS_MAX_PIXELS", + "ANALYSIS_MIN_SLICE_HEIGHT", + "ANALYSIS_SLICE_OVERLAP", + "ANALYSIS_TALL_ASPECT", + "ANALYSIS_MAX_SLICES", "ANALYSIS_RESPONSE_FORMAT", ] { std::env::remove_var(k); @@ -586,7 +625,10 @@ mod tests { let cfg = AnalysisConfig::from_env(); assert!(!cfg.enabled); assert_eq!(cfg.workers, 1); - assert_eq!(cfg.max_image_dim, 1024); + assert_eq!(cfg.max_pixels, 1_000_000); + assert_eq!(cfg.min_slice_height, 640); + assert_eq!(cfg.max_slices, 16); + assert_eq!(cfg.tall_aspect_threshold, 1.6); assert!(cfg.api_key.is_none()); assert_eq!(cfg.response_format, ResponseFormat::JsonSchema); } @@ -614,14 +656,18 @@ mod tests { std::env::set_var("ANALYSIS_WORKERS", "4"); std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat"); std::env::set_var("ANALYSIS_MODEL", "qwen2-vl"); - std::env::set_var("ANALYSIS_MAX_IMAGE_DIM", "768"); + std::env::set_var("ANALYSIS_MAX_PIXELS", "768000"); + std::env::set_var("ANALYSIS_MAX_SLICES", "8"); + std::env::set_var("ANALYSIS_SLICE_OVERLAP", "0.2"); let cfg = AnalysisConfig::from_env(); for k in [ "ANALYSIS_ENABLED", "ANALYSIS_WORKERS", "ANALYSIS_VISION_URL", "ANALYSIS_MODEL", - "ANALYSIS_MAX_IMAGE_DIM", + "ANALYSIS_MAX_PIXELS", + "ANALYSIS_MAX_SLICES", + "ANALYSIS_SLICE_OVERLAP", ] { std::env::remove_var(k); } @@ -629,7 +675,9 @@ mod tests { assert_eq!(cfg.workers, 4); assert_eq!(cfg.endpoint, "http://vis/v1/chat"); assert_eq!(cfg.model, "qwen2-vl"); - assert_eq!(cfg.max_image_dim, 768); + assert_eq!(cfg.max_pixels, 768_000); + assert_eq!(cfg.max_slices, 8); + assert_eq!(cfg.slice_overlap, 0.2); } #[test] diff --git a/frontend/package.json b/frontend/package.json index 8b02f25..f0bbea5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.77.1", + "version": "0.78.0", "private": true, "type": "module", "scripts": {