diff --git a/backend/Cargo.lock b/backend/Cargo.lock index bc64197..f59fcb5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.78.0" +version = "0.79.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8f004d9..125c279 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.78.0" +version = "0.79.0" edition = "2021" default-run = "mangalord" diff --git a/backend/src/analysis/prompt.rs b/backend/src/analysis/prompt.rs index db8ffc4..8fdbdbb 100644 --- a/backend/src/analysis/prompt.rs +++ b/backend/src/analysis/prompt.rs @@ -60,9 +60,11 @@ pub const OCR_PROMPT: &str = concat!( "You are an OCR engine for manga/comic pages. The image is one vertical ", "slice of a larger page. Transcribe EVERY visible text element verbatim ", "into ocr_results, each with its kind ", - "(speech|thought|narration|sfx|title|caption), in top-to-bottom order. ", - "If there is no text, return {\"ocr_results\":[]}. Output ONLY one ", - "minified JSON object, no prose, no markdown." + "(speech|thought|narration|sfx|title|caption) and y — the vertical center ", + "of the text as a fraction from 0.0 (top) to 1.0 (bottom) of THIS image. ", + "List them in top-to-bottom order. If there is no text, return ", + "{\"ocr_results\":[]}. Output ONLY one minified JSON object, no prose, no ", + "markdown." ); /// Pass B: tags + scene + safety for the whole page, grounded in the merged @@ -92,9 +94,10 @@ pub fn ocr_json_schema() -> serde_json::Value { "additionalProperties": false, "properties": { "text": { "type": "string" }, - "kind": { "type": "string", "enum": OCR_KINDS } + "kind": { "type": "string", "enum": OCR_KINDS }, + "y": { "type": "number", "minimum": 0, "maximum": 1 } }, - "required": ["text", "kind"] + "required": ["text", "kind", "y"] } } }, diff --git a/backend/src/analysis/vision.rs b/backend/src/analysis/vision.rs index 64420f0..d97ed1d 100644 --- a/backend/src/analysis/vision.rs +++ b/backend/src/analysis/vision.rs @@ -106,8 +106,8 @@ impl VisionClient { img.resize_exact(width, height, FilterType::Triangle) }; - // Pass A: OCR each band. - let mut slices: Vec> = Vec::with_capacity(bands.len()); + // Pass A: OCR each band (keep its y-range for seam dedup). + let mut slices: Vec = Vec::with_capacity(bands.len()); for (y0, y1) in &bands { let jpeg = render_slice(&work, *y0, *y1, self.slice.max_pixels) .ok_or_else(|| anyhow!("failed to encode page slice"))?; @@ -118,7 +118,11 @@ impl VisionClient { &data_url(&jpeg), ); let parsed = parse_chat_completion(&self.post_chat(body).await?)?; - slices.push(parsed.ocr_results); + slices.push(SliceOcr { + y0: *y0 as f64, + y1: *y1 as f64, + pieces: parsed.ocr_results, + }); } let merged = merge_ocr(slices); @@ -306,38 +310,93 @@ fn data_url(jpeg: &[u8]) -> String { /// How many pieces at each seam edge to consider for dedup. const SEAM: usize = 8; +/// One slice's OCR plus the vertical band `[y0, y1)` it covered in the +/// working image — so per-piece `y` fractions map to page-global positions. +pub struct SliceOcr { + pub y0: f64, + pub y1: f64, + pub pieces: Vec, +} + +/// One merged-list entry with the geometry used for dedup decisions. +struct Entry { + piece: OcrResult, + /// Page-global y of the text (working-image space); `None` when the + /// slice's positions weren't usable. + page_y: Option, + /// Center of the slice this piece came from. + center: f64, +} + /// Merge per-slice OCR (top-to-bottom) into one ordered list, de-duplicating -/// text that appears in both slices' overlap region. Dedup is *seam-scoped* -/// (only the first `SEAM` pieces of a slice vs. the last `SEAM` already -/// emitted), so a genuinely repeated line in non-adjacent slices is kept. -/// When two pieces match, the longer (more complete) transcription wins. -fn merge_ocr(slices: Vec>) -> Vec { - let mut out: Vec = Vec::new(); - // Index in `out` where the *previous* slice's pieces began — dedup - // compares only against that slice's tail, never earlier ones. - let mut prev_slice_start = 0usize; +/// text that straddles a slice seam. +/// +/// Pairing is *seam-scoped* (first `SEAM` pieces of a slice vs. the previous +/// slice's tail). A pair is a duplicate when the two are **position-close** +/// (page-global y within a text-line tolerance, same kind) OR text-similar — +/// so even a mis-OCR'd boundary line ("slightly different text, same spot") +/// is caught. The kept copy is the one whose text was **more central in its +/// slice** (farther from the cut = less cropped); when positions are +/// unavailable it falls back to keeping the longer transcription. Repeats in +/// non-adjacent slices are preserved. +fn merge_ocr(slices: Vec) -> Vec { + let mut out: Vec = Vec::new(); + let mut prev_start = 0usize; for (si, slice) in slices.into_iter().enumerate() { - let cur_slice_start = out.len(); - for (pi, piece) in slice.into_iter().enumerate() { + let center = (slice.y0 + slice.y1) / 2.0; + let slice_h = (slice.y1 - slice.y0).max(1.0); + let tol = 0.04 * slice_h; // ~one text line + let cur_start = out.len(); + + // Self-calibrate the model's y values: if they look like pixels + // (max > 1.5) normalize by the max; require some spread to trust + // them at all (a degenerate constant column is worthless). + let ys: Vec = slice.pieces.iter().filter_map(|p| p.y).collect(); + let (ymin, ymax) = ys.iter().fold((f64::MAX, f64::MIN), |(lo, hi), &v| { + (lo.min(v), hi.max(v)) + }); + let calib = if ymax > 1.5 { ymax } else { 1.0 }; + let usable = ys.len() >= 2 && (ymax - ymin) / calib > 0.05; + + for (pi, mut piece) in slice.pieces.into_iter().enumerate() { + let page_y = if usable { + piece.y.map(|y| slice.y0 + (y / calib).clamp(0.0, 1.0) * slice_h) + } else { + None + }; + piece.y = None; // internal-only, never persisted + if si > 0 && pi < SEAM { - let win_start = cur_slice_start.saturating_sub(SEAM).max(prev_slice_start); + let win = cur_start.saturating_sub(SEAM).max(prev_start); let np = normalize_text(&piece.text); - if !np.is_empty() { - if let Some(j) = (win_start..cur_slice_start) - .find(|&j| similar(&normalize_text(&out[j].text), &np)) - { - if piece.text.chars().count() > out[j].text.chars().count() { - out[j] = piece; + let matched = (win..cur_start).find(|&j| { + let e = &out[j]; + let pos = match (e.page_y, page_y) { + (Some(a), Some(b)) => { + (a - b).abs() <= tol && e.piece.kind == piece.kind } - continue; + _ => false, + }; + pos || (!np.is_empty() && similar(&normalize_text(&e.piece.text), &np)) + }); + if let Some(j) = matched { + let keep_new = match (out[j].page_y, page_y) { + (Some(ey), Some(ny)) => { + (ny - center).abs() < (ey - out[j].center).abs() + } + _ => piece.text.chars().count() > out[j].piece.text.chars().count(), + }; + if keep_new { + out[j] = Entry { piece, page_y, center }; } + continue; } } - out.push(piece); + out.push(Entry { piece, page_y, center }); } - prev_slice_start = cur_slice_start; + prev_start = cur_start; } - out + out.into_iter().map(|e| e.piece).collect() } /// Lowercased, alphanumeric-only, whitespace-collapsed form for comparison. @@ -572,9 +631,23 @@ mod tests { OcrResult { text: text.into(), kind: kind.into(), + y: None, } } + fn ocr_y(text: &str, kind: &str, y: f64) -> OcrResult { + OcrResult { + text: text.into(), + kind: kind.into(), + y: Some(y), + } + } + + /// A text-only slice (no positions → fuzzy fallback path). + fn tslice(pieces: Vec) -> SliceOcr { + SliceOcr { y0: 0.0, y1: 1000.0, pieces } + } + fn jpeg_of(w: u32, h: u32) -> Vec { let img = DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb([9, 9, 9]))); encode_jpeg(&img).unwrap() @@ -736,18 +809,18 @@ mod tests { #[test] fn merge_dedups_a_line_repeated_across_the_seam() { let merged = merge_ocr(vec![ - vec![ocr("Hello", "speech"), ocr("world", "speech")], - vec![ocr("world", "speech"), ocr("bye", "speech")], + tslice(vec![ocr("Hello", "speech"), ocr("world", "speech")]), + tslice(vec![ocr("world", "speech"), ocr("bye", "speech")]), ]); let texts: Vec<&str> = merged.iter().map(|o| o.text.as_str()).collect(); assert_eq!(texts, vec!["Hello", "world", "bye"]); } #[test] - fn merge_keeps_the_longer_transcription() { + fn merge_keeps_the_longer_transcription_without_positions() { let merged = merge_ocr(vec![ - vec![ocr("I don", "speech")], - vec![ocr("I don't think so", "speech")], + tslice(vec![ocr("I don", "speech")]), + tslice(vec![ocr("I don't think so", "speech")]), ]); assert_eq!(merged.len(), 1); assert_eq!(merged[0].text, "I don't think so"); @@ -757,9 +830,9 @@ mod tests { fn merge_keeps_non_adjacent_repeats() { // "boom" in slice 0 and slice 2 must both survive (not a seam dup). let merged = merge_ocr(vec![ - vec![ocr("boom", "sfx")], - vec![ocr("crash", "sfx")], - vec![ocr("boom", "sfx")], + tslice(vec![ocr("boom", "sfx")]), + tslice(vec![ocr("crash", "sfx")]), + tslice(vec![ocr("boom", "sfx")]), ]); let booms = merged.iter().filter(|o| o.text == "boom").count(); assert_eq!(booms, 2); @@ -768,14 +841,38 @@ mod tests { #[test] fn merge_preserves_distinct_lines_and_order() { let merged = merge_ocr(vec![ - vec![ocr("one", "narration")], - vec![ocr("two", "speech")], + tslice(vec![ocr("one", "narration")]), + tslice(vec![ocr("two", "speech")]), ]); assert_eq!(merged.len(), 2); assert_eq!(merged[0].kind, "narration"); assert_eq!(merged[1].text, "two"); } + #[test] + fn merge_pairs_by_position_when_text_differs_and_keeps_the_less_cropped() { + // The same physical line sits at page-y ~920 (in the [900,1000] + // overlap). The upper slice saw it well inside; the lower slice + // saw it garbled at its very top. The two transcriptions are NOT + // text-similar — only position pairs them, and centrality keeps + // the upper (less-cropped) one. + let upper = SliceOcr { + y0: 0.0, + y1: 1000.0, + pieces: vec![ocr_y("hi", "speech", 0.2), ocr_y("Hello", "speech", 0.92)], + }; + let lower = SliceOcr { + y0: 900.0, + y1: 1900.0, + pieces: vec![ocr_y("He110", "speech", 0.02), ocr_y("bye", "speech", 0.6)], + }; + let merged = merge_ocr(vec![upper, lower]); + let texts: Vec<&str> = merged.iter().map(|o| o.text.as_str()).collect(); + assert!(texts.contains(&"Hello"), "kept the less-cropped copy: {texts:?}"); + assert!(!texts.contains(&"He110"), "dropped the garbled copy: {texts:?}"); + assert_eq!(merged.len(), 3, "hi + Hello + bye"); + } + // --- rendering --- #[test] diff --git a/backend/src/domain/page_analysis.rs b/backend/src/domain/page_analysis.rs index d654e2d..d280f48 100644 --- a/backend/src/domain/page_analysis.rs +++ b/backend/src/domain/page_analysis.rs @@ -211,13 +211,19 @@ pub struct VisionAnalysis { pub safety_flag: SafetyFlag, } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] pub struct OcrResult { #[serde(default)] pub text: String, /// Free string from the model; mapped via [`OcrKind::from_model_str`]. #[serde(default)] pub kind: String, + /// Optional vertical center of the text as a fraction (0.0 top … 1.0 + /// bottom) of the slice image, requested in the OCR pass to dedup + /// duplicates across slice seams by position. `None` for the combined + /// single-call path. Internal-only — not persisted. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub y: Option, } #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] diff --git a/backend/tests/api_admin_analysis.rs b/backend/tests/api_admin_analysis.rs index d7acad1..9551059 100644 --- a/backend/tests/api_admin_analysis.rs +++ b/backend/tests/api_admin_analysis.rs @@ -434,6 +434,7 @@ async fn seed_partial(pool: &PgPool) -> (uuid::Uuid, uuid::Uuid, uuid::Uuid, uui ocr_results: vec![OcrResult { text: "Hello".into(), kind: "speech".into(), + y: None, }], tagging_results: vec!["action".into(), "city".into()], scene_description: "A rainy street.".into(), diff --git a/backend/tests/api_page_analysis.rs b/backend/tests/api_page_analysis.rs index 0647980..c70a94b 100644 --- a/backend/tests/api_page_analysis.rs +++ b/backend/tests/api_page_analysis.rs @@ -47,6 +47,7 @@ fn analysis( .map(|(text, kind)| OcrResult { text: (*text).to_string(), kind: (*kind).to_string(), + y: None, }) .collect(), tagging_results: tags.iter().map(|t| t.to_string()).collect(), diff --git a/backend/tests/api_page_search.rs b/backend/tests/api_page_search.rs index 91df59f..999ed02 100644 --- a/backend/tests/api_page_search.rs +++ b/backend/tests/api_page_search.rs @@ -54,6 +54,7 @@ fn analysis( .map(|(t, k)| OcrResult { text: (*t).into(), kind: (*k).into(), + y: None, }) .collect(), tagging_results: auto_tags.iter().map(|t| t.to_string()).collect(), diff --git a/frontend/package.json b/frontend/package.json index f0bbea5..4e7f21b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.78.0", + "version": "0.79.0", "private": true, "type": "module", "scripts": {