fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(analysis): position-aware OCR seam dedup for sliced pages

Browse Source 
Boundary text duplicated across slices (often with slightly different,
cropped transcriptions) survived the text-only merge. The OCR pass now
asks for a per-piece vertical position and the merge uses it:

- OcrResult gains optional `y` (fraction 0..1 of the slice); the Pass-A
  OCR schema/prompt request it (combined path leaves it None). Not
  persisted.
- merge_ocr takes each slice's working-image y-band and maps `y` to a
  page-global position. A seam pair is a duplicate when position-close
  (same kind) OR text-similar, so a mis-OCR'd boundary line is caught even
  when the text differs. The kept copy is the one more central in its slice
  (less cropped); falls back to keep-longer when positions are missing.
- Self-calibrates the model's y values (pixels vs. fraction) and ignores
  degenerate columns, so a bad localizer can't over-merge.

Tests: position pairs differing texts and keeps the less-cropped copy;
text-only fallback (dedup/keep-longer/non-adjacent/order) still holds.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 23:17:55 +02:00
parent 85d65f5eda
commit 25aba3ac58
9 changed files with 153 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/Cargo.lock generated
View File

@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mangalord"
version = "0.78.0"
version = "0.79.0"
dependencies = [
"anyhow",
"argon2",

2
backend/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "mangalord"
version = "0.78.0"
version = "0.79.0"
edition = "2021"
default-run = "mangalord"

13
backend/src/analysis/prompt.rs
View File

@@ -60,9 +60,11 @@ pub const OCR_PROMPT: &str = concat!(
"You are an OCR engine for manga/comic pages. The image is one vertical ",
"slice of a larger page. Transcribe EVERY visible text element verbatim ",
"into ocr_results, each with its kind ",
"(speech|thought|narration|sfx|title|caption), in top-to-bottom order. ",
"If there is no text, return {\"ocr_results\":[]}. Output ONLY one ",
"minified JSON object, no prose, no markdown."
"(speech|thought|narration|sfx|title|caption) and y — the vertical center ",
"of the text as a fraction from 0.0 (top) to 1.0 (bottom) of THIS image. ",
"List them in top-to-bottom order. If there is no text, return ",
"{\"ocr_results\":[]}. Output ONLY one minified JSON object, no prose, no ",
"markdown."
);
/// Pass B: tags + scene + safety for the whole page, grounded in the merged
@@ -92,9 +94,10 @@ pub fn ocr_json_schema() -> serde_json::Value {
"additionalProperties": false,
"properties": {
"text": { "type": "string" },
"kind": { "type": "string", "enum": OCR_KINDS }
"kind": { "type": "string", "enum": OCR_KINDS },
"y": { "type": "number", "minimum": 0, "maximum": 1 }
},
"required": ["text", "kind"]
"required": ["text", "kind", "y"]
}
}
},

167
backend/src/analysis/vision.rs
View File

@@ -106,8 +106,8 @@ impl VisionClient {
img.resize_exact(width, height, FilterType::Triangle)
};
// Pass A: OCR each band.
let mut slices: Vec<Vec<OcrResult>> = Vec::with_capacity(bands.len());
// Pass A: OCR each band (keep its y-range for seam dedup).
let mut slices: Vec<SliceOcr> = Vec::with_capacity(bands.len());
for (y0, y1) in &bands {
let jpeg = render_slice(&work, *y0, *y1, self.slice.max_pixels)
.ok_or_else(|| anyhow!("failed to encode page slice"))?;
@@ -118,7 +118,11 @@ impl VisionClient {
&data_url(&jpeg),
);
let parsed = parse_chat_completion(&self.post_chat(body).await?)?;
slices.push(parsed.ocr_results);
slices.push(SliceOcr {
y0: *y0 as f64,
y1: *y1 as f64,
pieces: parsed.ocr_results,
});
}
let merged = merge_ocr(slices);
@@ -306,38 +310,93 @@ fn data_url(jpeg: &[u8]) -> String {
/// How many pieces at each seam edge to consider for dedup.
const SEAM: usize = 8;
/// One slice's OCR plus the vertical band `[y0, y1)` it covered in the
/// working image — so per-piece `y` fractions map to page-global positions.
pub struct SliceOcr {
pub y0: f64,
pub y1: f64,
pub pieces: Vec<OcrResult>,
}
/// One merged-list entry with the geometry used for dedup decisions.
struct Entry {
piece: OcrResult,
/// Page-global y of the text (working-image space); `None` when the
/// slice's positions weren't usable.
page_y: Option<f64>,
/// Center of the slice this piece came from.
center: f64,
}
/// Merge per-slice OCR (top-to-bottom) into one ordered list, de-duplicating
/// text that appears in both slices' overlap region. Dedup is *seam-scoped*
/// (only the first `SEAM` pieces of a slice vs. the last `SEAM` already
/// emitted), so a genuinely repeated line in non-adjacent slices is kept.
/// When two pieces match, the longer (more complete) transcription wins.
fn merge_ocr(slices: Vec<Vec<OcrResult>>) -> Vec<OcrResult> {
let mut out: Vec<OcrResult> = Vec::new();
// Index in `out` where the *previous* slice's pieces began — dedup
// compares only against that slice's tail, never earlier ones.
let mut prev_slice_start = 0usize;
/// text that straddles a slice seam.
///
/// Pairing is *seam-scoped* (first `SEAM` pieces of a slice vs. the previous
/// slice's tail). A pair is a duplicate when the two are **position-close**
/// (page-global y within a text-line tolerance, same kind) OR text-similar —
/// so even a mis-OCR'd boundary line ("slightly different text, same spot")
/// is caught. The kept copy is the one whose text was **more central in its
/// slice** (farther from the cut = less cropped); when positions are
/// unavailable it falls back to keeping the longer transcription. Repeats in
/// non-adjacent slices are preserved.
fn merge_ocr(slices: Vec<SliceOcr>) -> Vec<OcrResult> {
let mut out: Vec<Entry> = Vec::new();
let mut prev_start = 0usize;
for (si, slice) in slices.into_iter().enumerate() {
let cur_slice_start = out.len();
for (pi, piece) in slice.into_iter().enumerate() {
let center = (slice.y0 + slice.y1) / 2.0;
let slice_h = (slice.y1 - slice.y0).max(1.0);
let tol = 0.04 * slice_h; // ~one text line
let cur_start = out.len();
// Self-calibrate the model's y values: if they look like pixels
// (max > 1.5) normalize by the max; require some spread to trust
// them at all (a degenerate constant column is worthless).
let ys: Vec<f64> = slice.pieces.iter().filter_map(|p| p.y).collect();
let (ymin, ymax) = ys.iter().fold((f64::MAX, f64::MIN), |(lo, hi), &v| {
(lo.min(v), hi.max(v))
});
let calib = if ymax > 1.5 { ymax } else { 1.0 };
let usable = ys.len() >= 2 && (ymax - ymin) / calib > 0.05;
for (pi, mut piece) in slice.pieces.into_iter().enumerate() {
let page_y = if usable {
piece.y.map(|y| slice.y0 + (y / calib).clamp(0.0, 1.0) * slice_h)
} else {
None
};
piece.y = None; // internal-only, never persisted
if si > 0 && pi < SEAM {
let win_start = cur_slice_start.saturating_sub(SEAM).max(prev_slice_start);
let win = cur_start.saturating_sub(SEAM).max(prev_start);
let np = normalize_text(&piece.text);
if !np.is_empty() {
if let Some(j) = (win_start..cur_slice_start)
.find(|&j| similar(&normalize_text(&out[j].text), &np))
{
if piece.text.chars().count() > out[j].text.chars().count() {
out[j] = piece;
let matched = (win..cur_start).find(|&j| {
let e = &out[j];
let pos = match (e.page_y, page_y) {
(Some(a), Some(b)) => {
(a - b).abs() <= tol && e.piece.kind == piece.kind
}
continue;
_ => false,
};
pos || (!np.is_empty() && similar(&normalize_text(&e.piece.text), &np))
});
if let Some(j) = matched {
let keep_new = match (out[j].page_y, page_y) {
(Some(ey), Some(ny)) => {
(ny - center).abs() < (ey - out[j].center).abs()
}
_ => piece.text.chars().count() > out[j].piece.text.chars().count(),
};
if keep_new {
out[j] = Entry { piece, page_y, center };
}
continue;
}
}
out.push(piece);
out.push(Entry { piece, page_y, center });
}
prev_slice_start = cur_slice_start;
prev_start = cur_start;
}
out
out.into_iter().map(|e| e.piece).collect()
}
/// Lowercased, alphanumeric-only, whitespace-collapsed form for comparison.
@@ -572,9 +631,23 @@ mod tests {
OcrResult {
text: text.into(),
kind: kind.into(),
y: None,
}
}
fn ocr_y(text: &str, kind: &str, y: f64) -> OcrResult {
OcrResult {
text: text.into(),
kind: kind.into(),
y: Some(y),
}
}
/// A text-only slice (no positions → fuzzy fallback path).
fn tslice(pieces: Vec<OcrResult>) -> SliceOcr {
SliceOcr { y0: 0.0, y1: 1000.0, pieces }
}
fn jpeg_of(w: u32, h: u32) -> Vec<u8> {
let img = DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb([9, 9, 9])));
encode_jpeg(&img).unwrap()
@@ -736,18 +809,18 @@ mod tests {
#[test]
fn merge_dedups_a_line_repeated_across_the_seam() {
let merged = merge_ocr(vec![
vec![ocr("Hello", "speech"), ocr("world", "speech")],
vec![ocr("world", "speech"), ocr("bye", "speech")],
tslice(vec![ocr("Hello", "speech"), ocr("world", "speech")]),
tslice(vec![ocr("world", "speech"), ocr("bye", "speech")]),
]);
let texts: Vec<&str> = merged.iter().map(|o| o.text.as_str()).collect();
assert_eq!(texts, vec!["Hello", "world", "bye"]);
}
#[test]
fn merge_keeps_the_longer_transcription() {
fn merge_keeps_the_longer_transcription_without_positions() {
let merged = merge_ocr(vec![
vec![ocr("I don", "speech")],
vec![ocr("I don't think so", "speech")],
tslice(vec![ocr("I don", "speech")]),
tslice(vec![ocr("I don't think so", "speech")]),
]);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].text, "I don't think so");
@@ -757,9 +830,9 @@ mod tests {
fn merge_keeps_non_adjacent_repeats() {
// "boom" in slice 0 and slice 2 must both survive (not a seam dup).
let merged = merge_ocr(vec![
vec![ocr("boom", "sfx")],
vec![ocr("crash", "sfx")],
vec![ocr("boom", "sfx")],
tslice(vec![ocr("boom", "sfx")]),
tslice(vec![ocr("crash", "sfx")]),
tslice(vec![ocr("boom", "sfx")]),
]);
let booms = merged.iter().filter(|o| o.text == "boom").count();
assert_eq!(booms, 2);
@@ -768,14 +841,38 @@ mod tests {
#[test]
fn merge_preserves_distinct_lines_and_order() {
let merged = merge_ocr(vec![
vec![ocr("one", "narration")],
vec![ocr("two", "speech")],
tslice(vec![ocr("one", "narration")]),
tslice(vec![ocr("two", "speech")]),
]);
assert_eq!(merged.len(), 2);
assert_eq!(merged[0].kind, "narration");
assert_eq!(merged[1].text, "two");
}
#[test]
fn merge_pairs_by_position_when_text_differs_and_keeps_the_less_cropped() {
// The same physical line sits at page-y ~920 (in the [900,1000]
// overlap). The upper slice saw it well inside; the lower slice
// saw it garbled at its very top. The two transcriptions are NOT
// text-similar — only position pairs them, and centrality keeps
// the upper (less-cropped) one.
let upper = SliceOcr {
y0: 0.0,
y1: 1000.0,
pieces: vec![ocr_y("hi", "speech", 0.2), ocr_y("Hello", "speech", 0.92)],
};
let lower = SliceOcr {
y0: 900.0,
y1: 1900.0,
pieces: vec![ocr_y("He110", "speech", 0.02), ocr_y("bye", "speech", 0.6)],
};
let merged = merge_ocr(vec![upper, lower]);
let texts: Vec<&str> = merged.iter().map(|o| o.text.as_str()).collect();
assert!(texts.contains(&"Hello"), "kept the less-cropped copy: {texts:?}");
assert!(!texts.contains(&"He110"), "dropped the garbled copy: {texts:?}");
assert_eq!(merged.len(), 3, "hi + Hello + bye");
}
// --- rendering ---
#[test]

8
backend/src/domain/page_analysis.rs
View File

@@ -211,13 +211,19 @@ pub struct VisionAnalysis {
pub safety_flag: SafetyFlag,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]
pub struct OcrResult {
#[serde(default)]
pub text: String,
/// Free string from the model; mapped via [`OcrKind::from_model_str`].
#[serde(default)]
pub kind: String,
/// Optional vertical center of the text as a fraction (0.0 top … 1.0
/// bottom) of the slice image, requested in the OCR pass to dedup
/// duplicates across slice seams by position. `None` for the combined
/// single-call path. Internal-only — not persisted.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub y: Option<f64>,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]

1
backend/tests/api_admin_analysis.rs
View File

@@ -434,6 +434,7 @@ async fn seed_partial(pool: &PgPool) -> (uuid::Uuid, uuid::Uuid, uuid::Uuid, uui
ocr_results: vec![OcrResult {
text: "Hello".into(),
kind: "speech".into(),
y: None,
}],
tagging_results: vec!["action".into(), "city".into()],
scene_description: "A rainy street.".into(),

1
backend/tests/api_page_analysis.rs
View File

@@ -47,6 +47,7 @@ fn analysis(
.map(|(text, kind)| OcrResult {
text: (*text).to_string(),
kind: (*kind).to_string(),
y: None,
})
.collect(),
tagging_results: tags.iter().map(|t| t.to_string()).collect(),

1
backend/tests/api_page_search.rs
View File

@@ -54,6 +54,7 @@ fn analysis(
.map(|(t, k)| OcrResult {
text: (*t).into(),
kind: (*k).into(),
y: None,
})
.collect(),
tagging_results: auto_tags.iter().map(|t| t.to_string()).collect(),

2
frontend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "mangalord-frontend",
"version": "0.78.0",
"version": "0.79.0",
"private": true,
"type": "module",
"scripts": {