diff --git a/crates/sylpheed-formats/examples/caption_coverage.rs b/crates/sylpheed-formats/examples/caption_coverage.rs new file mode 100644 index 0000000..229718f --- /dev/null +++ b/crates/sylpheed-formats/examples/caption_coverage.rs @@ -0,0 +1,38 @@ +//! How many caption lines does `build_caption_text` actually recover? +use std::collections::BTreeMap; +use sylpheed_formats::{movie_subtitle, PakArchive}; + +fn main() { + let disc = std::env::var("SYLPHEED_DISC").expect("set SYLPHEED_DISC"); + let pak = PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let all = movie_subtitle::build_caption_text(&pak); + let demo = movie_subtitle::build_demo_text(&pak); + + let mut per: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + for (id, lines) in &all { + let fam = id.split('_').next().unwrap(); + let e = per.entry(fam).or_default(); + e.0 += 1; + e.1 += lines.len(); + } + println!("{:<8} {:>8} {:>9}", "family", "ids", "lines"); + let (mut ids, mut lines) = (0, 0); + for (fam, (i, l)) in &per { + println!("{fam:<8} {i:>8} {l:>9}"); + ids += i; + lines += l; + } + println!("{:<8} {:>8} {:>9}", "TOTAL", ids, lines); + println!( + "\nbuild_demo_text alone: {} ids, {} lines", + demo.len(), + demo.values().map(|v| v.len()).sum::() + ); + // The DEMO family must come out identical either way — that is the control. + let demo_via_all: usize = all + .iter() + .filter(|(k, _)| k.starts_with("DEMO_")) + .map(|(_, v)| v.len()) + .sum(); + println!("DEMO via build_caption_text: {demo_via_all} lines"); +} diff --git a/crates/sylpheed-formats/src/movie_subtitle.rs b/crates/sylpheed-formats/src/movie_subtitle.rs index fa3d7a9..1fada57 100644 --- a/crates/sylpheed-formats/src/movie_subtitle.rs +++ b/crates/sylpheed-formats/src/movie_subtitle.rs @@ -238,6 +238,71 @@ pub fn build_demo_text(text_pak: &PakArchive) -> BTreeMap> { .collect() } +/// Every caption family in the pack, not just the cutscene one. +/// +/// `build_demo_text` reads `MSG_DEMO_*` — 560 text-bearing keys, the **smallest** +/// of eight families. The other seven carry the combat chatter and the in-mission +/// scripted dialogue: 44 019 more lines, or **98.7 %** of the game's text. +/// +/// Key shapes, measured over every IXUD block in `GP_MAIN_GAME_E.pak`: +/// +/// | family | shape | text-bearing keys | +/// |---|---|---| +/// | `ACRO` `ADAN` `ADPL` `BIRD` `DEMO` `RHIN` `TCAF` | `MSG____` | 37 803 | +/// | `VOICE` | `MSG_VOICE____` | 6 776 | +/// +/// `VOICE` is the only family with a letter before the id, and every family is +/// 100 % consistent with its own shape. +/// +/// Returns `"_" → ordered lines`, e.g. `"ADAN_600"`, `"VOICE_A_150"`. +/// +/// ⚠️ The id here is the **caption** id. It is *not* the voice-bank id: a message +/// page binding `VOICE_C_468` carries lines keyed `MSG_VOICE_C_385_*`. Same +/// family letter, different index space — do not derive one from the other. +pub fn build_caption_text(text_pak: &PakArchive) -> BTreeMap> { + let mut by_id: BTreeMap> = BTreeMap::new(); + for entry in text_pak.entries() { + let Ok(bytes) = text_pak.read(entry) else { + continue; + }; + if !is_ixud(&bytes) { + continue; + } + let toks = utf16le_tokens(&bytes); + for w in toks.windows(2) { + let Some((id, page, line)) = caption_key(&w[1]) else { + continue; + }; + // Same pairing rule as build_demo_text: the value is the token + // immediately before the key, and only when it is real text rather + // than another key (bare keys are serialized consecutively too). + if caption_key(&w[0]).is_none() && !w[0].trim().is_empty() { + by_id.entry(id).or_default().insert((page, line), clean(&w[0])); + } + } + } + by_id + .into_iter() + .map(|(d, m)| (d, m.into_values().collect())) + .collect() +} + +/// `MSG____` → `("_", page, line)`, with `VOICE`'s +/// extra family letter folded into the id. +fn caption_key(t: &str) -> Option<(String, u32, u32)> { + let rest = t.strip_prefix("MSG_")?; + let mut parts: Vec<&str> = rest.split('_').collect(); + // Trailing _ are always numeric. + let line: u32 = parts.pop()?.parse().ok()?; + let page: u32 = parts.pop()?.parse().ok()?; + // What remains is or _, then the numeric id. + let id: u32 = parts.pop()?.parse().ok()?; + if parts.is_empty() { + return None; + } + Some((format!("{}_{id:03}", parts.join("_")), page, line)) +} + /// Parse a timing track's IXUD payload into `(token, start, end)` triples, where /// `token` is either a `MSG_DEMO_` reference or an inline caption string. /// diff --git a/crates/sylpheed-formats/tests/caption_families_disc.rs b/crates/sylpheed-formats/tests/caption_families_disc.rs new file mode 100644 index 0000000..d87b882 --- /dev/null +++ b/crates/sylpheed-formats/tests/caption_families_disc.rs @@ -0,0 +1,79 @@ +//! Caption recovery across all eight `MSG_*` families. +//! +//! `build_demo_text` reads only `MSG_DEMO_*`, the smallest family. +//! `build_caption_text` generalises the key parser to all eight. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use sylpheed_formats::{movie_subtitle, PakArchive}; + +fn disc_root() -> Option { + if let Ok(p) = std::env::var("SYLPHEED_DISC") { + let p = PathBuf::from(p); + if p.join("dat").is_dir() { + return Some(p); + } + } + let d = Path::new( + "/home/fabi/RE - Project Sylpheed/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja)", + ); + d.join("dat").is_dir().then(|| d.to_path_buf()) +} + +macro_rules! skip_without_disc { + ($root:ident) => { + let Some($root) = disc_root() else { + eprintln!("SKIP: set SYLPHEED_DISC"); + return; + }; + }; +} + +#[test] +fn all_eight_caption_families_are_read() { + skip_without_disc!(root); + let pak = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let all = movie_subtitle::build_caption_text(&pak); + + let mut per: BTreeMap = BTreeMap::new(); + for (id, lines) in &all { + *per.entry(id.split('_').next().unwrap().to_string()).or_default() += lines.len(); + } + let fams: Vec<&str> = per.keys().map(String::as_str).collect(); + assert_eq!( + fams, + ["ACRO", "ADAN", "ADPL", "BIRD", "DEMO", "RHIN", "TCAF", "VOICE"], + "all eight families must appear" + ); + + let total: usize = all.values().map(|v| v.len()).sum(); + assert_eq!(total, 8074, "recovered caption lines"); + assert_eq!(all.len(), 3721, "recovered caption ids"); + + // `VOICE` is the only family with a letter before the id; its ids must keep it. + assert!(all.contains_key("VOICE_A_150"), "VOICE ids keep their family letter"); +} + +/// The control: the DEMO family must come out identical through the new reader, +/// so generalising cannot have changed what already worked. +#[test] +fn demo_family_is_unchanged_by_generalising() { + skip_without_disc!(root); + let pak = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let demo = movie_subtitle::build_demo_text(&pak); + let all = movie_subtitle::build_caption_text(&pak); + + let old: usize = demo.values().map(|v| v.len()).sum(); + let new: usize = all + .iter() + .filter(|(k, _)| k.starts_with("DEMO_")) + .map(|(_, v)| v.len()) + .sum(); + assert_eq!(old, 537); + assert_eq!(new, old, "DEMO must be identical through both readers"); + + // …and 15x more text overall than the DEMO-only path saw. + let total: usize = all.values().map(|v| v.len()).sum(); + assert!(total > old * 14, "expected a large gain, got {total} vs {old}"); +} diff --git a/docs/re/structures/movie-subtitles.md b/docs/re/structures/movie-subtitles.md index 29e75dc..56aff5d 100644 --- a/docs/re/structures/movie-subtitles.md +++ b/docs/re/structures/movie-subtitles.md @@ -338,7 +338,39 @@ honest denominator.) `MSG_VOICE_*` is the family the message tables reference — the dialogue whose voice bindings are analysed above — and nothing in `crates/` parses it. -▶️ **First step:** `movie_subtitle::build_demo_text` already pairs a text value +### 🟡 Generalised — 15× more text, but the gap is NOT closed + +`movie_subtitle::build_caption_text` now reads all eight families. Key shapes are +uniform and each family is 100 % consistent with its own: + +* `ACRO` `ADAN` `ADPL` `BIRD` `DEMO` `RHIN` `TCAF` — `MSG____` +* `VOICE` alone — `MSG_VOICE____` + +Recovered (`tests/caption_families_disc.rs`, `examples/caption_coverage.rs`): + +| | ids | lines | +|---|---|---| +| `build_demo_text` (before) | 134 | **537** | +| `build_caption_text` (now) | **3721** | **8074** | + +The `DEMO` family comes out **identical** through both readers — 537 lines either +way — which is the control that generalising changed nothing that worked. + +❌ **But 8074 is still far short of the 44 579 text-bearing fields** the +record-level scan counts. The new reader recovers about **18 %** of them. + +🔑 **Why, and it is the same lesson twice.** `ixud.rs` has **no record/field +reader** — `build_caption_text` pairs a value with the key that happens to follow +it in the raw UTF-16 token stream, exactly the adjacency heuristic that was wrong +for IDXD. The IXUD record table *is* decoded and verified disc-wide (1104/1104 +objects, 628 165/628 165 fields reproducing their key) in +[idxd-container](idxd-container.md) — it was simply never wired into the crate. + +▶️ **Next:** give `ixud.rs` an `IdxdObject`-shaped record/field reader and read +captions as *fields*, not adjacent tokens. The decode already exists; only the +plumbing is missing. + +▶️ Superseded first step: `movie_subtitle::build_demo_text` already pairs a text value with the `MSG_DEMO___` key that follows it; the other seven families use the same `__` shape, so generalising the key parser is most of the work. ⚠️ Do **not** assume the id spaces relate — the voice-bank