From 1b295e3cd23135963348702a8fb4b33be8120772 Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Wed, 26 Aug 2026 02:47:13 +0000 Subject: [PATCH] =?UTF-8?q?re:=20read=20all=20eight=20caption=20families?= =?UTF-8?q?=20=E2=80=94=2015x=20more=20text,=20and=20the=20same=20lesson?= =?UTF-8?q?=20twice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_caption_text generalises the key parser from MSG_DEMO_* to all eight families. The shapes are uniform and each family is 100% consistent with its own: seven use MSG____, and VOICE alone inserts a family letter before the id. ids lines build_demo_text 134 537 build_caption_text 3721 8074 The DEMO family comes out identical through both readers -- 537 lines either way -- which is the control that generalising changed nothing that already worked. Pinned by tests/caption_families_disc.rs, along with VOICE ids keeping their family letter. But this does NOT close the gap, and the write-up says so: 8074 against the 44579 text-bearing fields the record-level scan counts is about 18%. The reason is the same lesson this session already learned once. build_caption_text pairs a value with the key that happens to follow it in the raw UTF-16 token stream -- the adjacency heuristic that was wrong for IDXD and is wrong here for the same reason. ixud.rs has no record/field reader at all. The IXUD record table IS decoded and verified disc-wide (1104/1104 objects, 628165/628165 fields reproducing their key) and was simply never wired into the crate. Next step recorded: give ixud.rs an IdxdObject-shaped reader and read captions as fields rather than adjacent tokens. The decode exists; only the plumbing is missing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PMRJjbxLqZtsb5Vb7KunPE --- .../examples/caption_coverage.rs | 38 +++++++++ crates/sylpheed-formats/src/movie_subtitle.rs | 65 +++++++++++++++ .../tests/caption_families_disc.rs | 79 +++++++++++++++++++ docs/re/structures/movie-subtitles.md | 34 +++++++- 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 crates/sylpheed-formats/examples/caption_coverage.rs create mode 100644 crates/sylpheed-formats/tests/caption_families_disc.rs diff --git a/crates/sylpheed-formats/examples/caption_coverage.rs b/crates/sylpheed-formats/examples/caption_coverage.rs new file mode 100644 index 00000000..229718fd --- /dev/null +++ b/crates/sylpheed-formats/examples/caption_coverage.rs @@ -0,0 +1,38 @@ +//! How many caption lines does `build_caption_text` actually recover? +use std::collections::BTreeMap; +use sylpheed_formats::{movie_subtitle, PakArchive}; + +fn main() { + let disc = std::env::var("SYLPHEED_DISC").expect("set SYLPHEED_DISC"); + let pak = PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let all = movie_subtitle::build_caption_text(&pak); + let demo = movie_subtitle::build_demo_text(&pak); + + let mut per: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + for (id, lines) in &all { + let fam = id.split('_').next().unwrap(); + let e = per.entry(fam).or_default(); + e.0 += 1; + e.1 += lines.len(); + } + println!("{:<8} {:>8} {:>9}", "family", "ids", "lines"); + let (mut ids, mut lines) = (0, 0); + for (fam, (i, l)) in &per { + println!("{fam:<8} {i:>8} {l:>9}"); + ids += i; + lines += l; + } + println!("{:<8} {:>8} {:>9}", "TOTAL", ids, lines); + println!( + "\nbuild_demo_text alone: {} ids, {} lines", + demo.len(), + demo.values().map(|v| v.len()).sum::() + ); + // The DEMO family must come out identical either way — that is the control. + let demo_via_all: usize = all + .iter() + .filter(|(k, _)| k.starts_with("DEMO_")) + .map(|(_, v)| v.len()) + .sum(); + println!("DEMO via build_caption_text: {demo_via_all} lines"); +} diff --git a/crates/sylpheed-formats/src/movie_subtitle.rs b/crates/sylpheed-formats/src/movie_subtitle.rs index fa3d7a91..1fada570 100644 --- a/crates/sylpheed-formats/src/movie_subtitle.rs +++ b/crates/sylpheed-formats/src/movie_subtitle.rs @@ -238,6 +238,71 @@ pub fn build_demo_text(text_pak: &PakArchive) -> BTreeMap> { .collect() } +/// Every caption family in the pack, not just the cutscene one. +/// +/// `build_demo_text` reads `MSG_DEMO_*` — 560 text-bearing keys, the **smallest** +/// of eight families. The other seven carry the combat chatter and the in-mission +/// scripted dialogue: 44 019 more lines, or **98.7 %** of the game's text. +/// +/// Key shapes, measured over every IXUD block in `GP_MAIN_GAME_E.pak`: +/// +/// | family | shape | text-bearing keys | +/// |---|---|---| +/// | `ACRO` `ADAN` `ADPL` `BIRD` `DEMO` `RHIN` `TCAF` | `MSG____` | 37 803 | +/// | `VOICE` | `MSG_VOICE____` | 6 776 | +/// +/// `VOICE` is the only family with a letter before the id, and every family is +/// 100 % consistent with its own shape. +/// +/// Returns `"_" → ordered lines`, e.g. `"ADAN_600"`, `"VOICE_A_150"`. +/// +/// ⚠️ The id here is the **caption** id. It is *not* the voice-bank id: a message +/// page binding `VOICE_C_468` carries lines keyed `MSG_VOICE_C_385_*`. Same +/// family letter, different index space — do not derive one from the other. +pub fn build_caption_text(text_pak: &PakArchive) -> BTreeMap> { + let mut by_id: BTreeMap> = BTreeMap::new(); + for entry in text_pak.entries() { + let Ok(bytes) = text_pak.read(entry) else { + continue; + }; + if !is_ixud(&bytes) { + continue; + } + let toks = utf16le_tokens(&bytes); + for w in toks.windows(2) { + let Some((id, page, line)) = caption_key(&w[1]) else { + continue; + }; + // Same pairing rule as build_demo_text: the value is the token + // immediately before the key, and only when it is real text rather + // than another key (bare keys are serialized consecutively too). + if caption_key(&w[0]).is_none() && !w[0].trim().is_empty() { + by_id.entry(id).or_default().insert((page, line), clean(&w[0])); + } + } + } + by_id + .into_iter() + .map(|(d, m)| (d, m.into_values().collect())) + .collect() +} + +/// `MSG____` → `("_", page, line)`, with `VOICE`'s +/// extra family letter folded into the id. +fn caption_key(t: &str) -> Option<(String, u32, u32)> { + let rest = t.strip_prefix("MSG_")?; + let mut parts: Vec<&str> = rest.split('_').collect(); + // Trailing _ are always numeric. + let line: u32 = parts.pop()?.parse().ok()?; + let page: u32 = parts.pop()?.parse().ok()?; + // What remains is or _, then the numeric id. + let id: u32 = parts.pop()?.parse().ok()?; + if parts.is_empty() { + return None; + } + Some((format!("{}_{id:03}", parts.join("_")), page, line)) +} + /// Parse a timing track's IXUD payload into `(token, start, end)` triples, where /// `token` is either a `MSG_DEMO_` reference or an inline caption string. /// diff --git a/crates/sylpheed-formats/tests/caption_families_disc.rs b/crates/sylpheed-formats/tests/caption_families_disc.rs new file mode 100644 index 00000000..d87b8829 --- /dev/null +++ b/crates/sylpheed-formats/tests/caption_families_disc.rs @@ -0,0 +1,79 @@ +//! Caption recovery across all eight `MSG_*` families. +//! +//! `build_demo_text` reads only `MSG_DEMO_*`, the smallest family. +//! `build_caption_text` generalises the key parser to all eight. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use sylpheed_formats::{movie_subtitle, PakArchive}; + +fn disc_root() -> Option { + if let Ok(p) = std::env::var("SYLPHEED_DISC") { + let p = PathBuf::from(p); + if p.join("dat").is_dir() { + return Some(p); + } + } + let d = Path::new( + "/home/fabi/RE - Project Sylpheed/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja)", + ); + d.join("dat").is_dir().then(|| d.to_path_buf()) +} + +macro_rules! skip_without_disc { + ($root:ident) => { + let Some($root) = disc_root() else { + eprintln!("SKIP: set SYLPHEED_DISC"); + return; + }; + }; +} + +#[test] +fn all_eight_caption_families_are_read() { + skip_without_disc!(root); + let pak = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let all = movie_subtitle::build_caption_text(&pak); + + let mut per: BTreeMap = BTreeMap::new(); + for (id, lines) in &all { + *per.entry(id.split('_').next().unwrap().to_string()).or_default() += lines.len(); + } + let fams: Vec<&str> = per.keys().map(String::as_str).collect(); + assert_eq!( + fams, + ["ACRO", "ADAN", "ADPL", "BIRD", "DEMO", "RHIN", "TCAF", "VOICE"], + "all eight families must appear" + ); + + let total: usize = all.values().map(|v| v.len()).sum(); + assert_eq!(total, 8074, "recovered caption lines"); + assert_eq!(all.len(), 3721, "recovered caption ids"); + + // `VOICE` is the only family with a letter before the id; its ids must keep it. + assert!(all.contains_key("VOICE_A_150"), "VOICE ids keep their family letter"); +} + +/// The control: the DEMO family must come out identical through the new reader, +/// so generalising cannot have changed what already worked. +#[test] +fn demo_family_is_unchanged_by_generalising() { + skip_without_disc!(root); + let pak = PakArchive::open(root.join("dat/GP_MAIN_GAME_E.pak")).expect("pak"); + let demo = movie_subtitle::build_demo_text(&pak); + let all = movie_subtitle::build_caption_text(&pak); + + let old: usize = demo.values().map(|v| v.len()).sum(); + let new: usize = all + .iter() + .filter(|(k, _)| k.starts_with("DEMO_")) + .map(|(_, v)| v.len()) + .sum(); + assert_eq!(old, 537); + assert_eq!(new, old, "DEMO must be identical through both readers"); + + // …and 15x more text overall than the DEMO-only path saw. + let total: usize = all.values().map(|v| v.len()).sum(); + assert!(total > old * 14, "expected a large gain, got {total} vs {old}"); +} diff --git a/docs/re/structures/movie-subtitles.md b/docs/re/structures/movie-subtitles.md index 29e75dcc..56aff5d8 100644 --- a/docs/re/structures/movie-subtitles.md +++ b/docs/re/structures/movie-subtitles.md @@ -338,7 +338,39 @@ honest denominator.) `MSG_VOICE_*` is the family the message tables reference — the dialogue whose voice bindings are analysed above — and nothing in `crates/` parses it. -▶️ **First step:** `movie_subtitle::build_demo_text` already pairs a text value +### 🟡 Generalised — 15× more text, but the gap is NOT closed + +`movie_subtitle::build_caption_text` now reads all eight families. Key shapes are +uniform and each family is 100 % consistent with its own: + +* `ACRO` `ADAN` `ADPL` `BIRD` `DEMO` `RHIN` `TCAF` — `MSG____` +* `VOICE` alone — `MSG_VOICE____` + +Recovered (`tests/caption_families_disc.rs`, `examples/caption_coverage.rs`): + +| | ids | lines | +|---|---|---| +| `build_demo_text` (before) | 134 | **537** | +| `build_caption_text` (now) | **3721** | **8074** | + +The `DEMO` family comes out **identical** through both readers — 537 lines either +way — which is the control that generalising changed nothing that worked. + +❌ **But 8074 is still far short of the 44 579 text-bearing fields** the +record-level scan counts. The new reader recovers about **18 %** of them. + +🔑 **Why, and it is the same lesson twice.** `ixud.rs` has **no record/field +reader** — `build_caption_text` pairs a value with the key that happens to follow +it in the raw UTF-16 token stream, exactly the adjacency heuristic that was wrong +for IDXD. The IXUD record table *is* decoded and verified disc-wide (1104/1104 +objects, 628 165/628 165 fields reproducing their key) in +[idxd-container](idxd-container.md) — it was simply never wired into the crate. + +▶️ **Next:** give `ixud.rs` an `IdxdObject`-shaped record/field reader and read +captions as *fields*, not adjacent tokens. The decode already exists; only the +plumbing is missing. + +▶️ Superseded first step: `movie_subtitle::build_demo_text` already pairs a text value with the `MSG_DEMO___` key that follows it; the other seven families use the same `__` shape, so generalising the key parser is most of the work. ⚠️ Do **not** assume the id spaces relate — the voice-bank