localization: resolve the game's UTF-16 text keys to display strings

Project Sylpheed stores its UI/story text in IXUD entries as UTF-16LE,
interleaved `text, key` (a prose string immediately followed by the
identifier that names it). TextIndex::build(pak) scans a language pak's
IXUD entries and indexes every pair — 8457 English keys from
GP_MAIN_GAME_E: objectives, hints, lose conditions, character names,
and cutscene dialogue.

This makes the whole game read in English: the 16-mission campaign
(Repel the surprise attack → Shoot down Margras → the Prometheus Driver
finale), each mission's briefing, and the named cast (Katana, Raymond,
Ellen, Crichton, …) all resolve. API: get / objectives / lose_conditions
/ hints / character_name. 3 tests.

Examples: campaign / dossier print the readable campaign + cast.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

crates/sylpheed-formats/examples/campaign.rs

@@ -0,0 +1,24 @@
+use sylpheed_formats::{game_data, localization::TextIndex, PakArchive};
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let pak=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    let text=TextIndex::build(&pak);
+    let mut stages=game_data::load_stages(&pak);
+    stages.retain(|s|s.id.starts_with('S') && s.id.len()==3 && s.id[1..].parse::<u32>().map(|n|n<=16).unwrap_or(false));
+    stages.sort_by(|a,b|a.id.cmp(&b.id));
+    println!("═══ CAMPAIGN (S01–S16) ═══");
+    for s in &stages{
+        let obj=text.objectives(&s.id,1);
+        println!("\n{} · {}", s.id, s.location.as_deref().unwrap_or("?"));
+        for o in obj.iter().take(2){ println!("   ▸ {o}"); }
+    }
+    // roster with real names
+    let mut chars=game_data::load_characters(&pak);
+    chars.retain(|c|c.faction.as_deref()==Some("TCAF") && c.unique==Some(true) && c.faces.len()>=4);
+    println!("\n═══ PRINCIPAL CAST (TCAF, named) ═══");
+    for c in &chars{
+        let id=c.id.as_deref().unwrap_or("");
+        let name=text.character_name(id.trim_start_matches("Character")).or_else(||c.name_key.as_deref().and_then(|k|text.get(k))).unwrap_or("?");
+        println!("   {name:12} ({} portraits)  [{}]", c.faces.len(), id.trim_start_matches("Character"));
+    }
+}

crates/sylpheed-formats/examples/dossier.rs

@@ -0,0 +1,15 @@
+use sylpheed_formats::{game_data, localization::TextIndex, PakArchive};
+fn main(){
+    let disc=std::env::var("SYLPHEED_DISC").unwrap();
+    let pak=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap();
+    let text=TextIndex::build(&pak);
+    for n in 1..=16u32{
+        let sid=format!("S{n:02}");
+        // primary objective across phases (first that resolves)
+        let obj:Vec<String>=(1..=3).flat_map(|p|text.objectives(&sid,p)).map(|s|s.to_string()).collect();
+        let lose:Vec<String>=(1..=3).flat_map(|p|text.lose_conditions(&sid,p)).map(|s|s.to_string()).collect();
+        let full_obj=obj.join(" ");
+        let full_lose=lose.into_iter().take(2).collect::<Vec<_>>().join(" ");
+        println!("{sid}\t{full_obj}\t{full_lose}");
+    }
+}

crates/sylpheed-formats/src/lib.rs

@@ -63,6 +63,8 @@ pub mod movie_voice;
 
 pub mod game_data;
 
+pub mod localization;
+
 /// Re-export the most commonly used types at the crate root.
 pub use font::FontInfo;
 pub use idxd::{IdxdError, IdxdObject};

crates/sylpheed-formats/src/localization.rs

@@ -0,0 +1,179 @@
+//! Localization — resolve the game's text keys to display strings.
+//!
+//! Project Sylpheed stores its UI/story text in `IXUD` entries as **UTF-16LE**,
+//! interleaved `text, key, text, key, …`: a prose string immediately followed by
+//! the identifier that names it. Objectives (`S01_P1_Objective_00`), hints, lose
+//! conditions, character names (`CharacterName_CharacterRAYMOND`) and cutscene
+//! dialogue (`MSG_*`) all resolve this way. One pak per language — pass
+//! `GP_MAIN_GAME_E.pak` for English, `GP_MAIN_GAME_J.pak` for Japanese, etc.
+//!
+//! ```no_run
+//! use sylpheed_formats::{localization::TextIndex, PakArchive};
+//! let pak = PakArchive::open("dat/GP_MAIN_GAME_E.pak").unwrap();
+//! let text = TextIndex::build(&pak);
+//! assert_eq!(text.character_name("CharacterRAYMOND"), Some("Raymond"));
+//! for line in text.objectives("S01", 1) { println!("• {line}"); }
+//! ```
+
+use crate::pak::PakArchive;
+use std::collections::BTreeMap;
+
+/// A UTF-16 unit that belongs to a printable text run (ASCII + Latin-1 + Latin
+/// Extended-A). Control codes and the CJK planes split tokens.
+fn is_text_unit(u: u16) -> bool {
+    matches!(u, 0x20..=0x7e | 0xa0..=0xff | 0x100..=0x17f)
+}
+
+/// Split a UTF-16LE blob into printable tokens (mirrors the subtitle tokenizer:
+/// step 2 inside a run, 1 on a break to resync odd alignment).
+fn utf16_tokens(bytes: &[u8]) -> Vec<String> {
+    let mut out = Vec::new();
+    let mut cur = String::new();
+    let mut i = 0;
+    while i + 1 < bytes.len() {
+        let u = u16::from_le_bytes([bytes[i], bytes[i + 1]]);
+        if is_text_unit(u) {
+            if let Some(c) = char::from_u32(u as u32) {
+                cur.push(c);
+            }
+            i += 2;
+        } else {
+            if cur.chars().count() >= 2 {
+                out.push(std::mem::take(&mut cur));
+            } else {
+                cur.clear();
+            }
+            i += 1;
+        }
+    }
+    if cur.chars().count() >= 2 {
+        out.push(cur);
+    }
+    out
+}
+
+/// An identifier token — the key half of a `text, key` pair.
+fn is_key(s: &str) -> bool {
+    s.len() >= 5
+        && s.contains('_')
+        && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
+        && s.chars().any(|c| c.is_ascii_uppercase())
+}
+
+/// A display-text token — anything with a lowercase letter that isn't itself a key.
+fn is_prose(s: &str) -> bool {
+    !is_key(s) && s.chars().any(|c| c.is_ascii_lowercase())
+}
+
+/// Text-key → display-string index for one language, built from a `GP_MAIN_GAME_*`
+/// pak.
+#[derive(Debug, Clone, Default)]
+pub struct TextIndex {
+    entries: BTreeMap<String, String>,
+}
+
+impl TextIndex {
+    /// Scan a language pak's `IXUD` entries and index every `text → key` pair.
+    pub fn build(pak: &PakArchive) -> Self {
+        let mut entries = BTreeMap::new();
+        for e in pak.entries() {
+            let Ok(b) = pak.read(e) else { continue };
+            if b.len() < 4 || &b[0..4] != b"IXUD" {
+                continue;
+            }
+            let toks = utf16_tokens(&b);
+            for w in toks.windows(2) {
+                if is_key(&w[1]) && is_prose(&w[0]) {
+                    entries.entry(w[1].clone()).or_insert_with(|| w[0].clone());
+                }
+            }
+        }
+        TextIndex { entries }
+    }
+
+    /// The display string for a text key, if present.
+    pub fn get(&self, key: &str) -> Option<&str> {
+        self.entries.get(key).map(String::as_str)
+    }
+
+    /// Number of indexed keys.
+    pub fn len(&self) -> usize {
+        self.entries.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    /// The objective lines for a stage/phase, e.g. `("S01", 1)`. Empty if none
+    /// resolve (unused objective slots are simply absent on disc).
+    pub fn objectives(&self, stage: &str, phase: u32) -> Vec<&str> {
+        self.numbered(&format!("{stage}_P{phase}_Objective"))
+    }
+
+    /// The lose-condition lines for a stage/phase.
+    pub fn lose_conditions(&self, stage: &str, phase: u32) -> Vec<&str> {
+        self.numbered(&format!("{stage}_P{phase}_Lose"))
+    }
+
+    /// The hint lines for a stage/phase.
+    pub fn hints(&self, stage: &str, phase: u32) -> Vec<&str> {
+        self.numbered(&format!("{stage}_P{phase}_Hint"))
+    }
+
+    /// A character's display name from its id (`CharacterRAYMOND` → `Raymond`).
+    pub fn character_name(&self, character_id: &str) -> Option<&str> {
+        self.get(&format!("CharacterName_{character_id}"))
+    }
+
+    /// Collect a `<base>_NN` run of lines while they resolve.
+    fn numbered(&self, base: &str) -> Vec<&str> {
+        (0..16)
+            .filter_map(|i| self.get(&format!("{base}_{i:02}")))
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tokenizes_mixed_runs() {
+        // "Hi" NUL "K_1" as UTF-16LE → two tokens.
+        let mut b = Vec::new();
+        for s in ["Hi", "\0", "K_1x"] {
+            for c in s.chars() {
+                b.extend_from_slice(&(c as u16).to_le_bytes());
+            }
+        }
+        let t = utf16_tokens(&b);
+        assert!(t.contains(&"Hi".to_string()));
+        assert!(t.contains(&"K_1x".to_string()));
+    }
+
+    #[test]
+    fn key_and_prose_classify() {
+        assert!(is_key("S01_P1_Objective_00"));
+        assert!(is_key("CharacterName_CharacterRAYMOND"));
+        assert!(!is_key("Repel the attack."));
+        assert!(is_prose("Repel the attack."));
+        assert!(is_prose("Raymond"));
+        assert!(!is_prose("S01_P1_Objective_00"));
+    }
+
+    // Disc-backed — skipped unless SYLPHEED_DISC is set.
+    #[test]
+    fn resolves_real_text() {
+        let Ok(disc) = std::env::var("SYLPHEED_DISC") else { return };
+        let Ok(pak) = PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")) else { return };
+        let t = TextIndex::build(&pak);
+        assert!(t.len() > 5000, "expected a large index, got {}", t.len());
+        assert_eq!(t.character_name("CharacterRAYMOND"), Some("Raymond"));
+        assert_eq!(
+            t.get("S01_P1_Objective_00"),
+            Some("Repel the enemy's surprise attack.")
+        );
+        assert!(!t.objectives("S01", 1).is_empty());
+    }
+}