From d73601fe45d6829ea12f23ab764f3608597a4888 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Thu, 23 Jul 2026 19:49:42 +0200 Subject: [PATCH] localization: resolve the game's UTF-16 text keys to display strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project Sylpheed stores its UI/story text in IXUD entries as UTF-16LE, interleaved `text, key` (a prose string immediately followed by the identifier that names it). TextIndex::build(pak) scans a language pak's IXUD entries and indexes every pair — 8457 English keys from GP_MAIN_GAME_E: objectives, hints, lose conditions, character names, and cutscene dialogue. This makes the whole game read in English: the 16-mission campaign (Repel the surprise attack → Shoot down Margras → the Prometheus Driver finale), each mission's briefing, and the named cast (Katana, Raymond, Ellen, Crichton, …) all resolve. API: get / objectives / lose_conditions / hints / character_name. 3 tests. Examples: campaign / dossier print the readable campaign + cast. Co-Authored-By: Claude Opus 4.8 --- crates/sylpheed-formats/examples/campaign.rs | 24 +++ crates/sylpheed-formats/examples/dossier.rs | 15 ++ crates/sylpheed-formats/src/lib.rs | 2 + crates/sylpheed-formats/src/localization.rs | 179 +++++++++++++++++++ 4 files changed, 220 insertions(+) create mode 100644 crates/sylpheed-formats/examples/campaign.rs create mode 100644 crates/sylpheed-formats/examples/dossier.rs create mode 100644 crates/sylpheed-formats/src/localization.rs diff --git a/crates/sylpheed-formats/examples/campaign.rs b/crates/sylpheed-formats/examples/campaign.rs new file mode 100644 index 0000000..4b4bc46 --- /dev/null +++ b/crates/sylpheed-formats/examples/campaign.rs @@ -0,0 +1,24 @@ +use sylpheed_formats::{game_data, localization::TextIndex, PakArchive}; +fn main(){ + let disc=std::env::var("SYLPHEED_DISC").unwrap(); + let pak=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap(); + let text=TextIndex::build(&pak); + let mut stages=game_data::load_stages(&pak); + stages.retain(|s|s.id.starts_with('S') && s.id.len()==3 && s.id[1..].parse::().map(|n|n<=16).unwrap_or(false)); + stages.sort_by(|a,b|a.id.cmp(&b.id)); + println!("═══ CAMPAIGN (S01–S16) ═══"); + for s in &stages{ + let obj=text.objectives(&s.id,1); + println!("\n{} · {}", s.id, s.location.as_deref().unwrap_or("?")); + for o in obj.iter().take(2){ println!(" ▸ {o}"); } + } + // roster with real names + let mut chars=game_data::load_characters(&pak); + chars.retain(|c|c.faction.as_deref()==Some("TCAF") && c.unique==Some(true) && c.faces.len()>=4); + println!("\n═══ PRINCIPAL CAST (TCAF, named) ═══"); + for c in &chars{ + let id=c.id.as_deref().unwrap_or(""); + let name=text.character_name(id.trim_start_matches("Character")).or_else(||c.name_key.as_deref().and_then(|k|text.get(k))).unwrap_or("?"); + println!(" {name:12} ({} portraits) [{}]", c.faces.len(), id.trim_start_matches("Character")); + } +} diff --git a/crates/sylpheed-formats/examples/dossier.rs b/crates/sylpheed-formats/examples/dossier.rs new file mode 100644 index 0000000..35b3322 --- /dev/null +++ b/crates/sylpheed-formats/examples/dossier.rs @@ -0,0 +1,15 @@ +use sylpheed_formats::{game_data, localization::TextIndex, PakArchive}; +fn main(){ + let disc=std::env::var("SYLPHEED_DISC").unwrap(); + let pak=PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")).unwrap(); + let text=TextIndex::build(&pak); + for n in 1..=16u32{ + let sid=format!("S{n:02}"); + // primary objective across phases (first that resolves) + let obj:Vec=(1..=3).flat_map(|p|text.objectives(&sid,p)).map(|s|s.to_string()).collect(); + let lose:Vec=(1..=3).flat_map(|p|text.lose_conditions(&sid,p)).map(|s|s.to_string()).collect(); + let full_obj=obj.join(" "); + let full_lose=lose.into_iter().take(2).collect::>().join(" "); + println!("{sid}\t{full_obj}\t{full_lose}"); + } +} diff --git a/crates/sylpheed-formats/src/lib.rs b/crates/sylpheed-formats/src/lib.rs index c16e713..6c1c0c7 100644 --- a/crates/sylpheed-formats/src/lib.rs +++ b/crates/sylpheed-formats/src/lib.rs @@ -63,6 +63,8 @@ pub mod movie_voice; pub mod game_data; +pub mod localization; + /// Re-export the most commonly used types at the crate root. pub use font::FontInfo; pub use idxd::{IdxdError, IdxdObject}; diff --git a/crates/sylpheed-formats/src/localization.rs b/crates/sylpheed-formats/src/localization.rs new file mode 100644 index 0000000..2a636b8 --- /dev/null +++ b/crates/sylpheed-formats/src/localization.rs @@ -0,0 +1,179 @@ +//! Localization — resolve the game's text keys to display strings. +//! +//! Project Sylpheed stores its UI/story text in `IXUD` entries as **UTF-16LE**, +//! interleaved `text, key, text, key, …`: a prose string immediately followed by +//! the identifier that names it. Objectives (`S01_P1_Objective_00`), hints, lose +//! conditions, character names (`CharacterName_CharacterRAYMOND`) and cutscene +//! dialogue (`MSG_*`) all resolve this way. One pak per language — pass +//! `GP_MAIN_GAME_E.pak` for English, `GP_MAIN_GAME_J.pak` for Japanese, etc. +//! +//! ```no_run +//! use sylpheed_formats::{localization::TextIndex, PakArchive}; +//! let pak = PakArchive::open("dat/GP_MAIN_GAME_E.pak").unwrap(); +//! let text = TextIndex::build(&pak); +//! assert_eq!(text.character_name("CharacterRAYMOND"), Some("Raymond")); +//! for line in text.objectives("S01", 1) { println!("• {line}"); } +//! ``` + +use crate::pak::PakArchive; +use std::collections::BTreeMap; + +/// A UTF-16 unit that belongs to a printable text run (ASCII + Latin-1 + Latin +/// Extended-A). Control codes and the CJK planes split tokens. +fn is_text_unit(u: u16) -> bool { + matches!(u, 0x20..=0x7e | 0xa0..=0xff | 0x100..=0x17f) +} + +/// Split a UTF-16LE blob into printable tokens (mirrors the subtitle tokenizer: +/// step 2 inside a run, 1 on a break to resync odd alignment). +fn utf16_tokens(bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut i = 0; + while i + 1 < bytes.len() { + let u = u16::from_le_bytes([bytes[i], bytes[i + 1]]); + if is_text_unit(u) { + if let Some(c) = char::from_u32(u as u32) { + cur.push(c); + } + i += 2; + } else { + if cur.chars().count() >= 2 { + out.push(std::mem::take(&mut cur)); + } else { + cur.clear(); + } + i += 1; + } + } + if cur.chars().count() >= 2 { + out.push(cur); + } + out +} + +/// An identifier token — the key half of a `text, key` pair. +fn is_key(s: &str) -> bool { + s.len() >= 5 + && s.contains('_') + && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + && s.chars().any(|c| c.is_ascii_uppercase()) +} + +/// A display-text token — anything with a lowercase letter that isn't itself a key. +fn is_prose(s: &str) -> bool { + !is_key(s) && s.chars().any(|c| c.is_ascii_lowercase()) +} + +/// Text-key → display-string index for one language, built from a `GP_MAIN_GAME_*` +/// pak. +#[derive(Debug, Clone, Default)] +pub struct TextIndex { + entries: BTreeMap, +} + +impl TextIndex { + /// Scan a language pak's `IXUD` entries and index every `text → key` pair. + pub fn build(pak: &PakArchive) -> Self { + let mut entries = BTreeMap::new(); + for e in pak.entries() { + let Ok(b) = pak.read(e) else { continue }; + if b.len() < 4 || &b[0..4] != b"IXUD" { + continue; + } + let toks = utf16_tokens(&b); + for w in toks.windows(2) { + if is_key(&w[1]) && is_prose(&w[0]) { + entries.entry(w[1].clone()).or_insert_with(|| w[0].clone()); + } + } + } + TextIndex { entries } + } + + /// The display string for a text key, if present. + pub fn get(&self, key: &str) -> Option<&str> { + self.entries.get(key).map(String::as_str) + } + + /// Number of indexed keys. + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The objective lines for a stage/phase, e.g. `("S01", 1)`. Empty if none + /// resolve (unused objective slots are simply absent on disc). + pub fn objectives(&self, stage: &str, phase: u32) -> Vec<&str> { + self.numbered(&format!("{stage}_P{phase}_Objective")) + } + + /// The lose-condition lines for a stage/phase. + pub fn lose_conditions(&self, stage: &str, phase: u32) -> Vec<&str> { + self.numbered(&format!("{stage}_P{phase}_Lose")) + } + + /// The hint lines for a stage/phase. + pub fn hints(&self, stage: &str, phase: u32) -> Vec<&str> { + self.numbered(&format!("{stage}_P{phase}_Hint")) + } + + /// A character's display name from its id (`CharacterRAYMOND` → `Raymond`). + pub fn character_name(&self, character_id: &str) -> Option<&str> { + self.get(&format!("CharacterName_{character_id}")) + } + + /// Collect a `_NN` run of lines while they resolve. + fn numbered(&self, base: &str) -> Vec<&str> { + (0..16) + .filter_map(|i| self.get(&format!("{base}_{i:02}"))) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tokenizes_mixed_runs() { + // "Hi" NUL "K_1" as UTF-16LE → two tokens. + let mut b = Vec::new(); + for s in ["Hi", "\0", "K_1x"] { + for c in s.chars() { + b.extend_from_slice(&(c as u16).to_le_bytes()); + } + } + let t = utf16_tokens(&b); + assert!(t.contains(&"Hi".to_string())); + assert!(t.contains(&"K_1x".to_string())); + } + + #[test] + fn key_and_prose_classify() { + assert!(is_key("S01_P1_Objective_00")); + assert!(is_key("CharacterName_CharacterRAYMOND")); + assert!(!is_key("Repel the attack.")); + assert!(is_prose("Repel the attack.")); + assert!(is_prose("Raymond")); + assert!(!is_prose("S01_P1_Objective_00")); + } + + // Disc-backed — skipped unless SYLPHEED_DISC is set. + #[test] + fn resolves_real_text() { + let Ok(disc) = std::env::var("SYLPHEED_DISC") else { return }; + let Ok(pak) = PakArchive::open(format!("{disc}/dat/GP_MAIN_GAME_E.pak")) else { return }; + let t = TextIndex::build(&pak); + assert!(t.len() > 5000, "expected a large index, got {}", t.len()); + assert_eq!(t.character_name("CharacterRAYMOND"), Some("Raymond")); + assert_eq!( + t.get("S01_P1_Objective_00"), + Some("Repel the enemy's surprise attack.") + ); + assert!(!t.objectives("S01", 1).is_empty()); + } +}