Syplheed-Reborn/crates/sylpheed-formats/src/movie_subtitle.rs

//! Movie cutscene subtitles: the movie → track → text chain.
//!
//! Reverse-engineered statically (see `docs/re/structures/movie-subtitles.md`).
//! A cutscene's on-screen captions are assembled from three places on the disc:
//!
//! 1. `dat/movie/<lang>.pak` holds one **timing track** per movie, keyed by
//!    [`track_key`] = `name_hash("subtitle_<movie>.tbl")`. Each track is a
//!    `Z1`+zlib block that decompresses to an **IXUD** container whose UTF-16**LE**
//!    payload is `SUBTITLE  MSG_DEMO_<demo>  <mm:ss.cc>  …` — i.e. *which*
//!    demo-message shows *when* (radio / resupply movies), or inline text.
//! 2. `dat/GP_MAIN_GAME_<L>.pak` holds the **caption text**: more `Z1`+zlib IXUD
//!    blocks where each line is stored as `text` immediately followed by its key
//!    `MSG_DEMO_<demo>_<page>_<line>` (multi-line captions split across lines).
//!
//! [`load`] joins the two into timed [`SubCue`]s.
//!
//! Note: the IXUD payload here is UTF-16 **little-endian** (verified on the real
//! disc); this is deliberately separate from the older big-endian [`crate::ixud`]
//! presenter, which targets a different (uncompressed) variant.

use std::collections::BTreeMap;

use crate::hash::name_hash;
use crate::pak::PakArchive;

/// Subtitle language. `pak_code` selects `dat/movie/<code>.pak`; `game_code`
/// selects `dat/GP_MAIN_GAME_<code>.pak` (the caption text).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SubLang {
    /// Default only because the disc's own default is English.
    #[default]
    English,
    Japanese,
    German,
    French,
    Spanish,
    Italian,
}

impl SubLang {
    /// All languages, in menu order.
    pub const ALL: [SubLang; 6] = [
        SubLang::English,
        SubLang::Japanese,
        SubLang::German,
        SubLang::French,
        SubLang::Spanish,
        SubLang::Italian,
    ];

    /// `dat/movie/<code>.pak` stem (the timing tracks + caption font).
    pub fn pak_code(self) -> &'static str {
        match self {
            SubLang::English => "eng",
            SubLang::Japanese => "jpn",
            SubLang::German => "deu",
            SubLang::French => "fra",
            SubLang::Spanish => "esp",
            SubLang::Italian => "ita",
        }
    }

    /// `dat/GP_MAIN_GAME_<code>.pak` suffix (the caption text pack).
    pub fn game_code(self) -> &'static str {
        match self {
            SubLang::English => "E",
            SubLang::Japanese => "J",
            SubLang::German => "D",
            SubLang::French => "F",
            SubLang::Spanish => "S",
            SubLang::Italian => "I",
        }
    }

    /// Human label for a UI selector. Kept to Latin script so it renders in a
    /// default (non-CJK) UI font; Japanese is romanized for the same reason.
    pub fn label(self) -> &'static str {
        match self {
            SubLang::English => "English",
            SubLang::Japanese => "Japanese",
            SubLang::German => "Deutsch",
            SubLang::French => "Français",
            SubLang::Spanish => "Español",
            SubLang::Italian => "Italiano",
        }
    }
}

/// One timed caption line.
#[derive(Debug, Clone, PartialEq)]
pub struct SubCue {
    pub start: f32,
    pub end: Option<f32>,
    pub text: String,
}

/// The `<lang>.pak` TOC key for a movie's subtitle timing track.
pub fn track_key(movie_basename: &str) -> u32 {
    name_hash(&format!("subtitle_{movie_basename}.tbl"))
}

/// Load and resolve a movie's timed captions in `lang`.
///
/// `lang_pak` is `dat/movie/<lang>.pak` (+segments); `text_pak` is
/// `dat/GP_MAIN_GAME_<L>.pak` (+segments). Returns the cues in start order, or an
/// empty vec if the movie has no timed track (e.g. story movies whose text is a
/// pre-rendered title card).
pub fn load(movie_basename: &str, lang_pak: &PakArchive, text_pak: &PakArchive) -> Vec<SubCue> {
    let Some(Ok(track)) = lang_pak.read_by_hash(track_key(movie_basename)) else {
        return Vec::new();
    };
    if !is_ixud(&track) {
        return Vec::new();
    }
    let raw = parse_track(&track);
    if raw.is_empty() {
        return Vec::new();
    }
    // Only build the (large) text table if the track actually references demos.
    let needs_text = raw.iter().any(|(t, _, _)| demo_ref(t).is_some());
    let text = if needs_text {
        build_demo_text(text_pak)
    } else {
        BTreeMap::new()
    };

    let mut cues = Vec::new();
    for (token, start, end) in raw {
        let resolved = match demo_ref(&token) {
            Some(demo) => match text.get(&demo) {
                Some(lines) => lines.join("\n"),
                None => continue, // demo id with no text on disc → skip
            },
            None => clean(&token), // already inline text
        };
        if resolved.trim().is_empty() {
            continue;
        }
        cues.push(SubCue {
            start,
            end,
            text: resolved,
        });
    }
    cues.sort_by(|a, b| a.start.partial_cmp(&b.start).unwrap_or(std::cmp::Ordering::Equal));
    cues
}

/// The `MSG_DEMO_<id>` ids a movie's timing track references, in track order
/// (empty for inline-text or absent tracks). Language-independent — the same
/// demo ids appear in every `<lang>.pak`. Used to share a voice clip between
/// movies that play the same demo line (the resupply cutscenes reuse one clip
/// per line while only the video differs).
pub fn track_demo_ids(lang_pak: &PakArchive, movie_basename: &str) -> Vec<u32> {
    let Some(Ok(track)) = lang_pak.read_by_hash(track_key(movie_basename)) else {
        return Vec::new();
    };
    if !is_ixud(&track) {
        return Vec::new();
    }
    parse_track(&track)
        .iter()
        .filter_map(|(t, _, _)| demo_ref(t))
        .collect()
}

/// Per-line `(demo id, start seconds)` for a movie's timing track — the radio-box
/// voice schedule. Each `MSG_DEMO_<id>` reference takes the start time of the
/// timing that follows it (a caption may list two demo lines under one timing;
/// both then share that start). Empty for inline-text / absent tracks (those are
/// story movies whose voice is the single manifest `VOICETRACK`, not per-line).
pub fn track_voice_cues(lang_pak: &PakArchive, movie_basename: &str) -> Vec<(u32, f32)> {
    let Some(Ok(track)) = lang_pak.read_by_hash(track_key(movie_basename)) else {
        return Vec::new();
    };
    if !is_ixud(&track) {
        return Vec::new();
    }
    let toks = utf16le_tokens(&track);
    let start = toks
        .iter()
        .position(|t| t.ends_with("SUBTITLE"))
        .map(|i| i + 1)
        .unwrap_or(0);
    let mut out = Vec::new();
    let mut pending: Vec<u32> = Vec::new();
    for tok in &toks[start..] {
        match parse_timing(tok) {
            Some((s, _)) => {
                for d in pending.drain(..) {
                    out.push((d, s));
                }
            }
            None => {
                if let Some(d) = demo_ref(tok) {
                    pending.push(d);
                }
            }
        }
    }
    out
}

/// Build `demo id → ordered caption lines` from a `GP_MAIN_GAME_<L>` pack.
///
/// Scans every IXUD block, pairing each text token with the immediately
/// following `MSG_DEMO_<demo>_<page>_<line>` key, and groups the lines per demo
/// ordered by (page, line).
pub fn build_demo_text(text_pak: &PakArchive) -> BTreeMap<u32, Vec<String>> {
    // demo -> (page,line) -> text
    let mut by_demo: BTreeMap<u32, BTreeMap<(u32, u32), String>> = BTreeMap::new();
    for entry in text_pak.entries() {
        let Ok(bytes) = text_pak.read(entry) else {
            continue;
        };
        if !is_ixud(&bytes) {
            continue;
        }
        let toks = utf16le_tokens(&bytes);
        for w in toks.windows(2) {
            if let Some((demo, page, line)) = text_key(&w[1]) {
                // Pair only when the preceding token is real text — not another
                // MSG_DEMO key (bare keys are also serialized consecutively in the
                // record directory, which would otherwise masquerade as text).
                if demo_ref(&w[0]).is_none()
                    && text_key(&w[0]).is_none()
                    && !w[0].trim().is_empty()
                {
                    by_demo
                        .entry(demo)
                        .or_default()
                        .insert((page, line), clean(&w[0]));
                }
            }
        }
    }
    by_demo
        .into_iter()
        .map(|(d, m)| (d, m.into_values().collect()))
        .collect()
}

/// Every caption family in the pack, not just the cutscene one.
///
/// `build_demo_text` reads `MSG_DEMO_*` — 560 text-bearing keys, the **smallest**
/// of eight families. The other seven carry the combat chatter and the in-mission
/// scripted dialogue: 44 019 more lines, or **98.7 %** of the game's text.
///
/// Key shapes, measured over every IXUD block in `GP_MAIN_GAME_E.pak`:
///
/// | family | shape | text-bearing keys |
/// |---|---|---|
/// | `ACRO` `ADAN` `ADPL` `BIRD` `DEMO` `RHIN` `TCAF` | `MSG_<FAM>_<id>_<page>_<line>` | 37 803 |
/// | `VOICE` | `MSG_VOICE_<letter>_<id>_<page>_<line>` | 6 776 |
///
/// `VOICE` is the only family with a letter before the id, and every family is
/// 100 % consistent with its own shape.
///
/// Returns `"<FAM>_<id>" → ordered lines`, e.g. `"ADAN_600"`, `"VOICE_A_150"`.
///
/// ⚠️ The id here is the **caption** id. It is *not* the voice-bank id: a message
/// page binding `VOICE_C_468` carries lines keyed `MSG_VOICE_C_385_*`. Same
/// family letter, different index space — do not derive one from the other.
pub fn build_caption_text(text_pak: &PakArchive) -> BTreeMap<String, Vec<String>> {
    let mut by_id: BTreeMap<String, BTreeMap<(u32, u32), String>> = BTreeMap::new();
    for entry in text_pak.entries() {
        let Ok(bytes) = text_pak.read(entry) else {
            continue;
        };
        let Some(obj) = crate::IxudObject::parse(&bytes) else {
            continue;
        };
        // Read the caption as a FIELD -- its key is the `MSG_*` name and its
        // value is the text. The token-adjacency version of this recovered 8074
        // lines against the 44579 fields that actually carry text, because pool
        // adjacency is a consequence of how records are written rather than a
        // rule of the format. Same mistake the IDXD reader made.
        for rec in obj.records() {
            for f in &rec.fields {
                let Some(key) = f.name.as_deref() else { continue };
                let Some((id, page, line)) = caption_key(key) else {
                    continue;
                };
                if f.value.trim().is_empty() {
                    continue;
                }
                by_id.entry(id).or_default().insert((page, line), clean(&f.value));
            }
        }
    }
    by_id
        .into_iter()
        .map(|(d, m)| (d, m.into_values().collect()))
        .collect()
}

/// `MSG_<FAM>_<id>_<page>_<line>` → `("<FAM>_<id>", page, line)`, with `VOICE`'s
/// extra family letter folded into the id.
fn caption_key(t: &str) -> Option<(String, u32, u32)> {
    let rest = t.strip_prefix("MSG_")?;
    let mut parts: Vec<&str> = rest.split('_').collect();
    // Trailing <page>_<line> are always numeric.
    let line: u32 = parts.pop()?.parse().ok()?;
    let page: u32 = parts.pop()?.parse().ok()?;
    // What remains is <FAM> or <FAM>_<letter>, then the numeric id.
    let id: u32 = parts.pop()?.parse().ok()?;
    if parts.is_empty() {
        return None;
    }
    Some((format!("{}_{id:03}", parts.join("_")), page, line))
}

/// Parse a timing track's IXUD payload into `(token, start, end)` triples, where
/// `token` is either a `MSG_DEMO_<d>` reference or an inline caption string.
///
/// A single on-screen caption may be stored as **several consecutive text
/// tokens** followed by one timing (the source hard-splits a multi-line caption
/// into one token per line — e.g. `"Look at it father"` + `"& beautiful isn't
/// it"` share the `01:14.80-01:17.60` timing). We therefore accumulate every
/// text token seen since the last timing and join them with a newline when the
/// timing arrives, instead of pairing strictly 1:1 (which silently dropped every
/// line but the last of a multi-line caption).
fn parse_track(ixud: &[u8]) -> Vec<(String, f32, Option<f32>)> {
    let toks = utf16le_tokens(ixud);
    let start = toks
        .iter()
        .position(|t| t.ends_with("SUBTITLE"))
        .map(|i| i + 1)
        .unwrap_or(0);
    let rest = &toks[start..];
    let mut out = Vec::new();
    let mut pending: Vec<&str> = Vec::new();
    for tok in rest {
        match parse_timing(tok) {
            Some((s, e)) => {
                if !pending.is_empty() {
                    out.push((pending.join("\n"), s, e));
                    pending.clear();
                }
            }
            None => pending.push(tok),
        }
    }
    out
}

fn is_ixud(b: &[u8]) -> bool {
    b.len() >= 4 && &b[0..4] == b"IXUD"
}

/// Normalize a caption string: the source stores line breaks as the literal
/// two-character escape `\n`; turn it into a real newline and trim.
fn clean(s: &str) -> String {
    s.replace("\\n", "\n").trim().to_string()
}

/// `MSG_DEMO_<d>` → `Some(d)` (reference token, no page/line).
fn demo_ref(t: &str) -> Option<u32> {
    let rest = t.strip_prefix("MSG_DEMO_")?;
    if rest.contains('_') {
        return None; // that's a text key (has page/line), not a bare reference
    }
    rest.parse().ok()
}

/// `MSG_DEMO_<d>_<page>_<line>` → `Some((d,page,line))` (text key).
fn text_key(t: &str) -> Option<(u32, u32, u32)> {
    let rest = t.strip_prefix("MSG_DEMO_")?;
    let mut it = rest.split('_');
    let d = it.next()?.parse().ok()?;
    let p = it.next()?.parse().ok()?;
    let l = it.next()?.parse().ok()?;
    if it.next().is_some() {
        return None;
    }
    Some((d, p, l))
}

/// Extract UTF-16**LE** text runs from a blob, **independent of byte alignment**.
///
/// The IXUD string pool packs records at odd byte offsets, so a fixed 2-byte
/// stride from the block start misreads every character. Instead we scan a
/// sliding window, decoding each 16-bit LE unit and keeping runs of "text-like"
/// code points (ASCII, Latin-1 supplement — the German umlauts / accented
/// Latin — and Latin Extended-A). A non-text unit ends the run and we advance by
/// one byte to re-lock onto the correct parity of the next run. Accepting the
/// Latin ranges (not just ASCII) is what keeps `müsste`, `Français`, `español`
/// intact — earlier the accented char *and its predecessor* were dropped.
///
/// CJK (Japanese) is deliberately out of range: those units also occur as noise
/// at the wrong parity, and the UI font can't render them anyway.
fn utf16le_tokens(bytes: &[u8]) -> Vec<String> {
    let mut tokens = Vec::new();
    let mut cur = String::new();
    let mut i = 0;
    while i + 1 < bytes.len() {
        let u = u16::from_le_bytes([bytes[i], bytes[i + 1]]);
        if is_text_unit(u) {
            if let Some(c) = char::from_u32(u as u32) {
                cur.push(c);
            }
            i += 2;
        } else {
            if cur.chars().count() >= 2 {
                tokens.push(std::mem::take(&mut cur));
            } else {
                cur.clear();
            }
            i += 1;
        }
    }
    if cur.chars().count() >= 2 {
        tokens.push(cur);
    }
    tokens
}

/// Whether a UTF-16 unit is a printable Latin text character we keep in a run.
/// Excludes the C0/C1 control blocks (incl. 0x7F–0x9F) so binary noise breaks
/// runs instead of joining them.
fn is_text_unit(u: u16) -> bool {
    matches!(u, 0x20..=0x7e | 0xa0..=0xff | 0x100..=0x17f)
}

/// Parse `MM:SS.ss` or a `start-end` range into seconds.
fn parse_timing(s: &str) -> Option<(f32, Option<f32>)> {
    let one = |p: &str| -> Option<f32> {
        let (mm, ss) = p.trim().split_once(':')?;
        let m: f32 = if mm.trim().is_empty() {
            0.0
        } else {
            mm.trim().parse().ok()?
        };
        let sec: f32 = ss.trim().parse().ok()?;
        Some(m * 60.0 + sec)
    };
    match s.split_once('-') {
        Some((a, b)) => Some((one(a)?, Some(one(b)?))),
        None => Some((one(s)?, None)),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn track_key_matches_disc() {
        // Verified against dat/movie/eng.pak.
        assert_eq!(track_key("S00A"), 0x6F2D_9663);
        assert_eq!(track_key("hokyu_DS_s02A"), 0x3662_B1F8);
        assert_eq!(track_key("RT01C_1"), 0x756F_69FB);
    }

    #[test]
    fn demo_and_text_keys() {
        assert_eq!(demo_ref("MSG_DEMO_192"), Some(192));
        assert_eq!(demo_ref("MSG_DEMO_604_000_00"), None);
        assert_eq!(text_key("MSG_DEMO_604_000_01"), Some((604, 0, 1)));
        assert_eq!(text_key("MSG_DEMO_192"), None);
    }

    #[test]
    fn keeps_latin1_accents() {
        // "müsste" must survive intact (umlaut + its neighbour), not collapse to
        // "sste". Encode as UTF-16LE at an ODD start offset to exercise the
        // alignment-independent scan.
        let mut b = vec![0xAAu8]; // 1 junk byte → strings start at an odd offset
        for s in ["müsste", "Français", "español"] {
            for u in s.encode_utf16() {
                b.extend_from_slice(&u.to_le_bytes());
            }
            b.extend_from_slice(&[0, 0]);
        }
        let toks = utf16le_tokens(&b);
        assert!(toks.contains(&"müsste".to_string()), "got {toks:?}");
        assert!(toks.contains(&"Français".to_string()), "got {toks:?}");
        assert!(toks.contains(&"español".to_string()), "got {toks:?}");
    }

    /// Build a synthetic UTF-16LE IXUD and check the LE token walk + timing pair.
    #[test]
    fn parses_le_track() {
        let mut b = b"IXUD".to_vec();
        b.extend_from_slice(&[0, 1, 0, 0, 0x70, 0x3E, 0xC8, 0x6C]);
        for t in ["SUBTITLE", "MSG_DEMO_5", "00:01.50", "Hello", "00:03.00"] {
            for u in t.encode_utf16() {
                b.extend_from_slice(&u.to_le_bytes());
            }
            b.extend_from_slice(&[0, 0]);
        }
        let raw = parse_track(&b);
        assert_eq!(raw.len(), 2);
        assert_eq!(raw[0], ("MSG_DEMO_5".to_string(), 1.5, None));
        assert_eq!(raw[1], ("Hello".to_string(), 3.0, None));
    }

    /// Two text tokens before a single timing = one two-line caption; the first
    /// line must NOT be dropped (the real S13A `Look at it father` regression).
    #[test]
    fn joins_multiline_caption() {
        let mut b = b"IXUD".to_vec();
        b.extend_from_slice(&[0, 1, 0, 0, 0x70, 0x3E, 0xC8, 0x6C]);
        for t in [
            "SUBTITLE",
            "That is such magnificent power.",
            "01:09.70-01:12.20",
            "Look at it father",
            "& beautiful isn't it",
            "01:14.80-01:17.60",
        ] {
            for u in t.encode_utf16() {
                b.extend_from_slice(&u.to_le_bytes());
            }
            b.extend_from_slice(&[0, 0]);
        }
        let raw = parse_track(&b);
        assert_eq!(raw.len(), 2);
        assert_eq!(raw[0].0, "That is such magnificent power.");
        assert_eq!(
            raw[1].0, "Look at it father\n& beautiful isn't it",
            "both lines of the caption must be kept"
        );
        assert_eq!((raw[1].1, raw[1].2), (74.8, Some(77.6)));
    }
}