Sylpheed/crates/sylpheed-formats/examples/voice_stream_cue_map.rs

//! Who owns the bytes in front of a movie-voice region's first `RIFF`?
//!
//! [`voice-region-leading-chunk.md`] left one thing open: the leading chunk of
//! the 17 stream-opening regions is real XMA audio that **no other movie-voice
//! region claims** — but the census only enumerated the 95 movie cues, while the
//! same continuous stream also carries the in-mission `VOICE_D_*` lines. The
//! leading hypothesis was that the bytes belong to one of those, and the port
//! pointed out that the byte-span test already written settles it *without
//! anyone listening* if the enumeration is widened.
//!
//! So this widens it the whole way: rather than resolving cues one at a time
//! through the manifest, scan the stream itself for **every** trailer descriptor
//! — the `(id: u32be, 0x11, …)` pair whose id repeats at `+0x800`, which
//! `movie_voice` documents as the end of a cue's audio. Cue N's audio is
//! `[descriptor(N-1) .. descriptor(N)]`, so the full descriptor list IS the
//! complete cue partition of the stream, movie and mission alike.
//!
//!     cargo run -p sylpheed-formats --example voice_stream_cue_map -- <disc-dir>
use sylpheed_formats::media::{self, DirectorySource, DiscSource};
use sylpheed_formats::slb::VoiceLang;

const DESC_MARK: u32 = 0x11;
const DESC_REPEAT: usize = 0x800;
const ID_MAX: u32 = 0x1_0000;

/// Every trailer descriptor in `buf`, as `(offset, id)`.
///
/// Same predicate `movie_voice::find_descriptor` uses — the id-repeat at +0x800
/// is what makes a false match inside XMA audio ~2^-64.
fn all_descriptors(buf: &[u8]) -> Vec<(usize, u32)> {
    let be = |o: usize| u32::from_be_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]);
    let mut out = Vec::new();
    if buf.len() < DESC_REPEAT + 8 {
        return out;
    }
    let end = buf.len() - (DESC_REPEAT + 4);
    let mut o = 0;
    while o <= end {
        let id = be(o);
        if (1..ID_MAX).contains(&id) && be(o + 4) == DESC_MARK && be(o + DESC_REPEAT) == id {
            out.push((o, id));
        }
        o += 4;
    }
    out
}

fn main() {
    let disc = std::env::args()
        .nth(1)
        .unwrap_or_else(|| std::env::var("SYLPHEED_DISC").expect("disc dir"));
    let src = DirectorySource::new(std::path::PathBuf::from(&disc));

    // The cue-name -> id registry, so a descriptor id can be named.
    let tpak = src.open_pak("dat/tables.pak").expect("tables.pak");
    let marker = "eng\\Movie\\VOICE_ADV.slb";
    let registry = tpak
        .entries()
        .iter()
        .find_map(|e| {
            tpak.read(e)
                .ok()
                .filter(|b| b.windows(marker.len()).any(|w| w == marker.as_bytes()))
        })
        .expect("voice registry");
    let ids = sylpheed_formats::movie_voice::registry_voice_ids(&registry);
    let name_of: std::collections::HashMap<u32, String> =
        ids.iter().map(|(n, &i)| (i, n.clone())).collect();
    println!(
        "registry: {} cue names, {} distinct ids",
        ids.len(),
        name_of.len()
    );

    // The 17 regions that open with a headerless stream, from the manifest.
    let movies: Vec<String> = {
        use sylpheed_formats::movie_manifest;
        let manifest = tpak
            .entries()
            .iter()
            .find_map(|e| tpak.read(e).ok().filter(|b| movie_manifest::is_manifest(b)))
            .expect("manifest");
        movie_manifest::parse(&manifest)
            .into_iter()
            .map(|m| m.movie)
            .collect()
    };
    let mut regions = Vec::new();
    for m in &movies {
        if let Some((s, e)) = media::resolve_movie_voice_region(&src, m, VoiceLang::English) {
            regions.push((m.clone(), s, e));
        }
    }
    let lo = regions.iter().map(|r| r.1).min().unwrap();
    let hi = regions.iter().map(|r| r.2).max().unwrap();
    // Scan a window covering every region, with margin for cues either side.
    let win_start = lo.saturating_sub(8 * 1024 * 1024) & !3;
    let win_len = (hi - win_start + 8 * 1024 * 1024) as usize;
    println!(
        "scanning dat/sound {win_start}..{} ({:.1} MB)",
        win_start + win_len as u64,
        win_len as f64 / 1e6
    );
    let buf = src
        .read_segment_range("dat/sound", win_start, win_len)
        .expect("stream window");
    let descs = all_descriptors(&buf);
    println!("{} trailer descriptors found\n", descs.len());

    let named = descs
        .iter()
        .filter(|(_, id)| name_of.contains_key(id))
        .count();
    println!(
        "  of those, {named} carry an id the registry names, {} do not\n",
        descs.len() - named
    );

    // For each stream-opening region, name the cue that OWNS the leading span:
    // the cue whose [prev_desc .. desc] interval contains it.
    println!("leading span -> owning cue\n");
    let mut verdicts: std::collections::BTreeMap<&str, usize> = Default::default();
    for (m, s, e) in &regions {
        let Ok(bytes) = src.read_segment_range("dat/sound", *s, (e - s) as usize) else {
            continue;
        };
        let Some(fr) = bytes.windows(4).position(|w| w == b"RIFF") else {
            continue;
        };
        if fr == 0 || sylpheed_formats::slb::bank_header_len(&bytes) == Some(fr) {
            continue; // bank-header case, not ours
        }
        let (a, b) = (*s, *s + fr as u64); // the leading span, global offsets
                                           // Descriptors bracketing the MIDDLE of the leading span.
        let mid = (a + b) / 2;
        let mid_local = (mid - win_start) as usize;
        let before = descs
            .iter()
            .rev()
            .find(|(o, _)| (*o as u64) < mid_local as u64);
        let after = descs.iter().find(|(o, _)| *o >= mid_local);
        let owner = after.map(|(_, id)| *id);
        let owner_name = owner
            .and_then(|id| name_of.get(&id).cloned())
            .unwrap_or_else(|| {
                owner
                    .map(|i| format!("<unnamed id {i}>"))
                    .unwrap_or("<none>".into())
            });
        let kind = if owner_name.starts_with("VOICE_D_") {
            "MISSION line"
        } else if owner_name.starts_with("VOICE_") {
            "movie cue"
        } else {
            "unknown"
        };
        *verdicts.entry(kind).or_default() += 1;
        println!(
            "  {m:8} lead {:8} B  bracketed by desc@{:?} .. desc@{:?}  owner {owner_name} [{kind}]",
            b - a,
            before.map(|(o, i)| (*o as u64 + win_start, *i)),
            after.map(|(o, i)| (*o as u64 + win_start, *i)),
        );
    }
    println!("\nverdicts: {verdicts:?}");

    // WHY do exactly these 17 open mid-cue? `resolve_movie_voice_region` takes
    // the predecessor trailer as the region start, but guards it with
    // `end - start < 1_500_000` and falls back to the .slb TOC anchor when that
    // fails. If the guard is the cause, then the stream-opening regions are
    // exactly the cues whose true span exceeds the guard.
    println!("\ncue span vs the 1.5 MB guard, and what the region actually starts at:\n");
    let (mut over, mut under, mut over_is_stream, mut under_is_stream) = (0, 0, 0, 0);
    for (m, s, e) in &regions {
        let Ok(bytes) = src.read_segment_range("dat/sound", *s, (e - s) as usize) else {
            continue;
        };
        let fr = bytes.windows(4).position(|w| w == b"RIFF");
        let is_stream = matches!(fr, Some(f) if f > 0
            && sylpheed_formats::slb::bank_header_len(&bytes) != Some(f));
        // The true predecessor trailer for this cue, from the full descriptor list.
        let end_local = (*e - win_start) as usize;
        let prev = descs
            .iter()
            .rev()
            .find(|(o, _)| *o < end_local)
            .map(|(o, _)| *o as u64 + win_start);
        let Some(prev) = prev else { continue };
        let span = e - prev;
        let guarded = span >= 1_500_000;
        if guarded {
            over += 1;
            if is_stream {
                over_is_stream += 1
            }
        } else {
            under += 1;
            if is_stream {
                under_is_stream += 1
            }
        }
        if is_stream {
            println!(
                "  {m:8} true cue span {span:8} B (> guard: {guarded})  region starts at {s}, \
true start {prev}  -> {} B of the cue's own audio is OUTSIDE the region",
                s.saturating_sub(prev)
            );
        }
    }
    println!("\ncues over the 1.5 MB guard: {over}, of which stream-opening: {over_is_stream}");
    println!("cues under the guard:      {under}, of which stream-opening: {under_is_stream}");

    // How many streams is ONE cue stored as? The port measured that a region's
    // leading chunk is the TAIL of its first full-length chunk, i.e. the cue is
    // re-presented. Structurally that predicts a fixed number of stream starts
    // inside a cue's TRUE span [desc(N-1) .. desc(N)] -- which is measurable
    // from the bytes alone, with no decoder.
    println!("\nstream starts inside each cue's TRUE span (desc(N-1)..desc(N)):\n");
    let mut hist: std::collections::BTreeMap<usize, usize> = Default::default();
    let mut hist_long: std::collections::BTreeMap<usize, usize> = Default::default();
    for w in descs.windows(2) {
        let (a, b) = (w[0].0, w[1].0);
        if b <= a || b - a < 4096 {
            continue;
        }
        let span = &buf[a..b];
        // A stream start is a RIFF; plus the run before the first one, when it
        // is not a bank header, is itself a stream.
        let riffs = span
            .windows(4)
            .enumerate()
            .filter(|(_, w)| *w == b"RIFF")
            .count();
        let lead_is_stream = match span.windows(4).position(|w| w == b"RIFF") {
            Some(f) if f > 0 => sylpheed_formats::slb::bank_header_len(span) != Some(f),
            _ => false,
        };
        let streams = riffs + usize::from(lead_is_stream);
        *hist.entry(streams).or_default() += 1;
        if b - a >= 1_500_000 {
            *hist_long.entry(streams).or_default() += 1;
        }
    }
    println!("  all inter-descriptor spans:  {hist:?}");
    println!("  spans >= 1.5 MB (the long cues): {hist_long:?}");
}