Syplheed-Reborn/tools/re-capture/sound_cues.py

#!/usr/bin/env python3
"""The sound-cue table in `dat/tables.pak`, and what a message id binds to.

    sound_cues.py summary                 # table shape and cue families
    sound_cues.py resolve MSG_VOICE_D_257 # message id -> cue -> id + bank file
    sound_cues.py demo                    # the cutscene message table
    sound_cues.py unbound                 # script message ids with no recording

One IDXD object in `dat/tables.pak` carries five records: `SETTINGS`,
`BANK_SE`, `FILES` (5 135 `.slb` bank paths), `STAGES` (empty) and `SOUNDS`
(5 798 named fields, cue name -> numeric sound id). Cue names are the join key:
a script message `MSG_<X>` plays the cue `VOICE_<X>`, and the bank file is
`jpn\\Voice\\VOICE_<X>.slb`. See docs/re/structures/sound-cue-table.md.
"""
import collections
import os
import re
import struct
import sys
import zlib

TABLES = '/work/sylph_extract/dat/tables.pak'


def be32(b, o):
    return struct.unpack_from('>I', b, o)[0]


def cstr(b, pool, off):
    base = pool + off
    return b[base:b.index(b'\0', base)].decode('latin1')


def pak_entries(path):
    d = open(path, 'rb').read()
    segs, base, i = b'', path[:-4], 0
    while os.path.exists('%s.p%02d' % (base, i)):
        segs += open('%s.p%02d' % (base, i), 'rb').read()
        i += 1
    out = {}
    for k in range(be32(d, 4)):
        h, off, cs = struct.unpack_from('>III', d, 0x10 + 12 * k)
        raw = segs[off:off + cs]
        if raw[:2] == b'Z1':
            try:
                raw = zlib.decompress(raw[10:])
            except Exception:
                pass
        out[h] = raw
    return out


def records(b):
    """IDXD -> {record name: [(field name or None, value)]}."""
    n = be32(b, 4)
    o = 8 + 16 * n
    recs = [struct.unpack_from('>IIII', b, 8 + 16 * i) for i in range(n)]
    m = be32(b, o)
    o += 4
    fl = [struct.unpack_from('>III', b, o + 12 * i) for i in range(m)]
    o += 12 * m
    pool = o + 4
    out = {}
    for (_nh, no, fb, fe) in recs:
        name = cstr(b, pool, no) if no != 0xFFFFFFFF else None
        out[name] = [(cstr(b, pool, fno) if fno != 0xFFFFFFFF else None,
                      cstr(b, pool, fvo)) for (_k, fno, fvo) in fl[fb:fe]]
    return out


def sound_table():
    obj = [v for v in pak_entries(TABLES).values()
           if v[:4] == b'IDXD' and b'DEMO_017' in v][0]
    r = records(obj)
    sounds = {k: v for k, v in r['SOUNDS']}
    files = [v for _, v in r['FILES']]
    return sounds, files, r


def cue_for(message_id):
    """`MSG_VOICE_D_257` -> `VOICE_D_257`; `MSG_TCAF_592` -> `VOICE_TCAF_592`.

    The `MSG_` prefix becomes `VOICE_`, except that a name already carrying
    `VOICE_` keeps the one it has rather than doubling it.
    """
    x = message_id[4:] if message_id.startswith('MSG_') else message_id
    return x if x.startswith('VOICE_') else 'VOICE_' + x


def wstr(b, pool, off):
    base = pool + 2 * off
    e = base
    while b[e:e + 2] != b'\0\0':
        e += 2
    return ''.join(chr(c) for c in
                   struct.unpack('>%dH' % ((e - base) // 2), b[base:e]))


def captions(lang='E'):
    """`MSG_*_<page>_<line>` -> text, from the pack's IXUD field tables."""
    pak = '/work/sylph_extract/dat/GP_MAIN_GAME_%s.pak' % lang
    out = {}
    for b in pak_entries(pak).values():
        if b[:4] != b'IXUD':
            continue
        n = be32(b, 4)
        o = 8 + 16 * n
        m = be32(b, o)
        o += 4
        fl = [struct.unpack_from('>III', b, o + 12 * i) for i in range(m)]
        o += 12 * m
        pool = o + 4
        for (_k, no, vo) in fl:
            if no == 0xFFFFFFFF:
                continue
            try:
                key, val = wstr(b, pool, no), wstr(b, pool, vo)
            except Exception:
                continue
            if key.startswith('MSG_') and val.strip():
                out.setdefault(key, val)
    return out


def demo_messages(lang='E'):
    """The cutscene message table: id -> per-page (speaker, face, dur, cue)."""
    pak = '/work/sylph_extract/dat/GP_MAIN_GAME_%s.pak' % lang
    out = {}
    for b in pak_entries(pak).values():
        if b[:4] != b'IDXD' or b'MSG_DEMO' not in b:
            continue
        for name, fs in records(b).items():
            if not name or not name.startswith('Message_'):
                continue
            named = {k: v for k, v in fs if k}
            pos = [v for k, v in fs if k is None]
            pc = int(named['PageCount'])
            out.setdefault(named['ID'], []).append(
                [tuple(pos[9 * p:9 * p + 9]) for p in range(pc)])
    return out


def main():
    what = sys.argv[1] if len(sys.argv) > 1 else 'summary'
    sounds, files, recs = sound_table()
    stems = {os.path.basename(f.replace('\\', '/'))[:-4]
             for f in files if f.lower().endswith('.slb')}

    if what == 'resolve':
        mid = sys.argv[2]
        c = cue_for(mid)
        print('message %s' % mid)
        print('  cue        %s' % c)
        print('  sound id   %s' % sounds.get(c, '<not in SOUNDS>'))
        hit = [f for f in files
               if os.path.basename(f.replace('\\', '/'))[:-4] == c]
        print('  bank file  %s' % (hit[0] if hit else '<no bank>'))
        return

    if what == 'demo':
        cap = captions()
        print('# Cutscene message table — dat/GP_MAIN_GAME_E.pak, IDXD objects')
        print('#')
        print('# One line per page. A page is one subtitle box; successive pages are')
        print('# successive utterances, which is why the speaker changes mid-message.')
        print('# Columns: id, page, speaker, portrait, on-screen seconds, audio cue,')
        print('# then the caption text.')
        print()
        for mid, variants in sorted(demo_messages().items()):
            for pages in variants:
                for i, g in enumerate(pages):
                    text = ' '.join(cap.get('%s_%03d_%02d' % (mid, i, l), '')
                                    for l in range(4)).strip()
                    print('%-16s p%d  %-14s %-18s %5ss  %-11s %s'
                          % (mid if i == 0 else '', i,
                             g[0].replace('Character', ''),
                             g[1].replace('Face', ''), g[3], g[4], text))
        return

    if what == 'unbound':
        for c in sorted(set(sounds) - stems):
            print('%-24s id=%s  (cue with no bank file)' % (c, sounds[c]))
        return

    print('records            : %s' % ', '.join(str(k) for k in recs))
    print('SOUNDS cues        : %d' % len(sounds))
    print('FILES bank paths   : %d  (distinct stems %d)' % (len(files), len(stems)))
    print('cues with no file  : %d' % len(set(sounds) - stems))
    print('files with no cue  : %d' % len(stems - set(sounds)))
    fam = collections.Counter(re.match(r'[A-Za-z]+', k).group(0) for k in sounds)
    print('\ncue families and their sound-id ranges:')
    rng = collections.defaultdict(list)
    for k, v in sounds.items():
        if v.lstrip('-').isdigit():
            rng[re.match(r'[A-Za-z]+', k).group(0)].append(int(v))
    for p, _ in fam.most_common():
        v = rng.get(p, [])
        if v:
            print('  %-8s n=%-5d %6d .. %6d' % (p, len(v), min(v), max(v)))


if __name__ == '__main__':
    main()