Sylpheed/tools/re-capture/preset_messages.py

#!/usr/bin/env python3
"""Regenerate docs/re/data/preset-messages.txt.

The reactive combat-chatter system: a per-character *rule* table
(`message\\PresetMessage_<who>[_<NN>][_S<stage>-<phase>].tbl`) holding 64 event
records plus a `Sperkers` speaker roster, paired with a `_msg.tbl` message table
in the layout cutscene-message-table.md already documents.

Run:  python3 preset_messages.py > ../../docs/re/data/preset-messages.txt
"""
import sys, os, re, glob, collections
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from unit_substructures import pak_entries
import unitgroup as U

DAT = '/work/sylph_extract/dat'
PAK = os.path.join(DAT, 'GP_MAIN_GAME_E.pak')
MARKER = 'ORDOR_SQUADRON_EXTENDED'
STAGES = ['S%02d' % i for i in list(range(1, 17)) + list(range(18, 30))]


def idxd_entries(path):
    out = {}
    for h, s in pak_entries(path):
        if s[:4] == b'IDXD':
            out[h] = s
    return out


def cue_names():
    for h, s in pak_entries(os.path.join(DAT, 'tables.pak')):
        if s[:4] != b'IDXD':
            continue
        try:
            recs = U.parse(s)
        except Exception:
            continue
        for r in recs:
            if r['squadron'] == 'SOUNDS':
                return {n for _, n, _ in r['fields'] if n is not None}
    return set()


def to_cue(m):
    """sound-cue-table.md's rule: MSG_ becomes VOICE_, but a name that already
    carries VOICE_ keeps the one it has rather than doubling it."""
    if not m.startswith('MSG_'):
        return m
    rest = m[4:]
    return rest if rest.startswith('VOICE_') else 'VOICE_' + rest


def main():
    entries = idxd_entries(PAK)
    rule = {}
    for h, s in entries.items():
        try:
            recs = U.parse(s)
        except Exception:
            continue
        if any(r['squadron'] == MARKER for r in recs):
            rule[h] = recs

    # how many language packs carry the same set
    packs = sorted(glob.glob(os.path.join(DAT, 'GP_MAIN_GAME_?.pak')))
    percopy = {}
    for p in packs:
        n = 0
        for h, s in pak_entries(p):
            if s[:4] == b'IDXD' and MARKER.encode() in s:
                n += 1
        percopy[os.path.basename(p)] = n

    print('# Reactive combat-chatter rule tables (regenerated by preset_messages.py)')
    print()
    print('rule tables in GP_MAIN_GAME_E.pak : %d of %d IDXD entries' % (len(rule), len(entries)))
    print('per language pack                 : %s' % percopy)
    print('total across the disc             : %d' % sum(percopy.values()))

    # one record-name shape?
    shapes = collections.Counter(tuple(r['squadron'] for r in recs) for recs in rule.values())
    print('distinct record-name tuples       : %d' % len(shapes))
    shape = shapes.most_common(1)[0][0]
    print('records per table                 : %d  (64 events + Sperkers)' % len(shape))

    # named-field schema, and the MessageCount identity
    schemas = collections.Counter()
    ok = bad = 0
    msgs = set()
    flags = collections.Counter()
    perevent = collections.Counter()
    for recs in rule.values():
        for r in recs:
            if r['squadron'] == 'Sperkers':
                continue
            names = tuple(n for _, n, _ in r['fields'] if n is not None)
            schemas[names] += 1
            pos = [v for _, n, v in r['fields'] if n is None]
            try:
                mc = int(U.named(r)['MessageCount'])
            except Exception:
                mc = -1
            if mc * 2 == len(pos):
                ok += 1
            else:
                bad += 1
            if pos:
                perevent[r['squadron']] += 1
            for i, v in enumerate(pos):
                if i % 2:
                    flags[v] += 1
                else:
                    msgs.add(v)
    print()
    print('distinct named-field schemas      : %d' % len(schemas))
    for names, c in schemas.most_common():
        print('  x%-6d %s' % (c, ', '.join(names)))
    print('MessageCount*2 == positional count: %d/%d  (mismatches %d)' % (ok, ok + bad, bad))
    print('second-of-pair values             : %s' % dict(flags))

    print()
    print('the 64 event records, and how many of the %d tables give each one lines:' % len(rule))
    for name in sorted(n for n in shape if n != 'Sperkers'):
        print('  %-32s %3d' % (name, perevent[name]))

    # speakers
    spk = collections.Counter()
    per = collections.Counter()
    for recs in rule.values():
        for r in recs:
            if r['squadron'] == 'Sperkers':
                names = [n for _, n, _ in r['fields'] if n is not None]
                per[len(names)] += 1
                for n in names:
                    spk[n] += 1
    print()
    print('Sperkers roster: %d distinct speakers; sizes %s' % (
        len(spk), dict(sorted(per.items()))))
    for n, c in spk.most_common():
        print('  %-28s %3d' % (n, c))

    # the join into the sound-cue table
    cues = cue_names()
    hit = sorted(m for m in msgs if to_cue(m) in cues)
    miss = sorted(m for m in msgs if to_cue(m) not in cues)
    print()
    print('distinct message ids               : %d' % len(msgs))
    print('resolved in tables.pak SOUNDS      : %d  (%.1f%%)' % (
        len(hit), 100.0 * len(hit) / max(1, len(msgs))))
    print('unresolved                         : %d' % len(miss))
    fam = collections.Counter(m.split('_')[1] for m in miss if m.count('_') >= 2)
    print('  by family: %s' % dict(fam))
    for m in miss:
        print('  %s' % m)

    # naming, two independent routes
    harvest = set()
    for p in sorted(glob.glob(os.path.join(DAT, '**', '*.pak'), recursive=True)):
        for h, s in pak_entries(p):
            if s[:4] != b'IDXD':
                continue
            try:
                recs = U.parse(s)
            except Exception:
                continue
            for r in recs:
                harvest.add(r['squadron'])
                for _, n, v in r['fields']:
                    if n:
                        harvest.add(n)
                    harvest.add(v)
    byharvest = {}
    byderived = {}
    for c in harvest:
        if not c.startswith('PresetMessage_'):
            continue
        byharvest.setdefault(U.name_hash('message\\' + c), 'message\\' + c)
        # route 3: a message table's name gives its rule table's name
        if c.endswith('_msg.tbl'):
            b = 'message\\' + c[:-len('_msg.tbl')] + '.tbl'
            byderived.setdefault(U.name_hash(b), b)

    declared = collections.Counter()
    ums_found = []
    for st in STAGES:
        h = U.name_hash('message\\UnitMessageSet_%s.tbl' % st)
        if h not in entries:
            continue
        ums_found.append(st)
        for r in U.parse(entries[h]):
            d = U.named(r)
            for k in ('PresetMessage_Phase1', 'PresetMessage_Phase2', 'PresetMessage_Phase3'):
                if d.get(k):
                    declared[d[k]] += 1
    bydecl = {}
    for v in declared:
        bydecl.setdefault(U.name_hash('message\\' + v), 'message\\' + v)

    # route 4: names predicted from the Sperkers roster before they were hashed.
    # TCAF_Fleet<X><Y> speaks (TCAFOP<Y>, TCAF<X>); TCAF_..Ship<X> speaks
    # (ADANPL<X>, TCAF<X>).  Six gaps in two series, all six confirmed.
    PREDICTED = ['TCAF_FleetBB', 'TCAF_17thFleetBB', 'TCAF_17thFleetCA',
                 'TCAF_17thFleetCB', 'TCAF_17thFleetShipA', 'TCAF_17thFleetShipB']
    # same-shaped names that the series does NOT contain, as the control
    CONTROL = ['TCAF_FleetCC', 'TCAF_17thFleetCC', 'TCAF_ShipA_01',
               'TCAF_17thFleetShipC_01']
    bypred = {}
    for c in PREDICTED:
        n = 'message\\PresetMessage_%s.tbl' % c
        bypred.setdefault(U.name_hash(n), n)

    A = {h: n for h, n in byharvest.items() if h in rule}
    B = {h: n for h, n in bydecl.items() if h in rule}
    C = {h: n for h, n in byderived.items() if h in rule}
    D = {h: n for h, n in bypred.items() if h in rule}
    print()
    print('naming route 1 (any harvested PresetMessage_ string): %d/%d' % (len(A), len(rule)))
    print('naming route 2 (UnitMessageSet_S* declarations)     : %d/%d' % (len(B), len(rule)))
    print('naming route 3 (strip _msg from a message table)    : %d/%d' % (len(C), len(rule)))
    print('naming route 4 (predicted from the Sperkers roster) : %d/%d' % (len(D), len(rule)))
    print('routes 1 and 2 agree on the same set                : %s' % (set(A) == set(B)))
    print('route 3 is a superset of route 1                    : %s' % set(A).issubset(C))
    print('route 3 names nothing that is not a rule table      : %s' % all(
        h in rule for h in byderived if h in entries))
    print('CONTROL — same-shaped names outside the series, hits : %d/%d' % (
        sum(1 for c in CONTROL if U.name_hash('message\\PresetMessage_%s.tbl' % c) in rule),
        len(CONTROL)))
    A = dict(A); A.update(C); A.update(D)
    # route 5: sweep the naming grammar the other 143 names describe,
    # PresetMessage_<who>[_NN][_S<nn>-<p>].tbl, over every <who> already seen.
    whos = set()
    for n in A.values():
        base = n.rsplit('\\', 1)[-1]
        m = re.match(r'PresetMessage_(.+?)(?:_\d\d)?(?:_S\d\d-\d)?\.tbl$', base)
        if m:
            whos.add(m.group(1))
    stages = ['_S%02d-%d' % (st, ph) for st in range(1, 30) for ph in range(1, 5)]
    E = {}
    for w in sorted(whos):
        for nn in [''] + ['_%02d' % i for i in range(1, 21)]:
            for st in [''] + stages:
                n = 'message\\PresetMessage_%s%s%s.tbl' % (w, nn, st)
                h = U.name_hash(n)
                if h in rule and h not in A:
                    E[h] = n
    print('naming route 5 (sweep of the grammar, %d <who> tokens): %d/%d' % (
        len(whos), len(E), len(rule)))
    A.update(E)
    print('UNION of all five routes                            : %d/%d' % (len(A), len(rule)))
    missing_msg = [n for h, n in A.items()
                   if U.name_hash(n[:-4] + '_msg.tbl') not in entries]
    print('rule tables whose _msg companion is absent          : %d' % len(missing_msg))
    print('UnitMessageSet_S* tables present: %d of %d  (%s)' % (
        len(ums_found), len(STAGES), ' '.join(ums_found)))
    print('missing: %s' % ' '.join(s for s in STAGES if s not in ums_found))
    print('distinct declared PresetMessage tables: %d' % len(declared))

    # control: the same prefix also names a DIFFERENT shape
    other = {h: n for h, n in byharvest.items() if h in entries and h not in rule}
    print()
    print('CONTROL — PresetMessage_* names that hit an IDXD entry but are NOT rule tables: %d' % len(other))
    oshape = collections.Counter()
    for h in other:
        try:
            recs = U.parse(entries[h])
        except Exception:
            continue
        oshape[tuple(sorted({r['squadron'].rstrip('0123456789') for r in recs}))] += 1
    print('  their record-name families: %s' % dict(oshape))

    print()
    print('rule tables, by name:')
    for h in sorted(rule, key=lambda k: A.get(k, '~%#010x' % k)):
        print('  %#010x  %s' % (h, A.get(h, '<unnamed>')))
    un = [h for h in rule if h not in A]
    print()
    print('unnamed rule tables: %d' % len(un))


if __name__ == '__main__':
    main()