#!/usr/bin/env python3
"""Regenerate docs/re/data/preset-messages.txt.

The reactive combat-chatter system: a per-character *rule* table
(`message\\PresetMessage_<who>[_<NN>][_S<stage>-<phase>].tbl`) holding 64 event
records plus a `Sperkers` speaker roster, paired with a `_msg.tbl` message table
in the layout cutscene-message-table.md already documents.

Run:  python3 preset_messages.py > ../../docs/re/data/preset-messages.txt
"""
import sys, os, re, glob, collections
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from unit_substructures import pak_entries
import unitgroup as U

DAT = '/work/sylph_extract/dat'
PAK = os.path.join(DAT, 'GP_MAIN_GAME_E.pak')
MARKER = 'ORDOR_SQUADRON_EXTENDED'
STAGES = ['S%02d' % i for i in list(range(1, 17)) + list(range(18, 30))]


def idxd_entries(path):
    out = {}
    for h, s in pak_entries(path):
        if s[:4] == b'IDXD':
            out[h] = s
    return out


def cue_names():
    for h, s in pak_entries(os.path.join(DAT, 'tables.pak')):
        if s[:4] != b'IDXD':
            continue
        try:
            recs = U.parse(s)
        except Exception:
            continue
        for r in recs:
            if r['squadron'] == 'SOUNDS':
                return {n for _, n, _ in r['fields'] if n is not None}
    return set()


def to_cue(m):
    """sound-cue-table.md's rule: MSG_ becomes VOICE_, but a name that already
    carries VOICE_ keeps the one it has rather than doubling it."""
    if not m.startswith('MSG_'):
        return m
    rest = m[4:]
    return rest if rest.startswith('VOICE_') else 'VOICE_' + rest


def main():
    entries = idxd_entries(PAK)
    rule = {}
    for h, s in entries.items():
        try:
            recs = U.parse(s)
        except Exception:
            continue
        if any(r['squadron'] == MARKER for r in recs):
            rule[h] = recs

    # how many language packs carry the same set
    packs = sorted(glob.glob(os.path.join(DAT, 'GP_MAIN_GAME_?.pak')))
    percopy = {}
    for p in packs:
        n = 0
        for h, s in pak_entries(p):
            if s[:4] == b'IDXD' and MARKER.encode() in s:
                n += 1
        percopy[os.path.basename(p)] = n

    print('# Reactive combat-chatter rule tables (regenerated by preset_messages.py)')
    print()
    print('rule tables in GP_MAIN_GAME_E.pak : %d of %d IDXD entries' % (len(rule), len(entries)))
    print('per language pack                 : %s' % percopy)
    print('total across the disc             : %d' % sum(percopy.values()))

    # one record-name shape?
    shapes = collections.Counter(tuple(r['squadron'] for r in recs) for recs in rule.values())
    print('distinct record-name tuples       : %d' % len(shapes))
    shape = shapes.most_common(1)[0][0]
    print('records per table                 : %d  (64 events + Sperkers)' % len(shape))

    # named-field schema, and the MessageCount identity
    schemas = collections.Counter()
    ok = bad = 0
    msgs = set()
    flags = collections.Counter()
    perevent = collections.Counter()
    for recs in rule.values():
        for r in recs:
            if r['squadron'] == 'Sperkers':
                continue
            names = tuple(n for _, n, _ in r['fields'] if n is not None)
            schemas[names] += 1
            pos = [v for _, n, v in r['fields'] if n is None]
            try:
                mc = int(U.named(r)['MessageCount'])
            except Exception:
                mc = -1
            if mc * 2 == len(pos):
                ok += 1
            else:
                bad += 1
            if pos:
                perevent[r['squadron']] += 1
            for i, v in enumerate(pos):
                if i % 2:
                    flags[v] += 1
                else:
                    msgs.add(v)
    print()
    print('distinct named-field schemas      : %d' % len(schemas))
    for names, c in schemas.most_common():
        print('  x%-6d %s' % (c, ', '.join(names)))
    print('MessageCount*2 == positional count: %d/%d  (mismatches %d)' % (ok, ok + bad, bad))
    print('second-of-pair values             : %s' % dict(flags))

    # value census of the seven named fields, against sub_82213980's reader
    FIELDS = ['EffectiveTime', 'Interval', 'IntervalFluctuation', 'Priority',
              'Probability', 'Pattern', 'MessageCount']
    vals = {f: collections.Counter() for f in FIELDS}
    cross = collections.Counter()
    yes_bits = collections.Counter()
    over32 = 0
    for recs in rule.values():
        for r in recs:
            if r['squadron'] == 'Sperkers':
                continue
            d = U.named(r)
            for f in FIELDS:
                vals[f][d.get(f)] += 1
            n, pr = int(d['MessageCount']), int(d['Probability'])
            cross[(n == 0, pr == 0)] += 1
            if n > 32:
                over32 += 1
            pos = [v for _, nm, v in r['fields'] if nm is None]
            yes_bits[sum(1 for i, v in enumerate(pos) if i % 2 and v == 'Yes')] += 1
    print()
    print('value census of the seven named fields (all %d event records):' % sum(vals['Priority'].values()))
    for f in FIELDS:
        v = vals[f]
        try:
            order = sorted(v, key=lambda x: int(x))
        except (TypeError, ValueError):
            order = sorted(v, key=str)
        print('  %-20s %2d distinct  %s' % (
            f, len(v), ' '.join('%s=%d' % (k, v[k]) for k in order)))
    print('  (MessageCount==0, Probability==0) cross-tab: %s' % {
        'both zero': cross[(True, True)], 'both set': cross[(False, False)],
        'no lines but Probability>0': cross[(True, False)],
        'lines but Probability==0 (dead)': cross[(False, True)]})
    print('  records with MessageCount > 32 (the reader clamps here): %d' % over32)
    print('  Yes-bits per record: %s' % dict(sorted(yes_bits.items())))

    print()
    print('the 64 event records, and how many of the %d tables give each one lines:' % len(rule))
    for name in sorted(n for n in shape if n != 'Sperkers'):
        print('  %-32s %3d' % (name, perevent[name]))

    # speakers
    spk = collections.Counter()
    per = collections.Counter()
    for recs in rule.values():
        for r in recs:
            if r['squadron'] == 'Sperkers':
                names = [n for _, n, _ in r['fields'] if n is not None]
                per[len(names)] += 1
                for n in names:
                    spk[n] += 1
    print()
    print('Sperkers roster: %d distinct speakers; sizes %s' % (
        len(spk), dict(sorted(per.items()))))
    for n, c in spk.most_common():
        print('  %-28s %3d' % (n, c))

    # the join into the sound-cue table
    cues = cue_names()
    hit = sorted(m for m in msgs if to_cue(m) in cues)
    miss = sorted(m for m in msgs if to_cue(m) not in cues)
    print()
    print('distinct message ids               : %d' % len(msgs))
    print('resolved in tables.pak SOUNDS      : %d  (%.1f%%)' % (
        len(hit), 100.0 * len(hit) / max(1, len(msgs))))
    print('unresolved                         : %d' % len(miss))
    fam = collections.Counter(m.split('_')[1] for m in miss if m.count('_') >= 2)
    print('  by family: %s' % dict(fam))
    for m in miss:
        print('  %s' % m)

    # phase-merge reconciliation: sub_82215A58 loads a unit's Phase1/2/3 tables
    # into ONE map keyed by the event record name, and sub_82213840 compares a
    # duplicate against the incumbent on (message list, +16, +17, +18, +20, +24,
    # +28) -- not on the Yes mask at +32.
    CMP = ('Probability', 'Priority', 'Pattern', 'Interval',
           'IntervalFluctuation', 'EffectiveTime')
    def _core(rec):
        d = U.named(rec)
        pos = [v for _, nm, v in rec['fields'] if nm is None]
        return ((tuple(d.get(f) for f in CMP), tuple(pos[0::2])), tuple(pos[1::2]))
    sets = dup = same = diff = mask_only = 0
    why = collections.Counter()
    for st in STAGES:
        h = U.name_hash('message\\UnitMessageSet_%s.tbl' % st)
        if h not in entries:
            continue
        for r in U.parse(entries[h]):
            d = U.named(r)
            phases = [d.get('PresetMessage_Phase%d' % i) for i in (1, 2, 3)]
            phases = [n for n in phases if n]
            if len(phases) < 2:
                continue
            sets += 1
            seen = {}
            for n in phases:
                hh = U.name_hash('message\\' + n)
                if hh not in entries:
                    continue
                for rec in U.parse(entries[hh]):
                    if rec['squadron'] == 'Sperkers':
                        continue
                    core, mask = _core(rec)
                    k = rec['squadron']
                    if k not in seen:
                        seen[k] = (core, mask)
                        continue
                    dup += 1
                    if seen[k][0] == core:
                        same += 1
                        if seen[k][1] != mask:
                            mask_only += 1
                    else:
                        diff += 1
                        w = [f for f, a, b in zip(CMP, seen[k][0][0], core[0]) if a != b]
                        if seen[k][0][1] != core[1]:
                            w.append('MESSAGE_LIST')
                        why[tuple(w)] += 1
    print()
    print('phase merge (the loader folds Phase1/2/3 into one map):')
    print('  UnitMessageSet records naming >= 2 phase tables : %d' % sets)
    print('  duplicate-key insertions to reconcile           : %d' % dup)
    print('    identical on the compared fields              : %d' % same)
    print('    DIFFERENT -> loader clears its ok flag        : %d' % diff)
    print('    compared fields equal but Yes mask differs    : %d' % mask_only)
    for w, c in why.most_common():
        print('      x%-5d %s' % (c, ', '.join(w)))

    # naming, two independent routes
    harvest = set()
    for p in sorted(glob.glob(os.path.join(DAT, '**', '*.pak'), recursive=True)):
        for h, s in pak_entries(p):
            if s[:4] != b'IDXD':
                continue
            try:
                recs = U.parse(s)
            except Exception:
                continue
            for r in recs:
                harvest.add(r['squadron'])
                for _, n, v in r['fields']:
                    if n:
                        harvest.add(n)
                    harvest.add(v)
    byharvest = {}
    byderived = {}
    for c in harvest:
        if not c.startswith('PresetMessage_'):
            continue
        byharvest.setdefault(U.name_hash('message\\' + c), 'message\\' + c)
        # route 3: a message table's name gives its rule table's name
        if c.endswith('_msg.tbl'):
            b = 'message\\' + c[:-len('_msg.tbl')] + '.tbl'
            byderived.setdefault(U.name_hash(b), b)

    declared = collections.Counter()
    ums_found = []
    for st in STAGES:
        h = U.name_hash('message\\UnitMessageSet_%s.tbl' % st)
        if h not in entries:
            continue
        ums_found.append(st)
        for r in U.parse(entries[h]):
            d = U.named(r)
            for k in ('PresetMessage_Phase1', 'PresetMessage_Phase2', 'PresetMessage_Phase3'):
                if d.get(k):
                    declared[d[k]] += 1
    bydecl = {}
    for v in declared:
        bydecl.setdefault(U.name_hash('message\\' + v), 'message\\' + v)

    # route 4: names predicted from the Sperkers roster before they were hashed.
    # TCAF_Fleet<X><Y> speaks (TCAFOP<Y>, TCAF<X>); TCAF_..Ship<X> speaks
    # (ADANPL<X>, TCAF<X>).  Six gaps in two series, all six confirmed.
    PREDICTED = ['TCAF_FleetBB', 'TCAF_17thFleetBB', 'TCAF_17thFleetCA',
                 'TCAF_17thFleetCB', 'TCAF_17thFleetShipA', 'TCAF_17thFleetShipB']
    # same-shaped names that the series does NOT contain, as the control
    CONTROL = ['TCAF_FleetCC', 'TCAF_17thFleetCC', 'TCAF_ShipA_01',
               'TCAF_17thFleetShipC_01']
    bypred = {}
    for c in PREDICTED:
        n = 'message\\PresetMessage_%s.tbl' % c
        bypred.setdefault(U.name_hash(n), n)

    A = {h: n for h, n in byharvest.items() if h in rule}
    B = {h: n for h, n in bydecl.items() if h in rule}
    C = {h: n for h, n in byderived.items() if h in rule}
    D = {h: n for h, n in bypred.items() if h in rule}
    print()
    print('naming route 1 (any harvested PresetMessage_ string): %d/%d' % (len(A), len(rule)))
    print('naming route 2 (UnitMessageSet_S* declarations)     : %d/%d' % (len(B), len(rule)))
    print('naming route 3 (strip _msg from a message table)    : %d/%d' % (len(C), len(rule)))
    print('naming route 4 (predicted from the Sperkers roster) : %d/%d' % (len(D), len(rule)))
    print('routes 1 and 2 agree on the same set                : %s' % (set(A) == set(B)))
    print('route 3 is a superset of route 1                    : %s' % set(A).issubset(C))
    print('route 3 names nothing that is not a rule table      : %s' % all(
        h in rule for h in byderived if h in entries))
    print('CONTROL — same-shaped names outside the series, hits : %d/%d' % (
        sum(1 for c in CONTROL if U.name_hash('message\\PresetMessage_%s.tbl' % c) in rule),
        len(CONTROL)))
    A = dict(A); A.update(C); A.update(D)
    # route 5: sweep the naming grammar the other 143 names describe,
    # PresetMessage_<who>[_NN][_S<nn>-<p>].tbl, over every <who> already seen.
    whos = set()
    for n in A.values():
        base = n.rsplit('\\', 1)[-1]
        m = re.match(r'PresetMessage_(.+?)(?:_\d\d)?(?:_S\d\d-\d)?\.tbl$', base)
        if m:
            whos.add(m.group(1))
    stages = ['_S%02d-%d' % (st, ph) for st in range(1, 30) for ph in range(1, 5)]
    E = {}
    for w in sorted(whos):
        for nn in [''] + ['_%02d' % i for i in range(1, 21)]:
            for st in [''] + stages:
                n = 'message\\PresetMessage_%s%s%s.tbl' % (w, nn, st)
                h = U.name_hash(n)
                if h in rule and h not in A:
                    E[h] = n
    print('naming route 5 (sweep of the grammar, %d <who> tokens): %d/%d' % (
        len(whos), len(E), len(rule)))
    A.update(E)
    print('UNION of all five routes                            : %d/%d' % (len(A), len(rule)))
    missing_msg = [n for h, n in A.items()
                   if U.name_hash(n[:-4] + '_msg.tbl') not in entries]
    print('rule tables whose _msg companion is absent          : %d' % len(missing_msg))
    print('UnitMessageSet_S* tables present: %d of %d  (%s)' % (
        len(ums_found), len(STAGES), ' '.join(ums_found)))
    print('missing: %s' % ' '.join(s for s in STAGES if s not in ums_found))
    print('distinct declared PresetMessage tables: %d' % len(declared))

    # control: the same prefix also names a DIFFERENT shape
    other = {h: n for h, n in byharvest.items() if h in entries and h not in rule}
    print()
    print('CONTROL — PresetMessage_* names that hit an IDXD entry but are NOT rule tables: %d' % len(other))
    oshape = collections.Counter()
    for h in other:
        try:
            recs = U.parse(entries[h])
        except Exception:
            continue
        oshape[tuple(sorted({r['squadron'].rstrip('0123456789') for r in recs}))] += 1
    print('  their record-name families: %s' % dict(oshape))


    # the player-facing partition: which tables voice LOCKON_* / PLAYER_* /
    # ORDER_WINGMAN_*, and who speaks in them
    PLAYER_FACING = ['LOCKON_1', 'LOCKON_12', 'LOCKON_UP', 'LOCKON_DOWN',
                     'PLAYER_HP_LESS_10', 'PLAYER_HP_LESS_30', 'PLAYER_HP_LESS_50',
                     'PLAYER_AMMO_LESS_00', 'PLAYER_AMMO_LESS_10', 'PLAYER_AMMO_LESS_40',
                     'ORDER_WINGMAN_ATTACK', 'ORDER_WINGMAN_COVER',
                     'ORDER_WINGMAN_EXTENDED', 'ORDER_WINGMAN_FORMATION']
    full, none_, partial = [], [], []
    spk_full, spk_rest = set(), set()
    for h, recs in rule.items():
        byname = {r['squadron']: r for r in recs}
        got = [k for k in PLAYER_FACING
               if any(n is None for _, n, _ in byname[k]['fields'])]
        who = [n for _, n, _ in byname['Sperkers']['fields'] if n]
        nm = A.get(h, '%#010x' % h)
        if len(got) == len(PLAYER_FACING):
            full.append(nm); spk_full.update(who)
        elif not got:
            none_.append(nm); spk_rest.update(who)
        else:
            partial.append((nm, len(got))); spk_rest.update(who)
    print()
    print('player-facing events (%d of the 64): who voices them' % len(PLAYER_FACING))
    print('  tables voicing ALL of them : %d' % len(full))
    print('  tables voicing NONE        : %d' % len(none_))
    print('  tables voicing SOME        : %d  %s' % (len(partial), partial))
    for nm in sorted(full):
        print('    all  %s' % nm)
    print('  speakers in the all-set: %d' % len(spk_full))
    print('    exclusive to it: %s' % sorted(spk_full - spk_rest))
    print('    also elsewhere : %d' % len(spk_full & spk_rest))

    print()
    print('rule tables, by name:')
    for h in sorted(rule, key=lambda k: A.get(k, '~%#010x' % k)):
        print('  %#010x  %s' % (h, A.get(h, '<unnamed>')))
    un = [h for h in rule if h not in A]
    print()
    print('unnamed rule tables: %d' % len(un))


if __name__ == '__main__':
    main()