Sylpheed/tools/re-capture/mission_scoring.py

#!/usr/bin/env python3
"""Regenerate docs/re/data/mission-scoring.txt.

The per-stage mission scoring / rank-threshold tables: 24 IDXD objects per
language pack, each holding Score_Easy / Score_Normal / Score_Hard with one
22-field schema.  Loader: sub_8230D1F8.

Run:  python3 mission_scoring.py > ../../docs/re/data/mission-scoring.txt
"""
import sys, os, re, glob, collections
from disc import disc_root
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from unit_substructures import pak_entries
import unitgroup as U

DAT = disc_root() + '/dat'
KEY = 'RankScore_S'
DIFFS = ('Score_Easy', 'Score_Normal', 'Score_Hard')


def main():
    percopy = {}
    for p in sorted(glob.glob(os.path.join(DAT, '*.pak'))):
        n = sum(1 for _, s in pak_entries(p)
                if s[:4] == b'IDXD' and KEY.encode() in s)
        if n:
            percopy[os.path.basename(p)] = n

    P = os.path.join(DAT, 'GP_MAIN_GAME_E.pak')
    tables = []
    for h, s in pak_entries(P):
        if s[:4] != b'IDXD' or KEY.encode() not in s:
            continue
        try:
            recs = U.parse(s)
        except Exception:
            continue
        tables.append((h, {r['squadron']: U.named(r) for r in recs
                           if r['squadron'] in DIFFS}))
    tables.sort()

    print('# Mission scoring / rank thresholds (regenerated by mission_scoring.py)')
    print()
    print('IDXD objects carrying %s, per pack: %s' % (KEY, percopy))
    print('total across the disc                 : %d' % sum(percopy.values()))
    print('difficulty records per object         : %s' % sorted(
        {tuple(sorted(t)) for _, t in tables}))

    schemas = collections.Counter()
    for _, t in tables:
        for d in DIFFS:
            schemas[tuple(sorted(t[d]))] += 1
    print('distinct field sets                   : %d' % len(schemas))
    fields = sorted(schemas.most_common(1)[0][0])
    print('records                               : %d  (%d fields each)' % (
        sum(schemas.values()), len(fields)))

    # how much does each field actually vary?
    print()
    print('value census per field (over %d records):' % sum(schemas.values()))
    for f in fields:
        c = collections.Counter(t[d][f] for _, t in tables for d in DIFFS)
        vals = ' '.join('%s=%d' % (k, v) for k, v in sorted(
            c.items(), key=lambda kv: (-kv[1], kv[0])))
        print('  %-32s %2d distinct  %s' % (f, len(c), vals[:110]))

    # does difficulty move anything?
    print()
    print('fields that differ between the three difficulty records:')
    moved = collections.Counter()
    for _, t in tables:
        for f in fields:
            if len({t[d][f] for d in DIFFS}) > 1:
                moved[f] += 1
    print('  %s' % (dict(moved) if moved else 'NONE — Easy/Normal/Hard are identical everywhere'))

    # per-object variation
    print()
    print('objects whose Normal record differs from the commonest Normal record:')
    base = collections.Counter(tuple(sorted(t['Score_Normal'].items()))
                               for _, t in tables).most_common(1)[0][0]
    odd = [(h, t) for h, t in tables
           if tuple(sorted(t['Score_Normal'].items())) != base]
    print('  %d of %d objects' % (len(odd), len(tables)))
    for h, t in odd:
        diff = {f: t['Score_Normal'][f] for f in fields
                if t['Score_Normal'][f] != dict(base)[f]}
        print('    %#010x  %s' % (h, diff))

    # the rest of the settings object: Camera / Player / SplinterCell / Difficulty_*
    OTHER = ('Camera', 'Player', 'SplinterCell',
             'Difficulty_Easy', 'Difficulty_Normal', 'Difficulty_Hard')
    full = []
    for h, s2 in pak_entries(P):
        if s2[:4] != b'IDXD' or KEY.encode() not in s2:
            continue
        try:
            full.append((h, {r['squadron']: r for r in U.parse(s2)}))
        except Exception:
            pass
    full.sort()
    # name every settings object: harvest all IDXD strings and hash them
    harvest = set()
    for pp in sorted(glob.glob(os.path.join(DAT, '**', '*.pak'), recursive=True)):
        for _, s2 in pak_entries(pp):
            if s2[:4] != b'IDXD':
                continue
            try:
                for r in U.parse(s2):
                    for _, nm, v in r['fields']:
                        harvest.add(v)
            except Exception:
                pass
    want = {h for h, _ in tables}
    named = {}
    for c in harvest:
        if not c or len(c) > 90:
            continue
        for pre in ('', 'stage\\', 'message\\'):
            hh = U.name_hash(pre + c)
            if hh in want:
                named.setdefault(hh, pre + c)
    print()
    print('settings objects named from harvested disc strings: %d/%d' % (len(named), len(want)))
    for h in sorted(named, key=lambda k: named[k]):
        print('    %#010x  %s' % (h, named[h]))

    print()
    print('the rest of the settings object (%d objects):' % len(full))
    shapes = collections.Counter(tuple(sorted(d)) for _, d in full)
    print('  record-name sets: %d' % len(shapes))
    for t, c in shapes.most_common():
        print('    x%-3d %s' % (c, list(t)))
    for rn in OTHER:
        vals = collections.defaultdict(collections.Counter)
        n = 0
        for _, d in full:
            if rn not in d:
                continue
            n += 1
            for k, v in U.named(d[rn]).items():
                vals[k][v] += 1
        print()
        print('  %s — present in %d/%d objects, %d fields' % (rn, n, len(full), len(vals)))
        for k in sorted(vals):
            c = vals[k]
            print('    %-36s %2d distinct  %s' % (
                k, len(c), ' '.join('%s=%d' % (a, b) for a, b in c.most_common(5))[:70]))

    # the Phase_1/2/3 block of the settings family
    prec = [(h, i, U.named(d['Phase_%d' % i])) for h, d in full for i in (1, 2, 3)
            if 'Phase_%d' % i in d]
    allf = collections.Counter()
    for _, _, n in prec:
        for f in n:
            allf[f] += 1
    core = sorted(f for f, c in allf.items() if c == len(prec))
    print()
    print('the Phase_1/2/3 block (%d records over %d objects):' % (len(prec), len(full)))
    print('  distinct field names        : %d' % len(allf))
    print('  present in EVERY record     : %d' % len(core))
    print('    %s' % ' '.join(core))
    print('  field-count histogram       : %s' % dict(sorted(
        collections.Counter(len(n) for _, _, n in prec).items())))

    le = 0
    slots = collections.Counter()
    cnts = collections.Counter()
    for _, _, n in prec:
        c = int(n['BGOperateFrameCount'])
        k = sum(1 for f in n if re.fullmatch(r'BGOperateFrameName_\d+', f))
        slots[k] += 1
        cnts[c] += 1
        le += (c <= k)
    eq = sum(1 for _, _, n in prec
             if int(n['BGOperateFrameCount'])
             == sum(1 for f in n if re.fullmatch(r'BGOperateFrameName_\d+', f)))
    print('  BGOperateFrameCount == slots: %d/%d  (REFUTED)' % (eq, len(prec)))
    print('  BGOperateFrameCount <= slots: %d/%d' % (le, len(prec)))
    print('    slot histogram %s ; Count histogram %s' % (
        dict(sorted(slots.items())), dict(sorted(cnts.items()))))

    moved = collections.Counter()
    same = collections.Counter()
    for h, d in full:
        ns = [U.named(d['Phase_%d' % i]) for i in (1, 2, 3) if 'Phase_%d' % i in d]
        if len(ns) != 3:
            continue
        for f in set(ns[0]) & set(ns[1]) & set(ns[2]):
            (moved if len({n[f] for n in ns}) > 1 else same)[f] += 1
    never = sorted(f for f in same if f not in moved)
    print('  fields that DIFFER between phases (of %d objects):' % len(full))
    for f, c in sorted(moved.items(), key=lambda kv: (-kv[1], kv[0])):
        print('    %-28s %2d' % (f, c))
    print('  fields that NEVER differ between phases: %d' % len(never))
    print('    %s' % ' '.join(never))

    print()
    print('the commonest Normal record:')
    for k, v in sorted(dict(base).items()):
        print('  %-32s %s' % (k, v))


if __name__ == '__main__':
    main()