Sylpheed/tools/re-capture/main_game_unnamed.py

#!/usr/bin/env python3
"""What the 337 unnamed IDXD objects in each GP_MAIN_GAME_* pak actually are.

`archive_naming.py` can name 667 of the 1004 IDXD entries in each language copy
of the main-game pak.  This partitions the 337 it cannot by RECORD-NAME SHAPE,
maps each shape to the family the corpus already documents, and dumps in full
the two objects that belong to no documented family.
Regenerates docs/re/data/main-game-unnamed.txt.
"""
import sys, os, glob, re, collections

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import unitgroup as U
from unit_substructures import pak_entries

EXT = r'(prt|t32|rat|tbl|xpr|ttc|ttf|TTC|TTF|bin|col|rgn|prm)'
PRE = ['', '2d\\', 'ui\\', 'hud\\', 'Stage\\', 'stage\\', 'message\\', 'language\\',
       'prt\\', 'view\\'] + [l + '\\' for l in ('eng', 'jpn', 'fra', 'deu', 'ita', 'esp')]

# shape -> (family, owning doc).  Matched on a record name the shape must contain.
FAMILY = [
    ('Maneuver',     'unit datasheet',        'unit-datasheet-static.md'),
    ('Weapon',       'weapon datasheet',      'weapon-datasheet-static.md'),
    ('Faces',        'unit `Faces` table',    'unit-datasheet-static.md'),
    ('Message_000',  'unit message set',      'preset-message-rules.md'),
    ('ATTACK_SUCCEEDED', 'chatter rule table', 'preset-message-rules.md'),
    ('MessageSet_test',  'unit message set',   'unit-group-table.md'),
    ('EnumUnit',     'unit enumeration',      'stage-mission-tables.md'),
    ('EnumWeapon',   'weapon enumeration',    'weapon-struct-runtime.md'),
    ('AsteroidGroup_000', 'asteroid groups',   'unit-datasheet-static.md'),
    ('Formation_Fleet_01', 'formations',       'stage-mission-tables.md'),
    ('Placement_00', 'placement / route test', 'stage-mission-tables.md'),
    ('Enumerate',    'EMPTY declaration table', 'archive-naming.md'),
]

def name_map(paks):
    names, decl = set(), set()
    for pk in paks:
        for h, b in pak_entries(pk):
            for m in re.finditer((r'[A-Za-z0-9_\\.\-]{4,60}\.' + EXT).encode(), b):
                names.add(m.group(0).decode('latin-1'))
            if b[:4] == b'IDXD' and b'Enumerate' in b:
                try:
                    recs = U.parse(b)
                except Exception:
                    continue
                if len(recs) == 1 and recs[0]['squadron'] == 'Enumerate':
                    for _t, fn, _v in recs[0]['fields']:
                        if fn:
                            decl.add(fn)
    NAMES = {}
    for n in sorted(names):
        for p in PRE:
            NAMES.setdefault(U.name_hash(p + n), p + n)
    for n in sorted(decl):
        NAMES.setdefault(U.name_hash(n + '.tbl'), n + '.tbl')
    return NAMES

def classify(shape):
    for key, fam, doc in FAMILY:
        if key in shape:
            return fam, doc
    return None, None

def main():
    paks = sorted(glob.glob('/work/sylph_extract/**/*.pak', recursive=True))
    NAMES = name_map(paks)
    mains = sorted(p for p in paks
                   if re.match(r'GP_MAIN_GAME_[A-Z]\.pak$', os.path.basename(p)))

    print("# What the unnamed IDXD objects in GP_MAIN_GAME_* are")
    print("# Regenerate: python3 tools/re-capture/main_game_unnamed.py")
    print("# See docs/re/structures/archive-naming.md")

    print("\n## CONTROL  are the six language copies the same set?")
    sets = {}
    for pk in mains:
        e = pak_entries(pk)
        un = sorted(h for h, b in e if b[:4] == b'IDXD' and h not in NAMES)
        sets[os.path.basename(pk)] = un
        print("   %-24s %4d IDXD unnamed" % (os.path.basename(pk), len(un)))
    ref = sets[os.path.basename(mains[0])]
    print("   identical hash sets across all six: %s"
          % all(v == ref for v in sets.values()))

    ents = pak_entries(mains[0])
    un = [(h, b) for h, b in ents if b[:4] == b'IDXD' and h not in NAMES]
    sh, samp = collections.Counter(), {}
    for h, b in un:
        try:
            k = tuple(sorted({r['squadron'] for r in U.parse(b)}))
        except Exception:
            k = ('<parse fail>',)
        sh[k] += 1
        samp.setdefault(k, (h, b))

    print("\n## The %d unnamed objects by FAMILY  (%d distinct record-name shapes)"
          % (len(un), len(sh)))
    fam = collections.Counter()
    docs = {}
    for k, c in sh.items():
        f, d = classify(k)
        fam[f or '<UNCLASSIFIED>'] += c
        docs[f or '<UNCLASSIFIED>'] = d or '-'
    for f, c in sorted(fam.items(), key=lambda kv: (-kv[1], kv[0])):
        print("   x%-4d %-24s owned by %s" % (c, f, docs[f]))
    print("   classified: %d / %d" % (sum(v for k, v in fam.items()
                                          if k != '<UNCLASSIFIED>'), len(un)))

    print("\n## every shape, with its family")
    for k, c in sorted(sh.items(), key=lambda kv: (-kv[1], kv[0])):
        f, _d = classify(k)
        print("   x%-4d %-24s %s" % (c, f or '<UNCLASSIFIED>', ' '.join(k)[:88]))

    print("\n## The UNCLASSIFIED objects, in full")
    for k, c in sorted(sh.items(), key=lambda kv: (-kv[1], kv[0])):
        if classify(k)[0]:
            continue
        h, b = samp[k]
        print("\n   object %08x  (x%d)" % (h, c))
        for r in U.parse(b):
            f = [(n, v) for _t, n, v in r['fields']]
            pos = [v for n, v in f if n is None]
            if pos and len(pos) >= len(f) - 1:       # positional payload + Count
                fl = [float(v) for v in pos]
                mono = all(fl[i] <= fl[i + 1] for i in range(len(fl) - 1))
                cnt = dict((n, v) for n, v in f if n)
                ok = cnt.get('Count') == str(len(pos))
                print("      %-26s n=%2d Count==n:%-5s monotone=%-5s %s"
                      % (r['squadron'], len(pos), ok, mono, ' '.join(pos)))
            else:
                print("      %-26s %s" % (r['squadron'], f))

if __name__ == "__main__":
    main()