Syplheed-Reborn/tools/re-capture/archive_naming.py

#!/usr/bin/env python3
"""How much of each pak's TOC can be named from the asset strings on the disc.

Harvests every plausible asset-name string from every archive, hashes each under
the known path prefixes, and reports per archive how many of its entries are
explained.  Regenerates docs/re/data/archive-naming.txt.
"""
import sys, os, glob, re, collections

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import unitgroup as U
from unit_substructures import pak_entries

EXT = r'(prt|t32|rat|tbl|xpr|ttc|ttf|TTC|TTF|bin|col|rgn|prm)'
PRE = ['', '2d\\', 'ui\\', 'hud\\', 'Stage\\', 'stage\\', 'message\\', 'language\\',
       'prt\\', 'view\\'] + [l + '\\' for l in ('eng', 'jpn', 'fra', 'deu', 'ita', 'esp')]

def main():
    paks = sorted(glob.glob('/work/sylph_extract/**/*.pak', recursive=True))
    names = set()
    decl = set()
    nenum = collections.Counter()
    toc = {}
    for pk in paks:
        ents = pak_entries(pk)
        toc[os.path.basename(pk)] = [h for h, b in ents]
        for h, b in ents:
            for m in re.finditer((r'[A-Za-z0-9_\\.\-]{4,60}\.' + EXT).encode(), b):
                names.add(m.group(0).decode('latin-1'))
            # ROUTE 2 seed: an object whose single record is named `Enumerate`
            # is a DECLARATION TABLE — its field names are other objects' names.
            if b[:4] == b'IDXD' and b'Enumerate' in b:
                try:
                    recs = U.parse(b)
                except Exception:
                    continue
                if len(recs) == 1 and recs[0]['squadron'] == 'Enumerate':
                    nenum[os.path.basename(pk)] += 1
                    for _t, fn, _v in recs[0]['fields']:
                        if fn:
                            decl.add(fn)

    H = set()
    for n in names:
        for p in PRE:
            H.add(U.name_hash(p + n))

    print("# How much of each pak's TOC can be named from the disc's own strings")
    print("# Regenerate: python3 tools/re-capture/archive_naming.py")
    print("# See docs/re/structures/archive-naming.md")
    print("\n   candidate asset-name strings harvested: %d" % len(names))
    print("   path prefixes tried: %d" % len(PRE))
    print("\n   archive                          entries  named    pct")
    rows = sorted(((k, v) for k, v in toc.items()), key=lambda r: -len(r[1]))
    for nm, ent in rows:
        nd = sum(1 for h in ent if h in H)
        print("   %-30s %6d %6d  %5.1f%%" % (nm, len(ent), nd, 100.0 * nd / len(ent) if ent else 0))

    # WHICH entries got named — the per-archive content index.  The percentage
    # above is a statistic; this is the part that is actually usable.
    NAMES = {}
    for n in sorted(names):          # sorted: a set's order varies per run
        for p in PRE:
            NAMES.setdefault(U.name_hash(p + n), p + n)
    total = sum(1 for ent in toc.values() for h in ent if h in NAMES)
    print("\n## What each archive is made of  (resolved names, digits collapsed to #)")
    print("   total named entries across the disc: %d" % total)
    for nm, ent in rows:
        res = [NAMES[h] for h in ent if h in NAMES]
        if not res:
            continue
        fam = collections.Counter(
            re.sub(r'\d+', '#', r.rsplit('\\', 1)[-1]) for r in res)
        print("\n   %s  %d named, %d families" % (nm, len(res), len(fam)))
        for k, c in sorted(fam.items(), key=lambda kv: (-kv[1], kv[0])):
            print("      x%-4d %s" % (c, k))

    # ROUTE 2 -- the declaration tables.  An `Enumerate` object lists, as FIELD
    # NAMES, the names of other objects; each resolves as `<name>.tbl`.  This is
    # the naming source the string harvest cannot see, because these names are
    # never spelled out as `something.tbl` anywhere on the disc.
    DECL = {}
    for n in sorted(decl):
        DECL.setdefault(U.name_hash(n + '.tbl'), n + '.tbl')
    print("\n## ROUTE 2  the `Enumerate` declaration tables")
    print("   declaration objects found: %d   distinct names they declare: %d"
          % (sum(nenum.values()), len(decl)))
    for nm, c in sorted(nenum.items(), key=lambda kv: (-kv[1], kv[0])):
        print("      x%-5d %s" % (c, nm))
    gained = 0
    for nm, ent in rows:
        g = sum(1 for h in ent if h in DECL and h not in NAMES)
        if g:
            print("   %-30s +%d newly named" % (nm, g))
            gained += g
    print("   entries named by route 2 that route 1 could not: %d" % gained)
    miss = [v for k, v in sorted(DECL.items()) if not any(k in ent for ent in toc.values())]
    print("   declared names with NO pak entry: %d  %s" % (len(miss), sorted(miss)[:6]))
    NAMES.update({k: v for k, v in DECL.items() if k not in NAMES})

    # WHY an archive is low-coverage: split its UNNAMED entries by content.
    print("\n## What the UNNAMED entries are  (magic of every entry not resolved above)")
    print("   %-28s %6s %6s %8s %9s %6s" % (
        'archive', 'IDXD', 'named', 'unnamedI', 'unnamedUI', 'LSTA'))
    for nm, _ in rows:
        e = pak_entries(os.path.join(os.path.dirname(
            [p for p in paks if os.path.basename(p) == nm][0]), nm))
        idx = [(h, b) for h, b in e if b[:4] == b'IDXD']
        ui = [(h, b) for h, b in e if b[:4] in (b'T8aD', b'RATC')]
        ls = [(h, b) for h, b in e if b[:4] == b'LSTA']
        ui_un = sum(1 for h, _b in ui if h not in NAMES)
        ls_un = sum(1 for h, _b in ls if h not in NAMES)
        ix_un = sum(1 for h, _b in idx if h not in NAMES)
        if not (ix_un or ui_un or ls_un):
            continue
        print("   %-28s %6d %6d %8d %9d %6d" % (
            nm, len(idx), sum(1 for h, _b in idx if h in NAMES), ix_un, ui_un, ls_un))

    print("\n   record-name sets of the UNNAMED IDXD entries, per archive:")
    for nm, _ in rows:
        e = pak_entries(os.path.join(os.path.dirname(
            [p for p in paks if os.path.basename(p) == nm][0]), nm))
        un = [b for h, b in e if b[:4] == b'IDXD' and h not in NAMES]
        if not un:
            continue
        sh = collections.Counter()
        for b in un:
            try:
                sh[tuple(sorted({r['squadron'] for r in U.parse(b)}))] += 1
            except Exception:
                sh[('<parse fail>',)] += 1
        print("     %s  %d unnamed IDXD in %d shapes" % (nm, len(un), len(sh)))
        for shape, c in sorted(sh.items(), key=lambda kv: (-kv[1], kv[0])):
            print("        x%-5d %s" % (c, ' '.join(shape)[:96]))

    def _ix(t): return U.ixud_hash([ord(c) for c in t])
    FN = [('name_hash', U.name_hash), ('tag_hash', U.tag_hash), ('ixud_hash', _ix)]
    SUB = ['', '2d\\', 'eng\\', 'Data\\', 'ArmsSt\\', 'Marker\\']
    print("\n## CONTROL  is the unnameable pair keyed by a DIFFERENT hash?")
    print("   (6 prefixes x 3 hash families; the menu paks are the positive control)")
    for nm in ['GP_MAIN_GAME_E2D.pak', 'GP_READY_ROOM.pak', 'GP_TITLE.pak', 'GP_PAUSE_MENU.pak']:
        S = set(toc[nm])
        for fname, f in FN:
            best = (0, '')
            for p in SUB:
                n = sum(1 for v in names if f(p + v) in S)
                if n > best[0]: best = (n, p or '<bare>')
            print("   %-22s %-10s %4d / %-5d %s" % (nm, fname, best[0], len(S), best[1]))
    print("\n## CONTROL  the executable holds almost no asset names")
    print("   sylpheed.db `strings`: 7140 rows, of which 2 look like asset paths")
    print("        Data\\gmicon002_2.t32 and Data\\gmicon006_2.t32")
    for n in ['Data\\gmicon002_2.t32', 'Data\\gmicon006_2.t32', 'gmicon002_2.t32']:
        print("        %-28s resolves anywhere: %s" % (n, U.name_hash(n) in H))

if __name__ == "__main__":
    main()