Sylpheed/tools/re-capture/prt_parts.py

#!/usr/bin/env python3
"""Where `.prt` screen parts live, and why the five mission banners do not.

Regenerates docs/re/data/prt-parts.txt.
"""
import sys, os, glob, re, struct, collections
from disc import disc_root

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import unitgroup as U
from unit_substructures import pak_entries

FIVE = ['pgmsg_start.prt', 'pgmsg_end.prt', 'pgmsg_update.prt',
        'pgmsg_failed.prt', 'pgmsg_restart.prt']
PRE = ['', 'Stage\\', 'stage\\', 'message\\', 'language\\', '2d\\', 'ui\\', 'prt\\',
       'view\\'] + [l + '\\' for l in ('eng', 'jpn', 'fra', 'deu', 'ita', 'esp')]

def _elements(b):
    n = struct.unpack_from('>I', b, 0x14)[0]
    if not (0 < n < 4000) or 0x20 + n * 60 > len(b):
        return []
    return [b[0x20 + i * 60:0x20 + i * 60 + 28].split(b'\x00')[0].decode('latin-1', 'replace')
            for i in range(n)]

def _stem(e):
    return re.sub(r'\.(t32|rat|prm)$', '', e)

def main():
    H = collections.defaultdict(set)
    names = set()
    ref = collections.defaultdict(set)
    pref2d, prefall = set(), set()
    elem2d = collections.defaultdict(set)
    dirs = collections.Counter()
    rat = collections.defaultdict(set)
    for pk in sorted(glob.glob(disc_root() + '/**/*.pak', recursive=True)):
        base = os.path.basename(pk)
        for h, b in pak_entries(pk):
            H[h].add(base)
            for m in re.finditer(rb'[A-Za-z0-9_]{3,40}\.prt', b):
                names.add(m.group(0).decode()); ref[m.group(0).decode()].add(base)
            if b[:4] == b'RATC':
                for m in re.finditer(rb'[A-Za-z0-9_]{3,40}_sub\.rat', b):
                    rat[m.group(0).decode()].add(base)
                e = _elements(b)
                if e:
                    pfx = os.path.commonprefix([_stem(x) for x in e]).rstrip('_')
                    if pfx:
                        prefall.add(pfx)
                    if '2D.pak' in base:
                        if pfx: pref2d.add(pfx)
                        for x in e: elem2d[_stem(x)].add(pfx)
            if b[:4] == b'IDXD':
                try: rr = U.parse(b)
                except Exception: rr = []
                for r in rr:
                    for t, nm, v in r['fields']:
                        if '\\' in v and v.count('\\') <= 2 and len(v) < 64:
                            dirs[v.rsplit('\\', 1)[0] + '\\'] += 1

    print("# Where `.prt` screen parts live")
    print("# Regenerate: python3 tools/re-capture/prt_parts.py")
    print("# See docs/re/structures/mission-script-manifest.md")

    byp = collections.Counter()
    hit = set()
    for n in names:
        for p in PRE:
            if U.name_hash(p + n) in H:
                byp[p] += 1; hit.add(n)
    print("\n## `.prt` IS a pak entry name -- under a LANGUAGE directory")
    print("   distinct .prt names referenced anywhere : %d" % len(names))
    print("   resolving as a pak entry                : %d" % len(hit))
    for p, c in byp.most_common():
        print("        prefix %-8s %3d" % (repr(p), c))

    print("\n## the five mission-banner parts")
    for n in FIVE + ['prbase.prt', 'palogo1.prt']:
        ok = [p or '<bare>' for p in PRE if U.name_hash(p + n) in H]
        print("   %-20s resolves under: %s" % (n, ok if ok else 'NOTHING'))

    res = {n: any(U.name_hash(p + n) in H for p in PRE) for n in names}
    tab = lambda n: 'tables.pak' in ref[n]
    ct = collections.Counter((tab(n), res[n]) for n in names)
    print("\n## WHO REFERENCES a part decides whether it ships as an entry")
    print("   cross-tab (referenced by tables.pak?, resolves as an entry?):")
    for k in sorted(ct, key=lambda x: -ct[x]):
        print("        tables=%-5s resolves=%-5s  %3d" % (k[0], k[1], ct[k]))
    print("   => resolving IMPLIES referenced by tables.pak: %d counterexamples of %d"
          % (ct[(False, True)], len(names)))
    bad = sorted(n for n in names if tab(n) and not res[n])
    print("   the %d referenced-but-absent parts:" % len(bad))
    for n in bad:
        print("        %s" % n)
    pg = [n for n in names if n.startswith('pg')]
    print("   pg* parts: %d, of which %d resolve: %s"
          % (len(pg), sum(1 for n in pg if res[n]), sorted(n for n in pg if res[n])))

    ing = [n for n in names if not res[n] and 'tables.pak' not in ref[n]]
    sh = [n for n in names if res[n]]
    match = lambda ns, S: sum(1 for n in ns if n[:-4] in S or n[:-4] + '_sub' in S)
    print("\n## AN EMBEDDED PART IS A TOP-LEVEL RATC BUNDLE, named by its element prefix")
    print("   RATC bundle element-prefixes: %d distinct (2D paks: %d)" % (len(prefall), len(pref2d)))
    print("   in-game-only .prt (%d): match a 2D bundle prefix %d ; any bundle prefix %d"
          % (len(ing), match(ing, pref2d), match(ing, prefall)))
    print("   CONTROL shipped .prt (%d): match a 2D bundle prefix %d ; any bundle prefix %d"
          % (len(sh), match(sh, pref2d), match(sh, prefall)))
    resid = sorted(n for n in ing if n[:-4] not in pref2d and n[:-4] + '_sub' not in pref2d)
    print("   in-game .prt with NO 2D bundle (%d):" % len(resid))
    for n in resid:
        print("        %s" % n)

    have = [n for n in ing if n[:-4] in pref2d or n[:-4] + '_sub' in pref2d]
    lack = [n for n in ing if n not in have]
    asel = lambda n: n[:-4] in elem2d or n[:-4] + '_sub' in elem2d
    print("\n## REFUTED  the bundle-less names are NOT variants declared in a parent bundle")
    print("   in-game names WITHOUT their own bundle (%d): appear as an ELEMENT of some 2D bundle: %d"
          % (len(lack), sum(1 for n in lack if asel(n))))
    print("   CONTROL   in-game names WITH a bundle    (%d): appear as an element: %d"
          % (len(have), sum(1 for n in have if asel(n))))
    dangling = sorted(n for n in lack if not asel(n))
    print("   neither a bundle nor an element -- %d names:" % len(dangling))
    for n in dangling:
        print("        %s" % n)

    extra = [d for d, _ in dirs.most_common(60)]
    print("\n## CONTROL  a prefix list harvested from the DATA adds nothing")
    print("   directory prefixes taken from IDXD path values: %d" % len(extra))
    for d, c in dirs.most_common(8):
        print("        %-30s x%d" % (d, c))
    both = sum(1 for n in names if any(U.name_hash(q + n) in H for q in PRE + extra))
    print("   .prt resolving with the hand list %d ; with hand+harvested %d (of %d)"
          % (len(hit), both, len(names)))

    print("\n## `<stem>_sub.rat` sub-bundles inside RATC")
    print("   distinct _sub.rat names: %d" % len(rat))
    for k in sorted(rat):
        print("        %-26s in %d archives  %s" % (k, len(rat[k]), sorted(rat[k])[:2]))
    stems_prt = {n[:-4] for n in names}
    stems_rat = {k[:-8] for k in rat}
    print("   CONTROL  .prt stems that have a _sub.rat: %d / %d -- so this is NOT the general"
          % (len(stems_prt & stems_rat), len(stems_prt)))
    print("            container convention; `prbase` has a .prt entry and no _sub.rat.")

if __name__ == "__main__":
    main()