Sylpheed/tools/re-capture/isl_builtin_sites.py

#!/usr/bin/env python3
"""Every call site of one or more ISL built-ins, across all `StageNN.ssb`.

Pure static work — reads the extracted scripts, runs no emulator.

    python3 tools/re-capture/isl_builtin_sites.py 26,28,29 --ssb <dir>
    python3 tools/re-capture/isl_builtin_sites.py 101 --ctx 3 --ssb <dir>
    python3 tools/re-capture/isl_builtin_sites.py 28 --craft <unitgroup-dump>

Why this exists rather than `isl.py --calls`: naming a built-in needs the
OPERANDS and the NEIGHBOURS, not just a histogram of ids. `isl.py --calls`
scans on the encoding and so cannot see the `local[]` staging that carries a
call's arguments, and `isl.py dis` has to be re-synchronised by hand for each
site.

The walk here is a straight linear decode of the whole code region, from the
header's code offset (`+0x08`) to the symbol-table-1 offset (`+0x0C`, which is
where the code ends). That is safe: **measured on all 28 stages, the linear walk
lands on every call site the independent encoding scan finds** (Stage02:
2846/2846), so no call is skipped and no false instruction boundary is invented.

Columns are tab-separated:

    stage  offset  builtin  slot8  slot4-name  slot4-raw  symtype  before  after

`slot4` is the unit operand (a symbol-table-2 index; see isl.py for why the unit
sits at slot 4 and slot 0 is a tag word) and `slot8` is the second operand,
which is a bare double for the built-ins this was written for. `before`/`after`
are the ids of the neighbouring calls, which is what makes a fixed idiom —
`116 -> 101 -> 100 -> 124 -> 93` — visible.

With `--craft <file>` (the output of `unitgroup.py --all`) it also cross-tabs
value against the squadron's craft type, which is the test that separated
built-in 15's classes and separates these three.
"""
import argparse
import collections
import glob
import os
import re
import struct
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import isl                                                   # noqa: E402


def walk(b):
    """Yield `(offset, builtin_id, staged)` for every call in the code region.

    `staged` is the `local[]` slot -> value map as it stood when the call
    executed, i.e. the call's arguments. Both staging forms matter:
    `local[i] = special[0]` after an immediate, and the far commoner
    `local[i] = immediate` written straight in.
    """
    code_base = struct.unpack_from('>I', b, 0x08)[0]
    code_end = struct.unpack_from('>I', b, 0x0C)[0]
    off, staged, pending = code_base, {}, None
    while off + 4 <= code_end:
        w = struct.unpack_from('>I', b, off)[0]
        op, ln = w & 0xFF, (w >> 8) & 0xFF
        k1, k0 = (w >> 24) & 0xFF, (w >> 16) & 0xFF
        if ln == 0 or ln % 4:
            break
        words = [struct.unpack_from('>I', b, off + i)[0]
                 for i in range(4, max(ln, 4), 4) if off + i + 4 <= len(b)]
        if op in (0, 1) and len(words) >= 2:
            if k0 == 2 and k1 == 1:
                pending = _imm(op, words)
            elif k0 == 3 and k1 == 2 and pending is not None:
                staged[words[0]] = pending
            elif k0 == 3 and k1 == 1:
                staged[words[0]] = _imm(op, words)
        if op == 19 and words:
            yield off, words[0], dict(staged)
            staged = {}
        off += ln


def _imm(op, words):
    """An `op 1` immediate is a DOUBLE carried as two words, not a float."""
    if op == 1:
        lo = words[2] if len(words) > 2 else 0
        return struct.unpack('>d', struct.pack('>II', words[1], lo))[0]
    return words[1]


def _fmt(v):
    if v is None:
        return '-'
    return ('%g' % v) if isinstance(v, float) else '0x%X' % v


def load_craft(path):
    """{stage: {squadron: [craft, ...]}} from `unitgroup.py --all` output."""
    out, stage, squad = collections.defaultdict(dict), None, None
    for line in open(path):
        m = re.match(r'^(S\d\d):', line)
        if m:
            stage = m.group(1)
            continue
        m = re.match(r'^  (\S+)\s+\S+\s+Count=', line)
        if m:
            squad = m.group(1)
            out[stage][squad] = []
            continue
        m = re.match(r'^      unit=(\S+)', line)
        if m and squad:
            out[stage][squad].append(m.group(1))
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('ids', help='comma-separated built-in ids')
    ap.add_argument('--ssb', default='.', help='directory holding StageNN.ssb')
    ap.add_argument('--ctx', type=int, default=2, help='neighbour calls to show')
    ap.add_argument('--craft', help='unitgroup.py --all output, for the cross-tab')
    a = ap.parse_args()

    ids = {int(x) for x in a.ids.split(',')}
    craft = load_craft(a.craft) if a.craft else None
    vals = collections.defaultdict(collections.Counter)
    units = collections.defaultdict(collections.Counter)
    cross = collections.defaultdict(lambda: collections.defaultdict(collections.Counter))
    unresolved = collections.Counter()

    for path in sorted(glob.glob(os.path.join(a.ssb, 'Stage*.ssb'))):
        stage = os.path.basename(path)[:-4]
        b = isl.load(path)
        sym2 = isl.symbols(b, 2)
        calls = list(walk(b))
        for i, (off, bid, staged) in enumerate(calls):
            if bid not in ids:
                continue
            uidx = staged.get(4)
            typ, name = sym2.get(uidx, (None, '?')) if isinstance(uidx, int) \
                else (None, '?')
            val = _fmt(staged.get(8))
            print('%s\t0x%06X\t%d\t%s\t%s\t%s\ttype%s\t[%s]\t[%s]' % (
                stage, off, bid, val, name,
                ('0x%X' % uidx) if isinstance(uidx, int) else uidx, typ,
                ','.join(str(calls[j][1]) for j in range(max(0, i - a.ctx), i)),
                ','.join(str(calls[j][1])
                         for j in range(i + 1, min(len(calls), i + 1 + a.ctx)))))
            vals[bid][val] += 1
            units[bid][name] += 1
            if craft is not None:
                cs = craft.get('S' + stage[-2:], {}).get(name)
                if cs:
                    for c in set(cs):
                        cross[bid][c][val] += 1
                else:
                    unresolved[bid] += 1

    for bid in sorted(ids):
        n = sum(vals[bid].values())
        print('\n# builtin %d: %d sites, %d distinct values' % (bid, n, len(vals[bid])))
        print('#   values: ' + ', '.join('%s x%d' % kv for kv in vals[bid].most_common()))
        print('#   units:  %d distinct, top: %s' % (
            len(units[bid]), ', '.join('%s x%d' % kv for kv in units[bid].most_common(10))))
        if craft is not None:
            print('#   craft cross-tab (%d unresolved):' % unresolved[bid])
            for c in sorted(cross[bid], key=lambda c: -sum(cross[bid][c].values())):
                print('#     %-42s %4d  %s' % (
                    c, sum(cross[bid][c].values()),
                    ', '.join('%sx%d' % kv for kv in cross[bid][c].most_common())))


if __name__ == '__main__':
    main()