diff --git a/tools/re-capture/isl_builtin_sites.py b/tools/re-capture/isl_builtin_sites.py new file mode 100644 index 0000000..f5e0002 --- /dev/null +++ b/tools/re-capture/isl_builtin_sites.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Every call site of one or more ISL built-ins, across all `StageNN.ssb`. + +Pure static work — reads the extracted scripts, runs no emulator. + + python3 tools/re-capture/isl_builtin_sites.py 26,28,29 --ssb + python3 tools/re-capture/isl_builtin_sites.py 101 --ctx 3 --ssb + python3 tools/re-capture/isl_builtin_sites.py 28 --craft + +Why this exists rather than `isl.py --calls`: naming a built-in needs the +OPERANDS and the NEIGHBOURS, not just a histogram of ids. `isl.py --calls` +scans on the encoding and so cannot see the `local[]` staging that carries a +call's arguments, and `isl.py dis` has to be re-synchronised by hand for each +site. + +The walk here is a straight linear decode of the whole code region, from the +header's code offset (`+0x08`) to the symbol-table-1 offset (`+0x0C`, which is +where the code ends). That is safe: **measured on all 28 stages, the linear walk +lands on every call site the independent encoding scan finds** (Stage02: +2846/2846), so no call is skipped and no false instruction boundary is invented. + +Columns are tab-separated: + + stage offset builtin slot8 slot4-name slot4-raw symtype before after + +`slot4` is the unit operand (a symbol-table-2 index; see isl.py for why the unit +sits at slot 4 and slot 0 is a tag word) and `slot8` is the second operand, +which is a bare double for the built-ins this was written for. `before`/`after` +are the ids of the neighbouring calls, which is what makes a fixed idiom — +`116 -> 101 -> 100 -> 124 -> 93` — visible. + +With `--craft ` (the output of `unitgroup.py --all`) it also cross-tabs +value against the squadron's craft type, which is the test that separated +built-in 15's classes and separates these three. +""" +import argparse +import collections +import glob +import os +import re +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import isl # noqa: E402 + + +def walk(b): + """Yield `(offset, builtin_id, staged)` for every call in the code region. + + `staged` is the `local[]` slot -> value map as it stood when the call + executed, i.e. the call's arguments. Both staging forms matter: + `local[i] = special[0]` after an immediate, and the far commoner + `local[i] = immediate` written straight in. + """ + code_base = struct.unpack_from('>I', b, 0x08)[0] + code_end = struct.unpack_from('>I', b, 0x0C)[0] + off, staged, pending = code_base, {}, None + while off + 4 <= code_end: + w = struct.unpack_from('>I', b, off)[0] + op, ln = w & 0xFF, (w >> 8) & 0xFF + k1, k0 = (w >> 24) & 0xFF, (w >> 16) & 0xFF + if ln == 0 or ln % 4: + break + words = [struct.unpack_from('>I', b, off + i)[0] + for i in range(4, max(ln, 4), 4) if off + i + 4 <= len(b)] + if op in (0, 1) and len(words) >= 2: + if k0 == 2 and k1 == 1: + pending = _imm(op, words) + elif k0 == 3 and k1 == 2 and pending is not None: + staged[words[0]] = pending + elif k0 == 3 and k1 == 1: + staged[words[0]] = _imm(op, words) + if op == 19 and words: + yield off, words[0], dict(staged) + staged = {} + off += ln + + +def _imm(op, words): + """An `op 1` immediate is a DOUBLE carried as two words, not a float.""" + if op == 1: + lo = words[2] if len(words) > 2 else 0 + return struct.unpack('>d', struct.pack('>II', words[1], lo))[0] + return words[1] + + +def _fmt(v): + if v is None: + return '-' + return ('%g' % v) if isinstance(v, float) else '0x%X' % v + + +def load_craft(path): + """{stage: {squadron: [craft, ...]}} from `unitgroup.py --all` output.""" + out, stage, squad = collections.defaultdict(dict), None, None + for line in open(path): + m = re.match(r'^(S\d\d):', line) + if m: + stage = m.group(1) + continue + m = re.match(r'^ (\S+)\s+\S+\s+Count=', line) + if m: + squad = m.group(1) + out[stage][squad] = [] + continue + m = re.match(r'^ unit=(\S+)', line) + if m and squad: + out[stage][squad].append(m.group(1)) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('ids', help='comma-separated built-in ids') + ap.add_argument('--ssb', default='.', help='directory holding StageNN.ssb') + ap.add_argument('--ctx', type=int, default=2, help='neighbour calls to show') + ap.add_argument('--craft', help='unitgroup.py --all output, for the cross-tab') + a = ap.parse_args() + + ids = {int(x) for x in a.ids.split(',')} + craft = load_craft(a.craft) if a.craft else None + vals = collections.defaultdict(collections.Counter) + units = collections.defaultdict(collections.Counter) + cross = collections.defaultdict(lambda: collections.defaultdict(collections.Counter)) + unresolved = collections.Counter() + + for path in sorted(glob.glob(os.path.join(a.ssb, 'Stage*.ssb'))): + stage = os.path.basename(path)[:-4] + b = isl.load(path) + sym2 = isl.symbols(b, 2) + calls = list(walk(b)) + for i, (off, bid, staged) in enumerate(calls): + if bid not in ids: + continue + uidx = staged.get(4) + typ, name = sym2.get(uidx, (None, '?')) if isinstance(uidx, int) \ + else (None, '?') + val = _fmt(staged.get(8)) + print('%s\t0x%06X\t%d\t%s\t%s\t%s\ttype%s\t[%s]\t[%s]' % ( + stage, off, bid, val, name, + ('0x%X' % uidx) if isinstance(uidx, int) else uidx, typ, + ','.join(str(calls[j][1]) for j in range(max(0, i - a.ctx), i)), + ','.join(str(calls[j][1]) + for j in range(i + 1, min(len(calls), i + 1 + a.ctx))))) + vals[bid][val] += 1 + units[bid][name] += 1 + if craft is not None: + cs = craft.get('S' + stage[-2:], {}).get(name) + if cs: + for c in set(cs): + cross[bid][c][val] += 1 + else: + unresolved[bid] += 1 + + for bid in sorted(ids): + n = sum(vals[bid].values()) + print('\n# builtin %d: %d sites, %d distinct values' % (bid, n, len(vals[bid]))) + print('# values: ' + ', '.join('%s x%d' % kv for kv in vals[bid].most_common())) + print('# units: %d distinct, top: %s' % ( + len(units[bid]), ', '.join('%s x%d' % kv for kv in units[bid].most_common(10)))) + if craft is not None: + print('# craft cross-tab (%d unresolved):' % unresolved[bid]) + for c in sorted(cross[bid], key=lambda c: -sum(cross[bid][c].values())): + print('# %-42s %4d %s' % ( + c, sum(cross[bid][c].values()), + ', '.join('%sx%d' % kv for kv in cross[bid][c].most_common()))) + + +if __name__ == '__main__': + main()