#!/usr/bin/env python3 """Every call site of one or more ISL built-ins, across all `StageNN.ssb`. Pure static work — reads the extracted scripts, runs no emulator. python3 tools/re-capture/isl_builtin_sites.py 26,28,29 --ssb python3 tools/re-capture/isl_builtin_sites.py 101 --ctx 3 --ssb python3 tools/re-capture/isl_builtin_sites.py 28 --craft Why this exists rather than `isl.py --calls`: naming a built-in needs the OPERANDS and the NEIGHBOURS, not just a histogram of ids. `isl.py --calls` scans on the encoding and so cannot see the `local[]` staging that carries a call's arguments, and `isl.py dis` has to be re-synchronised by hand for each site. The walk here is a straight linear decode of the whole code region, from the header's code offset (`+0x08`) to the symbol-table-1 offset (`+0x0C`, which is where the code ends). That is safe: **measured on all 28 stages, the linear walk lands on every call site the independent encoding scan finds** (Stage02: 2846/2846), so no call is skipped and no false instruction boundary is invented. Columns are tab-separated: stage offset builtin slot8 slot4-name slot4-raw symtype before after `slot4` is the unit operand (a symbol-table-2 index; see isl.py for why the unit sits at slot 4 and slot 0 is a tag word) and `slot8` is the second operand, which is a bare double for the built-ins this was written for. `before`/`after` are the ids of the neighbouring calls, which is what makes a fixed idiom — `116 -> 101 -> 100 -> 124 -> 93` — visible. With `--craft ` (the output of `unitgroup.py --all`) it also cross-tabs value against the squadron's craft type, which is the test that separated built-in 15's classes and separates these three. """ import argparse import collections import glob import os import re import struct import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import isl # noqa: E402 def walk(b): """Yield `(offset, builtin_id, staged)` for every call in the code region. `staged` is the `local[]` slot -> value map as it stood when the call executed, i.e. the call's arguments. Both staging forms matter: `local[i] = special[0]` after an immediate, and the far commoner `local[i] = immediate` written straight in. """ code_base = struct.unpack_from('>I', b, 0x08)[0] code_end = struct.unpack_from('>I', b, 0x0C)[0] off, staged, pending = code_base, {}, None while off + 4 <= code_end: w = struct.unpack_from('>I', b, off)[0] op, ln = w & 0xFF, (w >> 8) & 0xFF k1, k0 = (w >> 24) & 0xFF, (w >> 16) & 0xFF if ln == 0 or ln % 4: break words = [struct.unpack_from('>I', b, off + i)[0] for i in range(4, max(ln, 4), 4) if off + i + 4 <= len(b)] if op in (0, 1) and len(words) >= 2: if k0 == 2 and k1 == 1: pending = _imm(op, words) elif k0 == 3 and k1 == 2 and pending is not None: staged[words[0]] = pending elif k0 == 3 and k1 == 1: staged[words[0]] = _imm(op, words) if op == 19 and words: yield off, words[0], dict(staged) staged = {} off += ln def _imm(op, words): """An `op 1` immediate is a DOUBLE carried as two words, not a float.""" if op == 1: lo = words[2] if len(words) > 2 else 0 return struct.unpack('>d', struct.pack('>II', words[1], lo))[0] return words[1] def _fmt(v): if v is None: return '-' return ('%g' % v) if isinstance(v, float) else '0x%X' % v def load_craft(path): """{stage: {squadron: [craft, ...]}} from `unitgroup.py --all` output.""" out, stage, squad = collections.defaultdict(dict), None, None for line in open(path): m = re.match(r'^(S\d\d):', line) if m: stage = m.group(1) continue m = re.match(r'^ (\S+)\s+\S+\s+Count=', line) if m: squad = m.group(1) out[stage][squad] = [] continue m = re.match(r'^ unit=(\S+)', line) if m and squad: out[stage][squad].append(m.group(1)) return out def main(): ap = argparse.ArgumentParser() ap.add_argument('ids', help='comma-separated built-in ids') ap.add_argument('--ssb', default='.', help='directory holding StageNN.ssb') ap.add_argument('--ctx', type=int, default=2, help='neighbour calls to show') ap.add_argument('--craft', help='unitgroup.py --all output, for the cross-tab') a = ap.parse_args() ids = {int(x) for x in a.ids.split(',')} craft = load_craft(a.craft) if a.craft else None vals = collections.defaultdict(collections.Counter) units = collections.defaultdict(collections.Counter) cross = collections.defaultdict(lambda: collections.defaultdict(collections.Counter)) unresolved = collections.Counter() for path in sorted(glob.glob(os.path.join(a.ssb, 'Stage*.ssb'))): stage = os.path.basename(path)[:-4] b = isl.load(path) sym2 = isl.symbols(b, 2) calls = list(walk(b)) for i, (off, bid, staged) in enumerate(calls): if bid not in ids: continue uidx = staged.get(4) typ, name = sym2.get(uidx, (None, '?')) if isinstance(uidx, int) \ else (None, '?') val = _fmt(staged.get(8)) print('%s\t0x%06X\t%d\t%s\t%s\t%s\ttype%s\t[%s]\t[%s]' % ( stage, off, bid, val, name, ('0x%X' % uidx) if isinstance(uidx, int) else uidx, typ, ','.join(str(calls[j][1]) for j in range(max(0, i - a.ctx), i)), ','.join(str(calls[j][1]) for j in range(i + 1, min(len(calls), i + 1 + a.ctx))))) vals[bid][val] += 1 units[bid][name] += 1 if craft is not None: cs = craft.get('S' + stage[-2:], {}).get(name) if cs: for c in set(cs): cross[bid][c][val] += 1 else: unresolved[bid] += 1 for bid in sorted(ids): n = sum(vals[bid].values()) print('\n# builtin %d: %d sites, %d distinct values' % (bid, n, len(vals[bid]))) print('# values: ' + ', '.join('%s x%d' % kv for kv in vals[bid].most_common())) print('# units: %d distinct, top: %s' % ( len(units[bid]), ', '.join('%s x%d' % kv for kv in units[bid].most_common(10)))) if craft is not None: print('# craft cross-tab (%d unresolved):' % unresolved[bid]) for c in sorted(cross[bid], key=lambda c: -sum(cross[bid][c].values())): print('# %-42s %4d %s' % ( c, sum(cross[bid][c].values()), ', '.join('%sx%d' % kv for kv in cross[bid][c].most_common()))) if __name__ == '__main__': main()