Syplheed-Reborn/tools/re-capture/name_block_bases.py

#!/usr/bin/env python3
"""Solve the BASE of every register-relative name block, and print the schema it names.

A loader that reads a table by field name usually does not take each string's
address directly -- it keeps one base pointer and emits `addi rX, rBASE, -N` per
name.  No static xref sees that, so those blocks look unreferenced.

This solves the base from the DISPLACEMENT SET alone: for a group of `addi`s
sharing a source register, try every candidate base implied by (string address -
displacement) and keep the one that lands the most displacements on a string
start.  Control: it must recover `r30 = 0x82088F94` for the unit loader
`sub_82341A20` with no prior knowledge.

Regenerates docs/re/data/name-block-bases.txt.
"""
import sys, os, glob, collections, bisect

DB = '/work/xenia-rs/sylpheed.db'
MIN_GROUP = 8          # displacements needed before a group is worth solving
MIN_RESOLVED = 12      # report a function only if the base explains this many
LOW, HIGH = 0x82000000, 0x82400000

def main():
    import duckdb
    con = duckdb.connect(DB, read_only=True)
    S = set(a for (a,) in con.execute("SELECT address FROM strings").fetchall())
    txt = dict(con.execute("SELECT address, content FROM strings").fetchall())
    funcs = con.execute(
        "SELECT address,end_address,name FROM functions ORDER BY address").fetchall()
    starts = [f[0] for f in funcs]
    ins = con.execute(
        "SELECT address,operands FROM instructions WHERE mnemonic='addi' ORDER BY address"
    ).fetchall()

    print("# Register-relative name blocks: the base solved from the displacements")
    print("# Regenerate: python3 tools/re-capture/name_block_bases.py")
    print("# See docs/re/structures/player-tuning-tables.md")
    print("\n   strings in the image: %d   addi instructions: %d" % (len(S), len(ins)))

    byf = collections.defaultdict(lambda: collections.defaultdict(list))
    order = collections.defaultdict(lambda: collections.defaultdict(list))
    for a, o in ins:
        i = bisect.bisect_right(starts, a) - 1
        if i < 0 or funcs[i][1] <= a:
            continue
        p = [x.strip() for x in o.split(',')]
        if len(p) != 3 or p[0] == p[1]:
            continue
        try:
            d = int(p[2], 0)
        except ValueError:
            continue
        byf[funcs[i][2]][p[1]].append(d)
        order[funcs[i][2]][p[1]].append((a, d))

    rows = []
    for name in sorted(byf):
        for reg in sorted(byf[name]):
            D = sorted(set(byf[name][reg]))
            if len(D) < MIN_GROUP or reg == 'r1':      # r1 is the stack pointer
                continue
            # Vote: every (string, displacement) pair implies one candidate base.
            # The true base collects a vote from each name it explains, so it wins
            # outright -- taking candidates from ONE displacement misses it.
            cand = collections.Counter()
            for A in S:
                for d in D:
                    B = A - d
                    if LOW <= B <= HIGH:
                        cand[B] += 1
            best = None
            # sorted, not most_common(): a Counter's tie order varies per run
            top = sorted(cand.items(), key=lambda kv: (-kv[1], kv[0]))[:40]
            for B, _v in top:
                tot = sum(1 for d in D if (B + d) in S)
                if best is None or tot > best[1] or (tot == best[1] and B < best[0]):
                    best = (B, tot)
            if best and best[1] >= MIN_RESOLVED:
                rows.append((name, reg, best[0], best[1], len(D)))

    rows.sort(key=lambda r: (-r[3], r[0]))
    print("\n## %d functions read a name block through a base register" % len(rows))
    print("   %-18s %-5s %-12s %s" % ('function', 'reg', 'base', 'resolved / displacements'))
    for name, reg, B, tot, n in rows:
        print("   %-18s %-5s 0x%08X   %d / %d" % (name, reg, B, tot, n))

    # Confidence.  A base whose low half is 0x0000 is a bare `addis` with no
    # `addi` of its own -- any scatter of displacements votes for it, so those
    # rows are the tool's false-positive mode and must be read with the
    # resolution ratio, not on their own.
    round_rows = [r for r in rows if (r[2] & 0xFFFF) == 0]
    solved_rows = [r for r in rows if (r[2] & 0xFFFF) != 0]
    print("\n## Confidence split")
    print("   bases with a non-zero low half (a real `addis`+`addi` pair): %d" % len(solved_rows))
    print("   bases on a 64K boundary (LOW CONFIDENCE, see below)          : %d" % len(round_rows))
    rb = collections.Counter(r[2] for r in round_rows)
    for B, c in sorted(rb.items(), key=lambda kv: (-kv[1], kv[0])):
        print("      0x%08X  x%d rows" % (B, c))
    print("   ⚠ the 0x820B0000 cluster is ~60 near-identical functions in"
          " 0x8281xxxx-0x8284xxxx that all name the same `rou_e0NN` list.")

    # Is a row a DATA-TABLE schema or engine/XDK vocabulary?  The objective test
    # is whether its names are IDXD record/field names on the disc.
    sys.path.insert(0, HERE) if False else None
    from unit_substructures import pak_entries
    import unitgroup as _U
    disc = set()
    for pk in sorted(glob.glob('/work/sylph_extract/**/*.pak', recursive=True)):
        for _h, b in pak_entries(pk):
            if b[:4] != b'IDXD':
                continue
            try:
                rs = _U.parse(b)
            except Exception:
                continue
            for r in rs:
                disc.add(r['squadron'])
                for _t, n, _v in r['fields']:
                    if n:
                        disc.add(n)
    print("\n## Is the row a DATA-TABLE schema?  (names that are IDXD record/field"
          " names on the disc)")
    print("   distinct IDXD record+field names disc-wide: %d" % len(disc))
    scored = []
    for name, reg, B, tot, n in rows:
        nm, seen = [], set()
        for a, d in order[name][reg]:
            t = B + d
            if t in txt and t not in seen:
                seen.add(t)
                nm.append(txt[t])
        if not nm:
            continue
        hit = sum(1 for x in nm if x in disc)
        scored.append((hit / len(nm), hit, len(nm), name, reg, B))
    tab = [r for r in scored if r[0] >= 0.5 and r[2] >= 8]
    print("   rows that are >=50%% disc names and >=8 names: %d / %d"
          % (len(tab), len(scored)))
    for r in sorted(tab, key=lambda r: (-r[2], r[3])):
        print("   %-14s %-4s 0x%08X %4d names %3.0f%% disc"
              % (r[3], r[4], r[5], r[2], 100 * r[0]))

    print("\n## The schema each one names, in code order")
    for name, reg, B, tot, n in rows:
        seen, names = set(), []
        for a, d in order[name][reg]:
            t = B + d
            if t in txt and t not in seen:
                seen.add(t)
                names.append(txt[t])
        if not names:
            continue
        print("\n   %s  (%s, base 0x%08X) names %d:" % (name, reg, B, len(names)))
        line = "     "
        for nm in names:
            if len(line) + len(nm) > 96:
                print(line)
                line = "     "
            line += nm + "  "
        if line.strip():
            print(line)

if __name__ == "__main__":
    main()