diff --git a/docs/re/roster-to-craft-link.md b/docs/re/roster-to-craft-link.md new file mode 100644 index 00000000..7ea04baf --- /dev/null +++ b/docs/re/roster-to-craft-link.md @@ -0,0 +1,64 @@ +# The 116 roster records and the ~300 live craft are not directly linked + +Status: 🔴 a direct pointer link is refuted in both directions; ✅ the two +structures are in distinct heap regions; ❔ how a roster member expands into +craft is still unknown. + +Four attempts have now failed to explain how 116 roster members become ~300 +live craft — `Count` (116), the member field `n` (387), formation slot count +(630, rejected), and now a pointer link. This records the fourth so it is not +retried. + +## The two structures + +| | vtable / locator | count | VA range (measured) | +|---|---|---|---| +| roster records | `0x820AF030` | **116** | `0xbc372c00 … 0xbc9bc720` | +| live craft | def-pointer site − `0x130` | **296–300** | `0xbdb2fd80 … 0xbdcd1d80` | +| unit definitions | `0x820AF844` | 14 | — | + +They are in **different regions**, ~19 MB apart, and neither range overlaps the +other. The 14 definitions match Stage 02's 14 distinct unit types exactly. + +The craft count is not fixed: three runs at comparable mission times gave 296, +298 and 300. That is expected now that kills are confirmed, but it means the +craft population must be compared *within* a run, never across runs. + +## 🔴 Refuted: neither structure points at the other + +`tools/re-capture/link_probe.py` scans the first `0x400` bytes of every object +for a word equal to a known address of the other kind. + +* roster record → craft base: **0 of 116** records hold one. +* craft → roster record base: **0 of 300** craft hold one. + +Both directions are empty, so whatever associates a member with its craft is not +a pointer at the head of either object. + +## ❔ The one loose thread, and why it is not evidence yet + +The same scan counted pointers into the *ranges* rather than at exact bases: + +* roster words pointing anywhere into the entity heap: **181**, of which only + **2** land within `0x400` of a known craft base; +* craft words pointing anywhere into the roster VA range: **5101**. + +5101 looks like a lot, but the roster range spans 6.5 MB and certainly holds +many allocations besides the 116 records, so pointers "into the range" may have +nothing to do with roster records at all. **This is deliberately not being read +as a link.** + +The test that would make it evidence: for each of those 5101 pointers, compute +the distance to the nearest roster-record base below it. If the association is +"craft points at `roster_base + X`" for some fixed `X`, that distribution spikes +at a single delta. If it is flat, the pointers are unrelated allocations and the +thread is dead. + +## What this means for the wave question + +The association may not be a pointer at all — it could be an index, a hash, or +mediated by a third object (a squadron or formation instance) that neither side +names directly. Since an arrival is already known to be a *state change* rather +than an allocation ([mission-liveness-probe.md](mission-liveness-probe.md)), +finding the mediating structure matters more than finding a flag: it is what +would let a change be attributed to a named squadron and therefore to a route. diff --git a/tools/re-capture/link_probe.py b/tools/re-capture/link_probe.py new file mode 100755 index 00000000..1d3c2ac3 --- /dev/null +++ b/tools/re-capture/link_probe.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Find the structural link between the 116 roster records and the live craft. + +Two structures coexist in a mission (mission-liveness-probe.md): + * 116 objects with vtable 0x820AF030 -- one per UnitGroup roster member, + confirmed by an exact unit-composition match; + * 298 heap objects, one per actual craft, located as (def-pointer site - 0x130). + +Counting has failed three times to explain 116 -> 298 (Count, n, formation +slots). This looks for a POINTER instead: does a roster record reference its +craft, or a craft reference its roster record? +""" +import os, sys, struct, collections +sys.path.insert(0, __file__.rsplit('/', 1)[0]) +import gmem, gworld, entities2 + +ROSTER_VT = struct.pack('>I', 0x820AF030) +DELTA = 0x130 +WIN = 0x400 + +def scan_vt(fd, size, vt): + out = [] + for a, b in gmem.extents(fd, size): + pos = a + while pos < b: + n = min(1 << 24, b - pos) + blob = os.pread(fd, n, pos) + i = blob.find(vt) + while i != -1: + if (pos + i) % 4 == 0: out.append(pos + i) + i = blob.find(vt, i + 1) + pos += n + return sorted(out) + +def craft(fd, defs): + lo, hi = gmem.va_to_off(entities2.ENT_VA_LO), gmem.va_to_off(entities2.ENT_VA_HI) + out, pos = [], lo + while pos < hi: + n = min(1 << 24, hi - pos) + blob = os.pread(fd, n, pos) + for k in range(0, len(blob) - 3, 4): + nm = defs.get(blob[k:k+4]) + if nm: out.append((pos + k - DELTA, nm)) + pos += n + return out + +def main(): + w = gworld.World(); fd = w.fd + defs = entities2.definitions(w) + if not defs: print('NOT IN A MISSION'); return 2 + roster = scan_vt(fd, w.size, ROSTER_VT) + cr = craft(fd, defs) + print('roster records: %d live craft: %d definitions: %d' + % (len(roster), len(cr), len(defs))) + + craft_va = {} + for off, nm in cr: + va = gmem.primary_va(off) + if va is not None: craft_va[va] = (off, nm) + cset = set(craft_va) + print('craft with resolvable VA: %d' % len(cset)) + + # forward: roster record -> craft pointer + fwd = collections.Counter(); hits_at = collections.Counter(); linked = 0 + for ro in roster: + blob = os.pread(fd, WIN, ro) + n = 0 + for k in range(0, len(blob) - 3, 4): + (p,) = struct.unpack_from('>I', blob, k) + if p in cset: + n += 1; hits_at[k] += 1 + fwd[n] += 1 + if n: linked += 1 + print('\n--- forward: roster record -> craft VA (window %#x) ---' % WIN) + print(' records with >=1 craft pointer: %d / %d' % (linked, len(roster))) + print(' fan-out histogram:', sorted(fwd.items())[:10]) + print(' hit offsets in record:', [('%#x' % k, c) for k, c in hits_at.most_common(8)]) + + # reverse: craft -> roster record pointer + rset = {} + for ro in roster: + va = gmem.primary_va(ro) + if va is not None: rset[va] = ro + rev = collections.Counter(); roff = collections.Counter(); rlinked = 0 + for off, nm in cr: + blob = os.pread(fd, WIN, off) + n = 0 + for k in range(0, len(blob) - 3, 4): + (p,) = struct.unpack_from('>I', blob, k) + if p in rset: + n += 1; roff[k] += 1 + rev[n] += 1 + if n: rlinked += 1 + # Is this a real "no link", or is my VA convention simply wrong? Count + # pointers into the RANGES rather than at exact computed bases. + ENT_LO, ENT_HI = entities2.ENT_VA_LO, entities2.ENT_VA_HI + rvas = [gmem.primary_va(o) for o in roster] + rvas = [v for v in rvas if v is not None] + print('\n--- sanity: address ranges ---') + print(' roster VAs: %#x .. %#x (%d resolvable)' + % (min(rvas), max(rvas), len(rvas))) + cv = sorted(cset) + print(' craft VAs: %#x .. %#x' % (min(cv), max(cv))) + into_ent = near = 0 + for ro in roster: + blob = os.pread(fd, WIN, ro) + for k in range(0, len(blob) - 3, 4): + (p,) = struct.unpack_from('>I', blob, k) + if ENT_LO <= p < ENT_HI: + into_ent += 1 + if any(abs(p - c) <= 0x400 for c in cv[:4000]): near += 1 + print(' roster words pointing into the entity heap: %d (within 0x400 of a ' + 'known craft base: %d)' % (into_ent, near)) + rlo, rhi = min(rvas), max(rvas) + into_ros = 0 + for off, nm in cr: + blob = os.pread(fd, WIN, off) + for k in range(0, len(blob) - 3, 4): + (p,) = struct.unpack_from('>I', blob, k) + if rlo <= p <= rhi + 0x400: into_ros += 1 + print(' craft words pointing into the roster VA range: %d' % into_ros) + + print('\n--- reverse: craft -> roster record VA ---') + print(' craft with >=1 roster pointer: %d / %d' % (rlinked, len(cr))) + print(' fan-in histogram:', sorted(rev.items())[:10]) + print(' hit offsets in craft:', [('%#x' % k, c) for k, c in roff.most_common(8)]) + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/re-capture/link_session.sh b/tools/re-capture/link_session.sh new file mode 100755 index 00000000..e52ffa31 --- /dev/null +++ b/tools/re-capture/link_session.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -u +export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98 +export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages +SD="$(cd "$(dirname "$0")" && pwd)" +SECS="${1:-180}"; EVERY="${2:-10}"; HUNT="${3:-1}" +CFG=/tmp/nav-live.json +"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; } +if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then + SYLPH_HUNT="$HUNT" SYLPH_KILL_TURRETS=1 nohup python3 "$SD/pilot.py" "$CFG" "$SECS" \ + /tmp/live-pilot.log 2>&1 & + PILOT=$!; echo "--- pilot (SYLPH_HUNT=$HUNT)" +else PILOT=""; echo "--- BIND FAILED, no pilot"; fi +python3 "$SD/link_probe.py" ; rc=$? +[ -n "$PILOT" ] && kill "$PILOT" 2>/dev/null +echo "LIVENESS DONE rc=$rc"