re: find the roster-to-craft link — a pointer at roster_base + 0x08

The previous iteration was right that neither structure points at the other's
base, and wrong to stop there. Craft point eight bytes into a roster record.

Resolving the 5101 "pointers into the roster range" by distance to the nearest
roster base gives a distribution that is neither flat nor diffuse: +0x08 appears
exactly 300 times, matching the 300 live craft, while every other delta appears
at most twice and 4780 of 5104 land more than 0x4000 from any base.

A count matching a count is the coincidence that has already misled this corpus
three times, so the link was tested on something a coincidence cannot survive.
Each side's unit type is resolved by a different chain -- the craft's from its
definition pointer at +0x130, the record's from its +0x04 name chain -- and they
must agree. They do, on all 300, with zero disagreements, and the fan-out
histogram is [(1, 300)]: every craft has exactly one such pointer, none has zero
or two.

The fan-in is the useful half. Only 41 of the 116 roster records are referenced
at all; the other 75 have no craft. Craft per referenced record is discrete:
24 records with 2, one with 4, four with 8, and twelve with 18 -- summing to
exactly 300. The twelve holding 18 are all e007 Turret.

This is the observable four iterations have been hunting. A member with zero
craft is not deployed; a member's craft count is its live strength; so an
arrival is craft appearing for a record that had none and a kill is that count
falling. Both are attributable to a named squadron, because the record is a
UnitGroup member and UnitGroup names the squadron that Route_S02.tbl ties to a
phase and an arrival time. The chain from a memory observation to a static route
entry is complete.

Not claimed: that the 75 undeployed members are waiting on phases 2 and 3. That
is a plausible reading, but the run only shows they have no craft.

tools/re-capture/link2_probe.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Verify the craft -> roster link at roster_base + 0x08.
+
+The delta histogram showed exactly 300 pointers at +0x08 with 300 craft present,
+every other delta appearing at most twice. That is a candidate 1:1 link. Before
+believing it, this checks the thing a coincidence cannot survive: the craft's own
+unit type (from its definition pointer) must equal the unit type of the roster
+record it points at (resolved independently via that record's +0x04 name chain).
+"""
+import os, sys, struct, collections, bisect
+sys.path.insert(0, __file__.rsplit('/', 1)[0])
+import gmem, gworld, entities2
+import importlib.util
+_w3 = importlib.util.spec_from_file_location('w3', __file__.rsplit('/', 1)[0] + '/wave3_probe.py')
+wave3 = importlib.util.module_from_spec(_w3); _w3.loader.exec_module(wave3)
+
+ROSTER_VT = struct.pack('>I', 0x820AF030)
+DELTA, WIN, LINK = 0x130, 0x400, 0x08
+
+def scan_vt(fd, size, vt):
+    out = []
+    for a, b in gmem.extents(fd, size):
+        pos = a
+        while pos < b:
+            n = min(1 << 24, b - pos)
+            blob = os.pread(fd, n, pos)
+            i = blob.find(vt)
+            while i != -1:
+                if (pos + i) % 4 == 0: out.append(pos + i)
+                i = blob.find(vt, i + 1)
+            pos += n
+    return sorted(out)
+
+def main():
+    w = gworld.World(); fd = w.fd
+    defs = entities2.definitions(w)
+    if not defs: print('NOT IN A MISSION'); return 2
+    roster = scan_vt(fd, w.size, ROSTER_VT)
+    f = os.fdopen(os.dup(fd), 'rb')
+    rec = {}                       # roster VA -> (offset, unit id)
+    for o in roster:
+        va = gmem.primary_va(o)
+        if va is None: continue
+        rec[va] = (o, wave3.resolve_id(f, w.size, o)[0] or '?')
+    print('roster records: %d (labelled %d)'
+          % (len(rec), sum(1 for _, n in rec.values() if n != '?')))
+
+    lo, hi = gmem.va_to_off(entities2.ENT_VA_LO), gmem.va_to_off(entities2.ENT_VA_HI)
+    craft, pos = [], lo
+    while pos < hi:
+        n = min(1 << 24, hi - pos)
+        blob = os.pread(fd, n, pos)
+        for k in range(0, len(blob) - 3, 4):
+            nm = defs.get(blob[k:k+4])
+            if nm: craft.append((pos + k - DELTA, nm))
+        pos += n
+    print('live craft: %d' % len(craft))
+
+    want = {va + LINK: va for va in rec}
+    fan = collections.Counter(); agree = disagree = nolink = multi = 0
+    per_record = collections.Counter(); mismatches = []
+    for off, nm in craft:
+        blob = os.pread(fd, WIN, off)
+        tgt = []
+        for k in range(0, len(blob) - 3, 4):
+            (p,) = struct.unpack_from('>I', blob, k)
+            if p in want: tgt.append((k, want[p]))
+        fan[len(tgt)] += 1
+        if not tgt: nolink += 1; continue
+        if len(tgt) > 1: multi += 1
+        rva = tgt[0][1]; per_record[rva] += 1
+        rn = rec[rva][1]
+        if rn == nm: agree += 1
+        else:
+            disagree += 1
+            if len(mismatches) < 6: mismatches.append((nm, rn))
+    print('\n--- link at roster_base + %#x ---' % LINK)
+    print('  craft fan-out histogram:', sorted(fan.items()))
+    print('  craft with no link: %d   with >1: %d' % (nolink, multi))
+    print('  UNIT TYPE AGREES: %d    disagrees: %d' % (agree, disagree))
+    if mismatches: print('  sample mismatches (craft vs record):', mismatches)
+    print('  pointer offsets used:',
+          collections.Counter(k for off, nm in craft[:0] for k in []).most_common())
+    print('\n--- craft per roster record ---')
+    print('  records referenced: %d / %d' % (len(per_record), len(rec)))
+    print('  fan-in histogram:', sorted(collections.Counter(per_record.values()).items()))
+    top = sorted(per_record.items(), key=lambda kv: -kv[1])[:6]
+    for va, c in top:
+        print('    %#010x  %-32s x%d' % (va, rec[va][1], c))
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/re-capture/link2_session.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -u
+export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
+export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages
+SD="$(cd "$(dirname "$0")" && pwd)"
+SECS="${1:-180}"; EVERY="${2:-10}"; HUNT="${3:-1}"
+CFG=/tmp/nav-live.json
+"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; }
+if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then
+  SYLPH_HUNT="$HUNT" SYLPH_KILL_TURRETS=1 nohup python3 "$SD/pilot.py" "$CFG" "$SECS" \
+      </dev/null >/tmp/live-pilot.log 2>&1 &
+  PILOT=$!; echo "--- pilot (SYLPH_HUNT=$HUNT)"
+else PILOT=""; echo "--- BIND FAILED, no pilot"; fi
+python3 "$SD/link2_probe.py"  ; rc=$?
+[ -n "$PILOT" ] && kill "$PILOT" 2>/dev/null
+echo "LIVENESS DONE rc=$rc"

tools/re-capture/link_probe.py

@@ -120,6 +120,32 @@ def main():
             if rlo <= p <= rhi + 0x400: into_ros += 1
     print('  craft words pointing into the roster VA range: %d' % into_ros)
 
+    # Settle the 5101: if craft point at roster_base + X for a fixed X, the
+    # distance to the nearest roster base below spikes at one delta. Flat means
+    # they are unrelated allocations sharing a heap.
+    import bisect
+    rb = sorted(rvas)
+    dh = collections.Counter(); far = 0
+    for off, nm in cr:
+        blob = os.pread(fd, WIN, off)
+        for k in range(0, len(blob) - 3, 4):
+            (p,) = struct.unpack_from('>I', blob, k)
+            if not (rlo <= p <= rhi + 0x400): continue
+            i = bisect.bisect_right(rb, p) - 1
+            if i < 0: continue
+            d = p - rb[i]
+            if d <= 0x4000: dh[d] += 1
+            else: far += 1
+    print('\n--- craft pointer -> distance to nearest roster base ---')
+    tot = sum(dh.values()) + far
+    print('  in range: %d  (>0x4000 from any base: %d)' % (tot, far))
+    for d, c in dh.most_common(12):
+        print('    +%#07x  x%-5d  %5.1f%%' % (d, c, 100.0 * c / max(1, tot)))
+    if dh:
+        top = dh.most_common(1)[0]
+        print('  top delta accounts for %.1f%% of in-range pointers'
+              % (100.0 * top[1] / max(1, tot)))
+
     print('\n--- reverse: craft -> roster record VA ---')
     print('  craft with >=1 roster pointer: %d / %d' % (rlinked, len(cr)))
     print('  fan-in histogram:', sorted(rev.items())[:10])