backlog: phase-advance solved, hashes located, nav bug fixed, tooling corrections

tools/re-capture/phase_probe.py

@@ -1,74 +1,62 @@
 #!/usr/bin/env python3
-"""Locate the loaded stage tables in guest RAM and watch for a phase counter.
+"""Watch the mission's phase ordinal change, live.
 
-The stage record splits a mission into Phase_1..3 and Route_S<NN>.tbl tags every
-arrival path with a phase (docs/re/structures/stage-mission-tables.md). Nothing
-static says what *advances* a phase, so this looks for the runtime side: find
-the table strings in RAM, then diff the words around them over time.
+The static work (docs/re/mission-phase-advance.md) found two 1/2/3 fields:
+
+  [ScriptMission+40]            the counter itself; incremented at 0x82260A00
+  [*(0x828F35F8) + 236]         a mirror, written by CScriptInterpreter::
+                                ChangePhase (opcode 995) after the fact
+
+The mirror is the one reachable without a debugger: 0x828F35F8 is a static
+singleton pointer in guest memory, so this is two reads through /dev/shm and
+needs no gdb -- which matters, because booting under gdb costs ~300s.
+
+This is the first DIRECT observation of a phase advance. Everything about phases
+so far is either static (route names, disassembly) or inferred; nobody has
+watched the number change.
+
+Usage: phase_probe.py [secs] [every_s]
 """
-import subprocess, sys, time, re, collections
+import sys, time
+sys.path.insert(0, __file__.rsplit('/', 1)[0])
+import gmem
 
-SD = __file__.rsplit('/', 1)[0]
+SINGLETON_PTR = 0x828F35F8      # static pointer to the mission-manager singleton
+PHASE_OFF = 236                 # the mirror ChangePhase writes
 
-def gmem(*args):
-    r = subprocess.run([sys.executable, SD + '/gmem.py'] + list(args),
-                       capture_output=True, text=True, timeout=300)
-    return r.stdout
+_FD = None
 
-def find(pat):
-    # gmem prints "<file offset>  va <guest va>" per hit. Match the va column
-    # only -- a bare 0x[0-9a-f]{8} also catches the offset, which is not an
-    # address and silently doubles the anchor list with junk.
-    out = gmem('find', pat)
-    return [int(m, 16) for m in re.findall(r'va 0x([0-9a-f]{8})', out)]
+def _fd():
+    """gmem exposes va_to_off/mem_path but no reader, so open the image once."""
+    global _FD
+    if _FD is None:
+        _FD = open(gmem.mem_path(), 'rb', buffering=0)
+    return _FD
 
-def words(va, n):
-    out = gmem('words', hex(va), str(n))
-    return [int(m, 16) for m in re.findall(r'\b([0-9a-f]{8})\b', out)]
+def u32(va):
+    off = gmem.va_to_off(va)
+    if off is None:
+        return None
+    f = _fd()
+    f.seek(off)
+    b = f.read(4)
+    return int.from_bytes(b, 'big') if len(b) == 4 else None
 
-NEEDLES = ['Phase_1', 'Phase_2', 'Route_ADN101_p1F', 'SUBOBJ_010',
-           'AI_ADAN_CraftSquadron_Veteran', 'UnitGroup_S02.tbl']
-
-def main():
-    secs = int(sys.argv[1]) if len(sys.argv) > 1 else 180
-    hits = {}
-    for n in NEEDLES:
-        v = find(n)
-        hits[n] = v
-        print('%-32s %d hit(s) %s' % (n, len(v), [hex(x) for x in v[:4]]))
-    anchors = []
-    for n, v in hits.items():
-        for va in v[:2]:
-            anchors.append((n, va))
-    if not anchors:
-        print('NO TABLE STRINGS IN RAM -- the stage data is not resident, or the '
-              'run never reached flight'); return 2
-    base = {}
-    for n, va in anchors:
-        lo = (va - 0x400) & ~3
-        base[(n, va)] = words(lo, 512)
-    print('\nbaseline captured for %d anchors; watching %ds' % (len(anchors), secs))
-    t0 = time.time()
-    changed = collections.Counter()
-    while time.time() - t0 < secs:
-        time.sleep(20)
-        for n, va in anchors:
-            lo = (va - 0x400) & ~3
-            now = words(lo, 512)
-            b = base[(n, va)]
-            for i, (x, y) in enumerate(zip(b, now)):
-                if x != y:
-                    changed[(n, lo + i * 4, x, y)] += 1
-            base[(n, va)] = now
-        print('  t=%4ds  distinct changing words so far: %d'
-              % (time.time() - t0, len(changed)))
-    print('\n--- words that changed near a stage-table string ---')
-    for (n, va, x, y), c in changed.most_common(40):
-        print('  %-32s va=0x%08x  %08x -> %08x   (%d times)' % (n, va, x, y, c))
-    if not changed:
-        print('  none -- the loaded tables sit in read-only memory, so the '
-              'runtime phase state is NOT adjacent to them')
-    return 0
+def sample():
+    base = u32(SINGLETON_PTR)
+    if not base or not (0x10000 <= base < 0xFFFFFFFF):
+        return None, base
+    return u32(base + PHASE_OFF), base
 
 if __name__ == '__main__':
-    sys.exit(main())
+    secs = float(sys.argv[1]) if len(sys.argv) > 1 else 600
+    every = float(sys.argv[2]) if len(sys.argv) > 2 else 5
+    t0 = time.time(); last = object()
+    print('singleton ptr 0x%08X, phase at +%d' % (SINGLETON_PTR, PHASE_OFF), flush=True)
+    while time.time() - t0 < secs:
+        ph, base = sample()
+        if ph != last:
+            print('  [%6.1fs] singleton=%s phase=%s' % (
+                time.time() - t0, ('0x%08X' % base) if base else base, ph), flush=True)
+            last = ph
+        time.sleep(every)