diff --git a/docs/re/remaining-ob-hunt.md b/docs/re/remaining-ob-hunt.md new file mode 100644 index 0000000..ecb74c4 --- /dev/null +++ b/docs/re/remaining-ob-hunt.md @@ -0,0 +1,63 @@ +# Hunting REMAINING OB by correlation — method works, run did not finish + +Status: ✅ the correlation method is sound and demonstrated; 🔴 the hunt is +**unfinished**; 🔴 two self-inflicted defects, one a repeat. + +## The method + +[mission-phase-objectives.md](mission-phase-objectives.md) settles what the +counter is: Stage 02 phase 1 asks to *"shoot down all invading enemy fighters"* +and the hints say red **[OB]** markers indicate the targets. So `REMAINING OB` +must fall when a marked fighter dies — and the per-record craft strength already +says exactly when that happens, for a *named* unit. + +So instead of scanning for a value, intersect: keep every word in the 32 MB heap +that fell by the same amount, in the same interval, as an `e010` loss. Each kill +event should cut the survivor set hard. + +`tools/re-capture/ob_probe2.py`. + +## ✅ It works — one event cut 8 million words to 1056 + +``` +t= 219s e010 losses=2 (all=4) words falling by 2: 1056 -> candidates 1056 +``` + +From ~8 M candidate words to **1056** on a single event. Two or three more should +leave a handful. + +## 🔴 The run did not finish, and the candidates were lost + +The turn's timeout fired at t = 219 s, and the probe saved its candidate set +**only at the end** — so the 1056 were discarded. The follow-up attach then +started from nothing. + +**This is the same mistake already recorded in +[guest-stalls.md](guest-stalls.md)**: "the first attempt deferred all analysis to +the end of the run, and the turn's timeout killed it with 240 s of collected data +still in memory and nothing written." I wrote that lesson down and then repeated +it in a new script four iterations later. + +Fixed: candidates are now written to `/tmp/ob_candidates.json` **after every +event**, and `SYLPH_OB_RESUME=1` loads them so a chained attach keeps +intersecting on the same mission. + +## 🔴 The attach could not tell a quiet mission from a frozen one + +The follow-up attach logged **535 s with zero losses of any kind**. That is +exactly what a freeze looks like, and `ob_probe2` had no stall witness, so the +run cannot say which it was. Also fixed — the witness from `wave7_probe` is now +carried here. + +Two defects in one iteration, both of them things this corpus had already +learned. The pattern is that each new probe starts from scratch and re-earns the +same lessons; the fix that would actually stick is a shared probe harness rather +than a family of one-off scripts. + +## What is still open + +The hunt itself. The method is demonstrated but no address is identified. What it +needs is a run that catches **two or three** `e010` kill events, which is the same +combat-effectiveness limit recorded in +[mission-objectives-text.md](mission-objectives-text.md) — the preference knob +gets about two marked-fighter kills per five minutes against a dozen turrets. diff --git a/tools/re-capture/ob2_attach.sh b/tools/re-capture/ob2_attach.sh new file mode 100755 index 0000000..aebeabb --- /dev/null +++ b/tools/re-capture/ob2_attach.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -u +export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98 +export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages +SD="$(cd "$(dirname "$0")" && pwd)" +pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } +CFG=/tmp/nav-ob2.json +for try in 1 2 3; do + python3 "$SD/pad.py" set "rt=1" >/dev/null 2>&1 || true; sleep 3 + python3 "$SD/pad.py" clear >/dev/null 2>&1 || true + if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then + SYLPH_HUNT=1 SYLPH_KILL_TURRETS=1 SYLPH_KEEPOUT=1400 SYLPH_PREFER=e010 \ + nohup python3 "$SD/pilot.py" "$CFG" "${1:-500}" /tmp/ob2-pilot.log 2>&1 & + P=$!; echo "--- pilot attached"; break + fi +done +SYLPH_OB_RESUME=1 python3 "$SD/ob_probe2.py" "${1:-500}" "${2:-20}" +[ -n "${P:-}" ] && kill "$P" 2>/dev/null +echo "OB2 ATTACH DONE" diff --git a/tools/re-capture/ob2_session.sh b/tools/re-capture/ob2_session.sh new file mode 100755 index 0000000..4c96b05 --- /dev/null +++ b/tools/re-capture/ob2_session.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -u +export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98 +export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages +SD="$(cd "$(dirname "$0")" && pwd)" +SECS="${1:-180}"; EVERY="${2:-10}"; HUNT="${3:-1}" +CFG=/tmp/nav-live.json +"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; } +# The bind is intermittent and a failed bind means no pilot, no kills and a +# completely uninformative run. Retry before giving up, and abort if it never +# takes rather than silently flying an unattended craft. +# entities2 self finds the player by MOTION between two samples, so a craft that +# is sitting still at mission start is invisible and the bind fails -- three +# times in a row on the run that added this retry. Nudge the throttle first so +# there is something to see, then bind. +BOUND=0 +for try in 1 2 3; do + python3 "$SD/pad.py" set "rt=1" >/dev/null 2>&1 || true + sleep 3 + python3 "$SD/pad.py" clear >/dev/null 2>&1 || true + if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then BOUND=1; break; fi + echo "--- bind attempt $try failed, retrying"; sleep 5 +done +if [ "$BOUND" = 1 ]; then + SYLPH_HUNT="$HUNT" SYLPH_KILL_TURRETS=1 SYLPH_KEEPOUT="${SYLPH_KEEPOUT:-1400}" SYLPH_HZ="${SYLPH_HZ:-8.0}" SYLPH_PREFER="${SYLPH_PREFER:-}" nohup python3 "$SD/pilot.py" "$CFG" "$SECS" \ + /tmp/live-pilot.log 2>&1 & + PILOT=$!; echo "--- pilot (SYLPH_HUNT=$HUNT)" +else echo "BIND FAILED after 3 attempts -- aborting, an unattended run tests nothing"; exit 4; fi +python3 "$SD/ob_probe2.py" "$SECS" "$EVERY"; rc=$? +[ -n "$PILOT" ] && kill "$PILOT" 2>/dev/null +echo "LIVENESS DONE rc=$rc" diff --git a/tools/re-capture/ob_probe2.py b/tools/re-capture/ob_probe2.py new file mode 100755 index 0000000..e03b1ba --- /dev/null +++ b/tools/re-capture/ob_probe2.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Find REMAINING OB by correlating it with marked-fighter kills. + +The objective text settles what the counter is: Stage 02 phase 1 asks to "shoot +down all invading enemy fighters", and the hints say red [OB] markers indicate +the targets (mission-phase-objectives.md). So REMAINING OB must decrement when a +marked fighter dies -- and the per-record craft strength already tells us +exactly when that happens, for a named unit. + +Rather than scan for a value, intersect candidates across kill events: keep every +word in the heap that fell by the same amount in the same interval as an e010 +loss. Two or three events should leave very few. +""" +import os, sys, time, struct, collections +sys.path.insert(0, __file__.rsplit('/', 1)[0]) +import gmem, gworld, entities2 +import numpy as np + +ROSTER_VT = struct.pack('>I', 0x820AF030) +DELTA, WIN, LINK, HULL = 0x130, 0x400, 0x08, 0x154 +LO, HI = 0xBD000000, 0xBE000000 +WATCH = os.environ.get('SYLPH_OB_UNIT', 'e010') + +def scan_vt(fd, size, vt): + out = [] + for a, b in gmem.extents(fd, size): + pos = a + while pos < b: + m = min(1 << 24, b - pos) + blob = os.pread(fd, m, pos) + i = blob.find(vt) + while i != -1: + if (pos + i) % 4 == 0: out.append(pos + i) + i = blob.find(vt, i + 1) + pos += m + return sorted(out) + +def region(fd): + lo, hi = gmem.va_to_off(LO), gmem.va_to_off(HI) + out, pos = bytearray(), lo + while pos < hi: + n = min(1 << 24, hi - pos); out += os.pread(fd, n, pos); pos += n + return np.frombuffer(bytes(out), dtype='>u4').astype(np.int64), lo + +def strengths(fd, defs, want): + lo, hi = gmem.va_to_off(entities2.ENT_VA_LO), gmem.va_to_off(entities2.ENT_VA_HI) + per = collections.Counter(); pos = lo + while pos < hi: + m = min(1 << 24, hi - pos) + blob = os.pread(fd, m, pos) + for needle, nm in defs.items(): + i = blob.find(needle) + while i != -1: + if (pos + i) % 4 == 0: + base = pos + i - DELTA + try: alive = struct.unpack('>f', os.pread(fd, 4, base + HULL))[0] > 0 + except Exception: alive = False + if alive: + head = os.pread(fd, WIN, base) + for j in range(0, len(head) - 3, 4): + (p,) = struct.unpack_from('>I', head, j) + if p in want: per[(want[p], nm)] += 1; break + i = blob.find(needle, i + 1) + pos += m + return per + +def main(): + secs = int(sys.argv[1]) if len(sys.argv) > 1 else 300 + every = int(sys.argv[2]) if len(sys.argv) > 2 else 20 + import json + w = gworld.World(); fd = w.fd + defs = entities2.definitions(w) + roster = scan_vt(fd, w.size, ROSTER_VT) + want = {} + for o in roster: + va = gmem.primary_va(o) + if va is not None: want[va + LINK] = o + print('roster %d, definitions %d, watching "%s"' % (len(roster), len(defs), WATCH), flush=True) + # A run with no losses reads exactly like a frozen guest. The attach that + # followed this probe's first run logged 535 s of zero losses with no way to + # tell which it was. Carry the witness. + r0, _ = region(fd); time.sleep(3.0); r1, _ = region(fd) + d = r1 - r0 + rates = collections.Counter(int(x // 3) for x in d[(d > 15) & (d < 600)][:200000]) + band = [r for r in rates if 8 <= r <= 40] + pick = max(band, key=lambda r: rates[r]) if band else None + ticks = list(np.nonzero((d // 3 == pick))[0][:32]) if pick else [] + print('tick witnesses: %d at %s/s' % (len(ticks), pick), flush=True) + prev_s = strengths(fd, defs, want) + prev_r, base = region(fd) + last_t = prev_r[ticks] if ticks else None + # Persist candidates so a chained attach can keep intersecting on the SAME + # mission -- one call rarely catches enough kill events on its own. + CAND = '/tmp/ob_candidates.json' + cand = None; events = 0 + if os.environ.get('SYLPH_OB_RESUME') == '1' and os.path.exists(CAND): + cand = set(json.load(open(CAND))) + print('resumed %d candidates from a previous call' % len(cand), flush=True) + t0 = time.time() + while time.time() - t0 < secs: + time.sleep(every) + cur_s = strengths(fd, defs, want) + cur_r, _ = region(fd) + el = round(time.time() - t0) + lost = sum(max(0, prev_s[k] - cur_s.get(k, 0)) + for k in prev_s if WATCH in k[1]) + allo = sum(max(0, prev_s[k] - cur_s.get(k, 0)) for k in prev_s) + if lost: + events += 1 + d = prev_r - cur_r # positive where a word FELL + hit = np.nonzero(d == lost)[0] + s = set(hit.tolist()) + cand = s if cand is None else (cand & s) + json.dump(sorted(cand), open(CAND, 'w')) # save NOW: the previous + # version saved only at the end and a turn timeout destroyed 1056 + # hard-won candidates -- the same "report at the end" mistake already + # recorded once in guest-stalls.md. + print(' t=%4ds %s losses=%d (all=%d) words falling by %d: %d -> candidates %d (saved)' + % (el, WATCH, lost, allo, lost, len(s), len(cand)), flush=True) + else: + st = '' + if ticks: + now_t = cur_r[ticks] + if int((now_t > last_t).sum()) == 0: st = ' *** GUEST STALLED ***' + last_t = now_t + print(' t=%4ds no %s loss (all losses=%d)%s' % (el, WATCH, allo, st), flush=True) + prev_s, prev_r = cur_s, cur_r + if cand is not None: + json.dump(sorted(cand), open(CAND, 'w')) + print('saved %d candidates for the next call' % len(cand), flush=True) + print('\nevents: %d' % events) + if cand: + print('surviving candidates: %d' % len(cand)) + for i in sorted(cand)[:12]: + off = base + i * 4 + va = gmem.primary_va(off) + print(' va %s value %d' % (('%#010x' % va) if va else '?', int(prev_r[i]))) + else: + print('no candidate survived -- either no kills, or OB is not a plain u32 here') + return 0 + +if __name__ == '__main__': + sys.exit(main())