From 015fb7d21e4d0e200b7c1502bf776ec4df5453fd Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Mon, 24 Aug 2026 16:54:52 +0000 Subject: [PATCH] re: persistence rule works; the stall witness gives false positives Run 12, cheap probe with a bound pilot: 16 losses over 290 s and zero confirmed arrivals, making twelve runs without one. The persistence rule earned its place immediately -- an increase of 13 to 15 was surfaced and correctly not counted, since it does not start from zero. Under the previous rule it would have been invisible, and a similar flicker straddling zero was nearly written up last iteration as the first arrival. The stall witness, on the other hand, is unreliable. Thirteen samples were flagged GUEST STALLED while recording losses in those same samples, and a frozen guest cannot destroy craft, so they are false positives and the run was healthy. The cause is the selection rule: it took the first word in a 4 MB window whose rate fell in a plausible band, and plenty of counters advance intermittently without saying anything about whether frames are being rendered. timer_probe had already solved this properly -- 286 candidates, a rate histogram with a dominant cluster near 17/s -- and that lesson was not carried over when the witness was bolted onto the probe. Now fixed to a majority vote: collect every candidate, keep the modal-rate cluster, sample up to 32 of them, and report a stall only when fewer than half advance. It also prints RUN UNVALIDATED when no witness is found, because an earlier run printed "stalled samples=0" alongside "tick witness: NONE", and a witness that does not exist cannot report zero stalls. Not yet run. Consequence worth flagging: the "0 stalled samples" that validated the cheap probe last iteration came from this same unreliable witness and should be re-confirmed under the majority rule. The pilot-log speed analysis that established the stalls in the first place is unaffected. Also fixed: the entity bind now retries three times and aborts if it never takes, instead of silently flying an unattended craft -- one run was wasted that way this iteration, producing no kills and no information. --- docs/re/BACKLOG.md | 16 ++++++++++- docs/re/guest-stalls.md | 45 +++++++++++++++++++++++++++++++ docs/re/mission-arrival-watch.md | 22 +++++++++++++++ tools/re-capture/wave7_probe.py | 44 +++++++++++++++++++++--------- tools/re-capture/wave7_session.sh | 12 +++++++-- 5 files changed, 124 insertions(+), 15 deletions(-) diff --git a/docs/re/BACKLOG.md b/docs/re/BACKLOG.md index 73c46bd..67b6edd 100644 --- a/docs/re/BACKLOG.md +++ b/docs/re/BACKLOG.md @@ -418,7 +418,21 @@ search cannot find a *schedule*. reverted to 0 fifteen seconds later, and the same log shows a record reading 13 then 14 with no event printed because only decreases were surfaced. Count stands at **0 confirmed arrivals in 11 runs**. Probe now prints every increase - and requires an arrival to **persist across 2 samples** (not yet run). ❔ Multi-squadron threshold test still not run (zero losses that run). + and requires an arrival to **persist across 2 samples**. +* ✅🔴 **Run 12 (2026-08-24): persistence rule works; stall witness does not.** + 16 losses / 290 s with a bound pilot, **0 confirmed arrivals** (twelve runs + now). One increase surfaced (`13→15`) and correctly not counted — exactly the + flicker that nearly became "first arrival". 🔴 **The single-word tick witness + gives FALSE POSITIVES**: 13 samples flagged `GUEST STALLED` *while recording + losses in the same samples*, which a frozen guest cannot do. Cause: it took the + first word in a 4 MB window with a plausible rate; intermittent counters pass + that test. `timer_probe` had already solved this (286 candidates, modal rate + ~17/s) and the lesson was not carried over. **Fixed to a majority vote over the + modal-rate cluster (≤32 witnesses, stall only if 15` — and correctly **not** +counted, because it does not start from zero. Under the pre-fix rule it would +have been invisible; under the old *arrival* rule a similar flicker straddling +zero was nearly written up as the first arrival. The new "print every increase" +behaviour makes the flicker visible as flicker. + +**Still zero confirmed arrivals, now across twelve runs.** + +Caveat on this run: its 13 "stalled" flags are false positives from the +single-word witness (see [guest-stalls.md](guest-stalls.md)), so the run was +healthy — but that also means the witness cannot yet certify it. diff --git a/tools/re-capture/wave7_probe.py b/tools/re-capture/wave7_probe.py index b7568a2..62e300d 100755 --- a/tools/re-capture/wave7_probe.py +++ b/tools/re-capture/wave7_probe.py @@ -74,14 +74,29 @@ def main(): label[o] = wave3.resolve_id(f, w.size, o)[0] or '?' want[va + LINK] = o - tick = None + # A 1 MB window was too narrow and found nothing, yet the summary still + # printed "stalled samples=0" -- a witness that does not exist cannot report + # zero stalls, and that is false reassurance, not a clean run. Search wider, + # and if there is still no witness say the run is UNVALIDATED. + # Taking the FIRST word that advances was wrong: it flagged 13 samples as + # stalled in a run that was recording losses in those same samples, which a + # frozen guest cannot do. Some counters simply advance intermittently. Use + # timer_probe's approach instead -- collect every candidate, keep the modal + # rate cluster, and call a stall only when a MAJORITY of that cluster fails + # to advance. lo = gmem.va_to_off(0xBC000000) - a = os.pread(fd, 1 << 20, lo); time.sleep(2.0); b = os.pread(fd, 1 << 20, lo) - for k in range(0, len(a) - 3, 4): + a = os.pread(fd, 1 << 22, lo); time.sleep(3.0); b = os.pread(fd, 1 << 22, lo) + cands = [] + for k in range(0, min(len(a), len(b)) - 3, 4): va, vb = struct.unpack_from('>I', a, k)[0], struct.unpack_from('>I', b, k)[0] - if va < vb and 5 < (vb - va) / 2.0 < 1 << 18: tick = lo + k; break - print('tick witness: %s' % (('%#x' % tick) if tick else 'NONE')) - last_tick = struct.unpack('>I', os.pread(fd, 4, tick))[0] if tick else 0 + if va < vb and 5 < (vb - va) / 3.0 < 200: + cands.append((lo + k, round((vb - va) / 3.0))) + modal = collections.Counter(r for _, r in cands).most_common(1) + ticks = [o for o, r in cands if modal and r == modal[0][0]][:32] + print('tick witnesses: %d candidates, modal rate %s/s, using %d' + % (len(cands), modal[0][0] if modal else '-', len(ticks)) + if ticks else 'tick witnesses: NONE -- RUN UNVALIDATED') + last_ticks = [struct.unpack('>I', os.pread(fd, 4, o))[0] for o in ticks] craft = enumerate_craft(fd, defs, want) print('craft %d, linked %d' % (len(craft), sum(1 for c in craft if c[2]))) @@ -97,10 +112,13 @@ def main(): craft = enumerate_craft(fd, defs, want); last_rescan = time.time() cur = collections.Counter(c[2] for c in craft if c[2] and alive(fd, c[0])) st = '' - if tick: - now = struct.unpack('>I', os.pread(fd, 4, tick))[0] - if now <= last_tick: st = ' *** GUEST STALLED ***'; stalls += 1 - last_tick = now + if ticks: + now = [struct.unpack('>I', os.pread(fd, 4, o))[0] for o in ticks] + moved = sum(1 for x, y in zip(last_ticks, now) if y > x) + if moved * 2 < len(ticks): + st = ' *** GUEST STALLED (%d/%d witnesses moved) ***' % (moved, len(ticks)) + stalls += 1 + last_ticks = now # Report EVERY increase, not just 0 -> n. The first run logged a record # reading 13 then 14 with no event printed, which is how flicker in the # hull-based liveness read hides: only decreases were being surfaced, so @@ -125,8 +143,10 @@ def main(): ' <- candidate arrival' if x == 0 else ''), flush=True) for o, x, y in l_: print(' loss %-30s %d -> %d' % (label.get(o, '?'), x, y), flush=True) prev = cur - print('\nTOTAL candidate-up=%d down=%d CONFIRMED arrivals=%d stalled samples=%d' - % (arr, los, confirmed, stalls)) + if not ticks: + print('\nRUN UNVALIDATED: no tick witness, so flat samples prove nothing.') + print('\nTOTAL candidate-up=%d down=%d CONFIRMED arrivals=%d stalled samples=%s' + % (arr, los, confirmed, stalls if ticks else 'UNKNOWN')) return 0 if __name__ == '__main__': diff --git a/tools/re-capture/wave7_session.sh b/tools/re-capture/wave7_session.sh index 579f3e2..6fb4003 100755 --- a/tools/re-capture/wave7_session.sh +++ b/tools/re-capture/wave7_session.sh @@ -6,11 +6,19 @@ SD="$(cd "$(dirname "$0")" && pwd)" SECS="${1:-180}"; EVERY="${2:-10}"; HUNT="${3:-1}" CFG=/tmp/nav-live.json "$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; } -if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then +# The bind is intermittent and a failed bind means no pilot, no kills and a +# completely uninformative run. Retry before giving up, and abort if it never +# takes rather than silently flying an unattended craft. +BOUND=0 +for try in 1 2 3; do + if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then BOUND=1; break; fi + echo "--- bind attempt $try failed, retrying"; sleep 5 +done +if [ "$BOUND" = 1 ]; then SYLPH_HUNT="$HUNT" SYLPH_KILL_TURRETS=1 SYLPH_KEEPOUT="${SYLPH_KEEPOUT:-1400}" nohup python3 "$SD/pilot.py" "$CFG" "$SECS" \ /tmp/live-pilot.log 2>&1 & PILOT=$!; echo "--- pilot (SYLPH_HUNT=$HUNT)" -else PILOT=""; echo "--- BIND FAILED, no pilot"; fi +else echo "BIND FAILED after 3 attempts -- aborting, an unattended run tests nothing"; exit 4; fi python3 "$SD/wave7_probe.py" "$SECS" "$EVERY"; rc=$? [ -n "$PILOT" ] && kill "$PILOT" 2>/dev/null echo "LIVENESS DONE rc=$rc"