diff --git a/docs/re/data/submenu-focus-sweep-unmeasured.txt b/docs/re/data/submenu-focus-sweep-unmeasured.txt new file mode 100644 index 00000000..0ff237c2 --- /dev/null +++ b/docs/re/data/submenu-focus-sweep-unmeasured.txt @@ -0,0 +1,70 @@ +# Do LOAD GAME, TUTORIAL and OPTIONS remember their cursors? +# 2026-08-30. ❔ STILL UNMEASURED. Two sweeps, two different instrument faults. +# +# The main menu PERSISTS; EXTRAS RESETS. Two screens, two behaviours, so every +# screen needs measuring and none can be inferred. NEW GAME stays deliberately +# untested. These three are what is left, and they are still open. +# +################################################################################ +# SWEEP 1 -- voided by a reader that only works on one screen. +# +# All three voided on "the ring did not move" after a DELIVERED DOWN: +# LOAD GAME ring 288.0 -> 288.0 +# TUTORIAL ring 385.0 -> None +# OPTIONS ring 287.5 -> 287.5 +# +# 🔴 THE RING HAD MOVED. Differencing S1 against S2 shows exactly where: +# LOAD GAME 0.25% of frame changed, x 97..231 +# TUTORIAL 2.31% x 338..1099 +# OPTIONS 2.59% x 153..479 +# +# ring_row.py scans x 500:542 -- the MAIN MENU's gutter. EXTRAS happened to put +# its ring in that column, which is why the EXTRAS measurement was sound; these +# three do not. The reader was finding a STATIC element and reporting no motion. +# ⚠️ A reader calibrated on one screen is not a reader for the others, and +# "it worked last time" is what made that invisible. +# +################################################################################ +# SWEEP 2 -- the reader replaced, and defeated by an OVERLAY instead. +# +# Decision rule changed to need no ring at all: S1 and S2 differ ONLY by cursor +# position, so compare the re-entry frame to each over the whole frame. +# ✅ CONTROLLED on the EXTRAS frames, whose answer is already known: +# E1 vs E2 (the move) 2.70% -> localised, control passes +# E3 vs E1 (opened) 0.02% +# E3 vs E2 (left on) 2.71% verdict RESETS, and RESETS is the known answer. +# +# LOAD GAME S1 opened (glyph 12838), DOWN changed 6.00% -- control PASSED +# then: TIMEOUT waiting for the main menu. TUTORIAL and OPTIONS +# skipped, because the probe refuses to measure from an unknown +# state rather than pressing on. +# +# 🔴 THE GUEST HAD CRASHED -- and it is the crash this corpus already documents. +# PC 0x82307128, guest thread 9, Access Violation read at 0x000000010000000C, +# preceded by HostPathDevice::ResolvePath(\aab216c3) and "Guest attempted to +# throw a C++ exception!" -- the STL map/set erase of title-crash-stl-tree.md. +# ⚠️ NOT a new finding: that page has it, and autopilot-knowledge-sources.md +# counts 384 dumps at the same PC. The first dump here is at log line 2204, so it +# fired EARLY IN THE BOOT and the dialogs stacked; it is not something Ⓑ did. +# +# ⚠️ AND THE "MORE ROBUST" RULE WAS THE MORE FRAGILE ONE. Xenia's crash dialog +# covers the middle of the screen, so a WHOLE-FRAME identity test can never match +# again once it appears. The narrow ring column the dialog does not cover read the +# screen correctly the whole time. A global comparison is defeated by any overlay; +# a narrow measured feature survives it. I replaced the reader with the global +# rule *because* the reader had just failed, and traded one blindness for another. +# +################################################################################ +# ✅ REFUTATION ATTEMPT -- "Ⓑ on a submenu returns to the parent with focus +# restored to the item you entered from" (4/4, another agent's). IT SURVIVES, and +# the run that looked like a failure is what corroborates it. +# +# The stuck frame -- main menu visible behind the crash dialog -- has the ring at +# y 303.5, which the calibrated reader names LOAD GAME. That is exactly the item +# the probe entered from. So Ⓑ did return AND did restore focus; only my screen +# test failed. A FIFTH instance, from a run I had written off. +# +# ❔ WHAT IS STILL OPEN: whether LOAD GAME, TUTORIAL or OPTIONS persist. Nothing +# here measures it. What a third attempt needs is a per-screen ring locator (the +# S1->S2 difference gives the cursor's own region on each screen) plus a screen +# test that ignores the dialog rectangle -- or a boot without the crash. diff --git a/tools/re-capture/extras_focus_session.sh b/tools/re-capture/extras_focus_session.sh index b70946cc..2c25fdb7 100755 --- a/tools/re-capture/extras_focus_session.sh +++ b/tools/re-capture/extras_focus_session.sh @@ -22,4 +22,4 @@ nohup run-canary --apu=sdl --log_mask=13 --log_level=2 \ # reach the title and the menu with the probe that already does it python3 "$SD/focus_persistence.py" "$LOG" "$OUT/reach" 900 >"$OUT/reach.log" 2>&1 echo "-- reached the menu; now the EXTRAS question --" -python3 "$SD/extras_focus_persistence.py" "$LOG" "$OUT" 600 +python3 "${SWEEP:-$SD/extras_focus_persistence.py}" "$LOG" "$OUT" 600 diff --git a/tools/re-capture/submenu_focus_sweep.py b/tools/re-capture/submenu_focus_sweep.py new file mode 100644 index 00000000..a4a30aaa --- /dev/null +++ b/tools/re-capture/submenu_focus_sweep.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Does EACH submenu remember its cursor across leave -> re-enter? + +The main menu PERSISTS (focus-persists-across-title.txt); EXTRAS RESETS +(extras-focus-resets.txt). Two screens, two behaviours, so there is no menu-wide +rule and every screen has to be measured. This sweeps the three that are left: +LOAD GAME, TUTORIAL, OPTIONS. + +❔ NEW GAME is deliberately NOT entered -- the corpus has held it untested +because it starts a game, and nothing here is worth breaking that for. + +Controls, each of which exists because an earlier run failed without it: + * ABSOLUTE row check after EVERY navigation press, not just the total -- a + constant offset passes a differential control exactly + (menu-focus-reader-offset.txt); + * SCREEN IDENTITY against a reference frame captured in this same run -- the + glyph window cannot separate the main menu from a submenu (327 / 324 / 317); + * 🔴 NO RING READER INSIDE A SUBMENU. ring_row.py scans x 500:542, which is + the MAIN MENU's gutter. EXTRAS happened to put its ring in that column; + LOAD GAME, TUTORIAL and OPTIONS do not (their cursors move at x 97..231, + 338..1099 and 153..479), so the first sweep read a STATIC element and all + three voided on "the ring did not move". They had moved. The decision here + needs no ring at all: S1 and S2 differ ONLY by cursor position, so compare + S3 to each of them over the whole frame; + * every press confirmed from the guest's own [RE-INPUT] log. + +Any control failing SKIPS that screen and moves on; it never reports a number it +cannot justify. + + submenu_focus_sweep.py LOG OUTDIR [wait_s] +""" +import os, re, subprocess, sys, time +import numpy as np +from PIL import Image + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from ring_row import ring_row, main_menu_item, NAMES + +LOG, OUT = sys.argv[1], sys.argv[2] +W, H = 1280, 720 +PAD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pad.py") +TARGETS = [1, 2, 3] # LOAD GAME, TUTORIAL, OPTIONS. NEW GAME excluded. +T0 = time.time() + + +def deliveries(vk): + pat = re.compile((r"RE-INPUT\] XamInputGetKeystrokeEx -> user=\d+ vk=%s flags=0001" % vk).encode()) + try: + return len(pat.findall(open(LOG, "rb").read())) + except FileNotFoundError: + return 0 + + +def press(btn, vk, tries=5): + for _ in range(tries): + before = deliveries(vk) + subprocess.run([sys.executable, PAD, "tap", btn, "0.5"], check=False) + for _ in range(20): + time.sleep(0.25) + if deliveries(vk) > before: + return True + print(f"[{time.time()-T0:7.1f}s] 🔴 {btn} NEVER delivered", flush=True) + return False + + +def _open(): + return subprocess.Popen( + ["ffmpeg", "-loglevel", "error", "-f", "x11grab", "-draw_mouse", "0", + "-video_size", f"{W}x{H}", "-i", ":98", "-r", "4", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + stdout=subprocess.PIPE, bufsize=W * H * 3 * 2) + + +def fresh(): + q = _open(); a = None + for _ in range(3): + b = q.stdout.read(W * H * 3) + if len(b) == W * H * 3: + a = np.frombuffer(b, np.uint8).reshape(H, W, 3).astype(int) + q.kill() + return a + + +def img(a): + return Image.fromarray(a.astype(np.uint8)) + + +def differs(a, b): + return float((np.abs(a - b).max(axis=2) > 24).mean()) + + +def glyph(a): + r, g, bl = a[:, :, 0], a[:, :, 1], a[:, :, 2] + return int(((g > 130) & (g - r > 45) & (g - bl > 45)).sum()) + + +def wait_until(pred, what, limit=60): + t = time.time() + while time.time() - t < limit: + a = fresh() + if a is not None and pred(a): + return a + print(f"[{time.time()-T0:7.1f}s] 🔴 TIMEOUT waiting for {what}", flush=True) + return None + + +os.makedirs(OUT, exist_ok=True) +MAIN = wait_until(lambda a: main_menu_item(ring_row(img(a))) is not None + and 250 <= glyph(a) <= 420, "the main menu", 150) +if MAIN is None: + sys.exit("never identified the main menu") +img(MAIN).save(f"{OUT}/main-ref.png") +print(f"[{time.time()-T0:7.1f}s] MAIN MENU reference, focus = " + f"{NAMES[main_menu_item(ring_row(img(MAIN)))]}", flush=True) + +results = {} +for tgt in TARGETS: + name = NAMES[tgt] + print(f"\n=========== {name} ===========", flush=True) + a = wait_until(lambda a: differs(a, MAIN) < 0.15, "the main menu", 60) + if a is None: + print(f" SKIP {name}: not on the main menu"); continue + cur = main_menu_item(ring_row(img(a))) + if cur is None: + print(f" SKIP {name}: no main-menu ring row"); continue + ok = True + for step in range((tgt - cur) % 5): + if not press("DOWN", "5811"): + ok = False; break + time.sleep(1.2) + y = ring_row(img(fresh())); i = main_menu_item(y) + want = (cur + step + 1) % 5 + print(f" step {step+1}: ring y {y} -> {NAMES[i] if i is not None else '??'} " + f"(want {NAMES[want]})", flush=True) + if i != want: + print(f" 🔴 CONTROL FAILED walking to {name}"); ok = False; break + if not ok: + results[name] = "skipped (navigation control failed)"; continue + + if not press("A", "5800"): + results[name] = "skipped (A not delivered)"; continue + if wait_until(lambda x: differs(x, MAIN) > 0.20, f"{name} to open", 40) is None: + results[name] = "skipped (A did not change the screen)"; continue + time.sleep(3.0) + S1 = fresh(); img(S1).save(f"{OUT}/{name.replace(' ','_')}-S1.png") + print(f" S1 opened: glyph {glyph(S1)}, " + f"{100*differs(S1, MAIN):.1f}% from main", flush=True) + + if not press("DOWN", "5811"): + results[name] = "skipped (DOWN not delivered inside)"; continue + time.sleep(2.5) + S2 = fresh(); img(S2).save(f"{OUT}/{name.replace(' ','_')}-S2.png") + moved = differs(S1, S2) + print(f" S2 after 1 DOWN: {100*moved:.2f}% of the frame changed", flush=True) + # CONTROL: the cursor must have moved, and by a LOCALISED amount -- a whole + # screen changing means the DOWN left the screen, not moved a cursor. + if moved < 0.0005: + results[name] = f"VOID (control): nothing changed on DOWN ({100*moved:.3f}%)" + press("B", "5801"); time.sleep(3); continue + if moved > 0.20: + results[name] = f"VOID (control): {100*moved:.1f}% changed — DOWN left the screen" + press("B", "5801"); time.sleep(3); continue + print(f" ✅ CONTROL PASSED: a localised change ({100*moved:.2f}%)", flush=True) + + if not press("B", "5801"): + results[name] = "skipped (B not delivered)"; continue + if wait_until(lambda x: differs(x, MAIN) < 0.15, "the main menu", 60) is None: + results[name] = "VOID: B did not return to the main menu"; continue + if not press("A", "5800"): + results[name] = "skipped (A not delivered on re-entry)"; continue + if wait_until(lambda x: differs(x, MAIN) > 0.20, f"{name} to reopen", 40) is None: + results[name] = "VOID: re-entry did not change the screen"; continue + time.sleep(3.0) + S3 = fresh(); img(S3).save(f"{OUT}/{name.replace(' ','_')}-S3.png") + d1, d2 = differs(S3, S1), differs(S3, S2) + print(f" S3 re-entered: {100*d1:.2f}% from S1 (opened on), " + f"{100*d2:.2f}% from S2 (left on)", flush=True) + if min(d1, d2) > 0.15: + results[name] = f"VOID: re-entry matches neither ({100*d1:.1f}% / {100*d2:.1f}%)" + elif d2 < d1 * 0.5: + results[name] = f"PERSISTS (S3 {100*d2:.2f}% from where left, {100*d1:.2f}% from opened)" + elif d1 < d2 * 0.5: + results[name] = f"RESETS (S3 {100*d1:.2f}% from opened, {100*d2:.2f}% from where left)" + else: + results[name] = f"UNDECIDED ({100*d1:.2f}% / {100*d2:.2f}%)" + print(f" => {results[name]}", flush=True) + press("B", "5801"); time.sleep(3) + +print("\n================ SUMMARY ================") +for k, v in results.items(): + print(f" {k:12} {v}") +print("SUBMENU FOCUS SWEEP DONE", flush=True)