diff --git a/docs/re/captures/title-builds/live-submenu-unidentified.png b/docs/re/captures/title-builds/live-submenu-unidentified.png new file mode 100644 index 00000000..c1d84513 Binary files /dev/null and b/docs/re/captures/title-builds/live-submenu-unidentified.png differ diff --git a/docs/re/data/tbm-submenu-reached.txt b/docs/re/data/tbm-submenu-reached.txt new file mode 100644 index 00000000..20990c04 --- /dev/null +++ b/docs/re/data/tbm-submenu-reached.txt @@ -0,0 +1,42 @@ +# A submenu REACHED and captured -- but not identified. +# +# 2026-08-30, tools/re-capture/tbm_submenu_v2.py. Third attempt; the two +# earlier ones are in tbm-submenu-not-reached.md. All three of that page's +# fixes were applied and all three were needed: +# [ 288.6s] TITLE (glyph 1520) +# [ 289.7s] A delivered (attempt 1) <- confirmed from [RE-INPUT], not the pad +# [ 295.4s] MENU (glyph 327) <- detected, not timed +# [ 299.3s] A delivered (attempt 1) +# [ 303.4s] SUBMENU: 87.1% of pixels differ from the menu, glyph 314 +# +# THE CAPTURE: 1280x720, mean 52.77, 99.7 % of pixels above 16, coverage +# uniform top (0.993) / middle (1.000) / bottom (0.990). A FULL-SCREEN +# BACKGROUND. +# +# OUR RENDERER, on the archives that carry a .tbm decider: +# GP_SAVE_LOAD 19 builds 1.9 - 3.0 % inked mean 2.2 - 3.1 +# GP_TUTORIAL 3 builds 6.0 - 6.4 % inked mean 5.3 - 6.0 +# GP_SYSTEM builds 0,1 78.4 % inked mean 38.7 - 39.4 +# +# So GP_SAVE_LOAD and GP_TUTORIAL render essentially NOTHING while the game +# draws a full screen -- but which screen was captured is NOT established. +# +# 🔴 WHY IDENTIFICATION FAILED, and it is a method point: +# Correlation cannot discriminate when the candidate renders are near-blank. +# A near-empty image has almost no structure to correlate with, so every +# GP_SAVE_LOAD build scores -0.004..-0.010 against the capture -- a ranking +# with no information in it. A matching statistic is useless against a +# hypothesis that predicts an empty image, which is exactly the hypothesis +# under test here. +# +# 🔴 AND THE FOCUS COULD NOT BE READ from the menu capture. Against the two +# labelled references the whole-frame mean absolute difference is 2.52 +# (NEW GAME focused) and 2.48 (OPTIONS focused) -- a 1.6 % separation, far +# too weak to call. s00a-drive-blocked-by-focus.md already records a +# per-row brightness statistic failing its control; this is a second +# statistic failing on the same problem. +# +# ✅ ONE THING WORTH KEEPING: the game surface sits at y=45 in the 1280x720 +# display frame. Cropping m[45:45+675, 0:1279] fits the committed 1279x675 +# captures to a mean absolute difference of 2.5. That is the alignment the +# earlier cross-geometry floor comparison got wrong. diff --git a/docs/re/structures/tbm-submenu-not-reached.md b/docs/re/structures/tbm-submenu-not-reached.md index 3c06b424..00d37e41 100644 --- a/docs/re/structures/tbm-submenu-not-reached.md +++ b/docs/re/structures/tbm-submenu-not-reached.md @@ -1,4 +1,4 @@ -# 🟡 Does a `.tbm` draw pixels? Still open — two runs, and the third timing bug in a row +# 🟡 Does a `.tbm` draw pixels? Still open — but a submenu is now REACHED **Classification: not an answer.** Recorded per *"do not improvise around a blocker"*, and because the *reason* both runs failed is the same mistake three @@ -55,6 +55,52 @@ thing".** The title detector, the menu detector and the delivery check are all t substitution, and each was written only after the timed version had already produced a confident wrong answer. +## ✅ Run 3 — all three fixes applied, all three needed, submenu reached + +``` +[ 288.6s] TITLE (glyph 1520) +[ 289.7s] A delivered (attempt 1) ← confirmed from [RE-INPUT], not from the pad +[ 295.4s] MENU (glyph 327) ← detected, not timed +[ 299.3s] A delivered (attempt 1) +[ 303.4s] SUBMENU: 87.1 % of pixels differ from the menu, glyph 314 +``` + +**The capture is 99.7 % inked**, uniformly top to bottom — a full-screen background +([capture](../captures/title-builds/live-submenu-unidentified.png), +[numbers](../data/tbm-submenu-reached.txt)). + +**Our renderer, on the archives carrying a `.tbm` decider:** + +| archive | builds | inked | +|---|---|---| +| `GP_SAVE_LOAD` | 19 | **1.9 – 3.0 %** | +| `GP_TUTORIAL` | 3 | **6.0 – 6.4 %** | +| `GP_SYSTEM` | 0, 1 | 78.4 % | + +So two of the three render essentially nothing where the game draws a full screen. + +## 🔴 But which screen this is, is NOT established — and the reason is a method point + +**Correlation cannot discriminate when the candidate renders are near-blank.** An +almost-empty image has no structure to correlate against, so all 19 `GP_SAVE_LOAD` +builds score **−0.004 … −0.010** — a ranking with no information in it. ⚠️ **A +matching statistic is useless against a hypothesis that predicts an empty image**, +and that is precisely the hypothesis under test. The instrument is disabled by the +thing it was brought in to detect. + +🔴 **And the focus could not be read either.** Against the two labelled menu +captures the whole-frame mean absolute difference is **2.52** (NEW GAME) and +**2.48** (OPTIONS) — 1.6 % apart, far too weak to call. +[`s00a-drive-blocked-by-focus.md`](../s00a-drive-blocked-by-focus.md) already +records a per-row brightness statistic failing its control; this is a **second** +statistic failing on the same problem, which makes focus identification a real +open item rather than an oversight. + +✅ **One thing worth keeping regardless:** the game surface sits at **y = 45** in the +1280×720 display frame — `m[45:45+675, 0:1279]` fits the committed 1279×675 captures +to a mean absolute difference of **2.5**. That is the alignment the earlier +cross-geometry floor comparison got wrong. + ## What is now validated, and what the next run needs ✅ Working: the plate-pulse title detector (three runs); the **menu detector**, glyph diff --git a/tools/re-capture/tbm_submenu_v2.py b/tools/re-capture/tbm_submenu_v2.py new file mode 100755 index 00000000..bcb5b03f --- /dev/null +++ b/tools/re-capture/tbm_submenu_v2.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Reach a `.tbm`-bearing submenu — with delivery confirmed and change detected. + +Third attempt. `tbm-submenu-not-reached.md` records why the first two failed, and +all three fixes here come from that page: + +1. **Hold Ⓐ longer.** A 0.12 s press issued while the guest is loading a screen is + missed outright — the guest is not polling. 0.5 s. +2. **Confirm delivery from the log, not from the pad.** The pad driver reports what + *it* emitted; only `[RE-INPUT] … -> user=0 vk=5800` says the guest received one. + "The press did nothing" and "there was no press" are identical on screen. +3. **Detect the screen change; never time it.** Timing failed twice in one + iteration. A submenu load may pass through a `pgloading_*` screen, so a fixed + wait cannot work — "different from the menu, and then stable" is the signal. + + tbm_submenu_v2.py LOG OUTDIR [wait_s] +""" +import os +import re +import subprocess +import sys +import time + +import numpy as np +from PIL import Image + +LOG, OUT = sys.argv[1], sys.argv[2] +WAIT = float(sys.argv[3]) if len(sys.argv) > 3 else 480 +W, H = 1280, 720 +NEED, CEIL, HOLD = 500, 2500, 12 # the title's plate pulse +MENU_LO, MENU_HI, MENU_HOLD = 250, 420, 6 # the main menu counts 327 +PAD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pad.py") +DELIV = re.compile(rb"RE-INPUT\] XamInputGetKeystrokeEx -> user=\d+ vk=5800 flags=0001") + + +def deliveries(): + try: + return len(DELIV.findall(open(LOG, "rb").read())) + except FileNotFoundError: + return 0 + + +def press_confirmed(tries=5, hold="0.5"): + """Press until the GUEST reports a keydown, not until the pad says it sent one.""" + for k in range(tries): + before = deliveries() + subprocess.run([sys.executable, PAD, "tap", "A", hold], check=False) + for _ in range(20): + time.sleep(0.25) + if deliveries() > before: + print(f"[{time.time()-T0:7.1f}s] A delivered (attempt {k+1})", flush=True) + return True + print(f"[{time.time()-T0:7.1f}s] A NOT delivered (attempt {k+1}) — retrying", flush=True) + print(f"[{time.time()-T0:7.1f}s] A never delivered after {tries} tries", flush=True) + return False + + +def _open(): + return subprocess.Popen( + ["ffmpeg", "-loglevel", "error", "-f", "x11grab", "-draw_mouse", "0", + "-video_size", f"{W}x{H}", "-i", ":98", "-r", "4", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + stdout=subprocess.PIPE, bufsize=W * H * 3 * 2) + + +def glyph(a): + r, g, b = a[:, :, 0], a[:, :, 1], a[:, :, 2] + return int(((g > 130) & (g - r > 45) & (g - b > 45)).sum()) + + +T0 = time.time() +p, n, seg = _open(), W * H * 3, time.time() +log = open(f"{OUT}/series.tsv", "w"); log.write("# t_s\tglyph\tmean\tphase\n") +phase, streak, base, stable = "wait", 0, None, 0 +while True: + el = time.time() - T0 + if el > WAIT: + print(f"TIMEOUT in phase {phase}", flush=True); break + if time.time() - seg > 30: + p.kill(); p = _open(); seg = time.time() + buf = p.stdout.read(n) + if len(buf) < n: + p.kill(); p = _open(); seg = time.time(); continue + a = np.frombuffer(buf, np.uint8).reshape(H, W, 3).astype(int) + c = glyph(a) + log.write(f"{el:.3f}\t{c}\t{a.mean():.3f}\t{phase}\n"); log.flush() + if phase == "wait": + streak = streak + 1 if NEED <= c <= CEIL else 0 + if streak >= HOLD: + print(f"[{el:7.1f}s] TITLE (glyph {c})", flush=True) + press_confirmed(); phase, streak = "tomenu", 0 + elif phase == "tomenu": + streak = streak + 1 if MENU_LO <= c <= MENU_HI else 0 + if streak >= MENU_HOLD: + Image.fromarray(a.astype(np.uint8)).save(f"{OUT}/menu.png") + base = a.astype(float) + print(f"[{el:7.1f}s] MENU (glyph {c}) — pressing into a submenu", flush=True) + time.sleep(2.0) # let the menu settle before pressing + press_confirmed(); phase, streak = "tosub", 0 + elif phase == "tosub": + diff = float((np.abs(a - base).max(axis=2) > 12).mean()) + if diff > 0.25: + stable += 1 + if stable >= 8: # changed, and holding still + Image.fromarray(a.astype(np.uint8)).save(f"{OUT}/submenu.png") + print(f"[{el:7.1f}s] SUBMENU: {100*diff:.1f}% of pixels differ " + f"from the menu, glyph {c}, mean {a.mean():.1f}", flush=True) + break + else: + stable = 0 +p.kill(); log.close()