#!/usr/bin/env python3 """Reach a `.tbm`-bearing submenu — with delivery confirmed and change detected. Third attempt. `tbm-submenu-not-reached.md` records why the first two failed, and all three fixes here come from that page: 1. **Hold Ⓐ longer.** A 0.12 s press issued while the guest is loading a screen is missed outright — the guest is not polling. 0.5 s. 2. **Confirm delivery from the log, not from the pad.** The pad driver reports what *it* emitted; only `[RE-INPUT] … -> user=0 vk=5800` says the guest received one. "The press did nothing" and "there was no press" are identical on screen. 3. **Detect the screen change; never time it.** Timing failed twice in one iteration. A submenu load may pass through a `pgloading_*` screen, so a fixed wait cannot work — "different from the menu, and then stable" is the signal. tbm_submenu_v2.py LOG OUTDIR [wait_s] """ import os import re import subprocess import sys import time import numpy as np from PIL import Image LOG, OUT = sys.argv[1], sys.argv[2] WAIT = float(sys.argv[3]) if len(sys.argv) > 3 else 480 W, H = 1280, 720 NEED, CEIL, HOLD = 500, 2500, 12 # the title's plate pulse MENU_LO, MENU_HI, MENU_HOLD = 250, 420, 6 # the main menu counts 327 PAD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pad.py") DELIV = re.compile(rb"RE-INPUT\] XamInputGetKeystrokeEx -> user=\d+ vk=5800 flags=0001") def deliveries(): try: return len(DELIV.findall(open(LOG, "rb").read())) except FileNotFoundError: return 0 def press_confirmed(tries=5, hold="0.5"): """Press until the GUEST reports a keydown, not until the pad says it sent one.""" for k in range(tries): before = deliveries() subprocess.run([sys.executable, PAD, "tap", "A", hold], check=False) for _ in range(20): time.sleep(0.25) if deliveries() > before: print(f"[{time.time()-T0:7.1f}s] A delivered (attempt {k+1})", flush=True) return True print(f"[{time.time()-T0:7.1f}s] A NOT delivered (attempt {k+1}) — retrying", flush=True) print(f"[{time.time()-T0:7.1f}s] A never delivered after {tries} tries", flush=True) return False def _open(): return subprocess.Popen( ["ffmpeg", "-loglevel", "error", "-f", "x11grab", "-draw_mouse", "0", "-video_size", f"{W}x{H}", "-i", ":98", "-r", "4", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], stdout=subprocess.PIPE, bufsize=W * H * 3 * 2) def glyph(a): r, g, b = a[:, :, 0], a[:, :, 1], a[:, :, 2] return int(((g > 130) & (g - r > 45) & (g - b > 45)).sum()) T0 = time.time() p, n, seg = _open(), W * H * 3, time.time() log = open(f"{OUT}/series.tsv", "w"); log.write("# t_s\tglyph\tmean\tphase\n") phase, streak, base, stable = "wait", 0, None, 0 while True: el = time.time() - T0 if el > WAIT: print(f"TIMEOUT in phase {phase}", flush=True); break if time.time() - seg > 30: p.kill(); p = _open(); seg = time.time() buf = p.stdout.read(n) if len(buf) < n: p.kill(); p = _open(); seg = time.time(); continue a = np.frombuffer(buf, np.uint8).reshape(H, W, 3).astype(int) c = glyph(a) log.write(f"{el:.3f}\t{c}\t{a.mean():.3f}\t{phase}\n"); log.flush() if phase == "wait": streak = streak + 1 if NEED <= c <= CEIL else 0 if streak >= HOLD: print(f"[{el:7.1f}s] TITLE (glyph {c})", flush=True) press_confirmed(); phase, streak = "tomenu", 0 elif phase == "tomenu": streak = streak + 1 if MENU_LO <= c <= MENU_HI else 0 if streak >= MENU_HOLD: Image.fromarray(a.astype(np.uint8)).save(f"{OUT}/menu.png") base = a.astype(float) print(f"[{el:7.1f}s] MENU (glyph {c}) — pressing into a submenu", flush=True) time.sleep(2.0) # let the menu settle before pressing press_confirmed(); phase, streak = "tosub", 0 elif phase == "tosub": diff = float((np.abs(a - base).max(axis=2) > 12).mean()) if diff > 0.25: stable += 1 if stable >= 8: # changed, and holding still Image.fromarray(a.astype(np.uint8)).save(f"{OUT}/submenu.png") print(f"[{el:7.1f}s] SUBMENU: {100*diff:.1f}% of pixels differ " f"from the menu, glyph {c}, mean {a.mean():.1f}", flush=True) break else: stable = 0 p.kill(); log.close()