#!/usr/bin/env bash # Read WHICH object the guest threads are waiting on, HEALTHY vs FROZEN. # # Groundwork (mission-freeze-resume-spin.md): the waiting functions keep their # interesting argument in %rbx, and .eh_frame lets gdb restore callee-saved # registers during the unwind, so no DWARF and no rebuild are needed. # # TWO functions end up in these backtraces and %rbx does NOT mean the same # thing in each -- reading them as one set is what produced the earlier # "8 of 18 threads have an unreadable rbx" claim (waitobj-two-functions.md): # # XObject::Wait(this, ...) 8fbc90: mov %rdi,%rbx -> rbx = this # XObject::WaitMultiple(count, objects, .) 8fbfc0: mov %rsi,%rbx -> rbx = objects[] # mov %edi,%ebp -> ebp = count # # So a Wait frame is read with one deref and a WaitMultiple frame with two, and # every reading is checked by `info symbol`: a polymorphic object's first word # is a vtable, so a value that does not resolve to a `vtable for ...` symbol is # discarded rather than interpreted. # # Captures TWICE so the comparison is within-run: once while the mission is # healthy, once frozen. The frozen one is reached by WAITING FOR THE EVENT, not # by a clock -- measured onsets are 27/45/83/183/255s (median ~83), only about # half of runs freeze at all, and the "~270s" figure this script used to sleep # for was never a real bound. # # "Frozen" is frozen.py's test -- two byte-identical frames while the flight HUD # is still up. It is NOT `screen_id == flight` being false: a frozen mission # still classifies as `flight`, which is the whole reason frozen.py exists. # # Subcommands, because the boot and the wait do not fit one Bash call but the # emulator survives BETWEEN calls in a turn: # freeze_waitobj.sh boot [fly_s] boot, fly, capture `healthy`, leave running # freeze_waitobj.sh watch [secs] poll for the freeze, capture `frozen` # freeze_waitobj.sh run [fly_s] boot + healthy + watch, end to end # freeze_waitobj.sh repeat [n] [gap] N captures of a HEALTHY run, tags h1..hN # freeze_waitobj.sh stable [n] [gap] boot, then repeat # freeze_waitobj.sh dist [n] [gap] boot, N healthy captures, wait for the # freeze, N MORE captures, compare # # `repeat`/`stable` exist because a one-sample-per-state diff cannot tell a # freeze transition from ordinary variation: run 1's "T74/T75 move off a # semaphore" did not reproduce, because the HEALTHY state varies between # instants too. Sample the healthy run several times first, and only treat a # frozen difference as a signature if it is not something healthy play does # anyway. # # `run` exists because the whole experiment does not fit one Bash call and a # `timeout` kills the process group -- it took the emulator down once. Launch it # detached (`nohup ... &`) and poll the log instead. # # SYLPH_INDUCE=1 starts heavy_read.py the moment flight begins, to test whether # the freeze follows our instrument rather than the mission (see the n=1 entry in # mission-freeze-resume-spin.md). The control is the same script without it. set -u export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98 export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages export XENIA_BIN=/sylph-home/re/bin/gdb-wrap/xenia_canary SD="$(cd "$(dirname "$0")" && pwd)" FLY1="${1:-200}" # healthy checkpoint FLY2="${2:-140}" # extra seconds -> lands past the ~270s black-screen CMD=/tmp/gdb-cmd; OUT=/tmp/gdb-out.log sleepfor(){ python3 -c "import time,sys; time.sleep(float(sys.argv[1]))" "$1"; } capture(){ # capture local tag="$1" pid before pid=$(pgrep -x gdb | head -1) [ -n "$pid" ] || { echo "[$tag] no gdb"; return 2; } before=$(wc -c < "$OUT") screenshot "/tmp/fz-$tag.png" >/dev/null 2>&1 echo "[$tag] screen: $(python3 "$SD/screen_id.py" "/tmp/fz-$tag.png" 2>/dev/null | head -1)" kill -INT "$pid"; sleepfor 3 { echo 'echo === BT '"$tag"' ===\n'; echo 'thread apply all bt 6' echo 'echo === END BT ===\n'; } >> "$CMD" sleepfor 10 tail -c +$((before + 1)) "$OUT" > "/tmp/fz-bt-$tag.txt" # emit per-thread reads, with the frame index and the function taken from the # backtrace itself rather than assumed TAG="$tag" python3 - "/tmp/fz-bt-$tag.txt" <<'PY' >> "$CMD" import os,re,sys txt=open(sys.argv[1],errors='replace').read(); tag=os.environ['TAG'] cur=None; n=0 for line in txt.splitlines(): m=re.match(r'Thread (\d+) ',line) if m: cur=m.group(1); continue if not cur: continue f=re.search(r'#(\d+)\s+0x[0-9a-f]+ in xe::kernel::XObject::(WaitMultiple|Wait)\(',line) if not f: continue fr,kind=f.group(1),f.group(2) print(r'echo === %s T%s %s f%s ===\n'%(tag,cur,kind,fr)) print('thread %s'%cur); print('frame %s'%fr) if kind=='Wait': print('info registers rbx') print('info symbol *(unsigned long*)$rbx') else: print('info registers rbx rbp') for i in range(4): print('info symbol *(unsigned long*)*(unsigned long*)($rbx+%d)'%(8*i)) cur=None; n+=1 print(r'echo === END OBJ %s ===\n'%tag); print('continue') sys.stderr.write('[%s] %d wait frames\n'%(tag,n)) PY sleepfor 14 tail -c +$((before + 1)) "$OUT" > "/tmp/fz-obj-$tag.txt" local epid; epid=$(pgrep -x xenia_canary | head -1) [ -n "$epid" ] && cp "/proc/$epid/maps" "/tmp/fz-maps-$tag.txt" 2>/dev/null echo "[$tag] captured" } MODE="${1:-boot}" FLY="${2:-}" if [ "$MODE" = frozenrepeat ]; then N="${FLY:-6}"; GAP="${3:-25}" pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } for i in $(seq 1 "$N"); do capture "f$i" [ "$i" -lt "$N" ] && sleepfor "$GAP" done python3 "$SD/waitobj_report.py" --stability $(seq -f 'f%g' 1 "$N") python3 "$SD/waitobj_report.py" --dist "$N" echo "DIST DONE"; exit 0 fi if [ "$MODE" = repeat ]; then N="${FLY:-5}"; GAP="${3:-45}" pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } for i in $(seq 1 "$N"); do capture "h$i" [ "$i" -lt "$N" ] && sleepfor "$GAP" done python3 "$SD/waitobj_report.py" --stability $(seq -f 'h%g' 1 "$N") echo "STABILITY DONE"; exit 0 fi if [ "$MODE" = watchonly ]; then SECS="${FLY:-900}"; end=$((SECONDS + SECS)) while [ $SECONDS -lt $end ]; do pgrep -x xenia_canary >/dev/null || { echo "EMULATOR GONE"; exit 4; } if python3 "$SD/frozen.py" 5 >/dev/null 2>&1; then if python3 -c "import sys;sys.path.insert(0,'$SD');import frozen;sys.exit(0 if frozen.in_flight() else 1)"; then echo "FROZEN IN FLIGHT at ${SECONDS}s of this watch"; exit 0 fi echo "frozen but NOT in flight at ${SECONDS}s"; exit 5 fi sleepfor 12 done echo "NO FREEZE within ${SECS}s"; exit 1 fi if [ "$MODE" = watch ]; then SECS="${FLY:-500}" pgrep -x xenia_canary >/dev/null || { echo "NO EMULATOR"; exit 1; } end=$((SECONDS + SECS)) while [ $SECONDS -lt $end ]; do pgrep -x xenia_canary >/dev/null || { echo "EMULATOR GONE at ${SECONDS}s"; exit 4; } if python3 "$SD/frozen.py" 5 >/dev/null 2>&1; then if python3 -c "import sys;sys.path.insert(0,'$SD');import frozen;sys.exit(0 if frozen.in_flight() else 1)"; then echo "FROZEN IN FLIGHT at ${SECONDS}s of this watch" capture frozen python3 "$SD/waitobj_report.py" healthy frozen echo "FREEZE WAITOBJ DONE"; exit 0 fi echo "frozen but NOT in flight (mission over) at ${SECONDS}s"; exit 5 fi sleepfor 12 done echo "NO FREEZE within ${SECS}s (still animating)"; exit 1 fi # ---- boot / run mode ---- FLY="${FLY:-150}" "$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; } CFG=/tmp/nav-fz.json for t in 1 2 3; do python3 "$SD/pad.py" set "rt=1" >/dev/null 2>&1 || true; sleepfor 3 python3 "$SD/pad.py" clear >/dev/null 2>&1 || true if python3 "$SD/entities2.py" self 0x130 "$CFG" >/dev/null 2>&1; then SYLPH_HUNT=1 SYLPH_KEEPOUT=1400 nohup python3 "$SD/pilot.py" "$CFG" 3000 \ /tmp/fz-pilot.log 2>&1 & echo "--- pilot flying"; break fi done if [ "${SYLPH_INDUCE:-0}" = 1 ]; then nohup python3 "$SD/heavy_read.py" 4000 2 cpu /tmp/heavy.log 2>&1 & echo "--- INDUCER ON from flight start ($(date +%T))" else echo "--- inducer OFF (control run)" fi echo "--- flying ${FLY}s to the healthy control capture" sleepfor "$FLY"; capture healthy python3 "$SD/waitobj_report.py" healthy echo "BOOT PHASE DONE -- emulator left running" if [ "$MODE" = run ]; then echo "--- watching for the freeze ($(date +%T))" exec "$0" watch "${WATCH_S:-1500}" fi if [ "$MODE" = stable ]; then echo "--- sampling the HEALTHY run ($(date +%T))" exec "$0" repeat "${REPEAT_N:-6}" "${REPEAT_GAP:-45}" fi if [ "$MODE" = dist ]; then N="${REPEAT_N:-6}" echo "--- $N HEALTHY captures ($(date +%T))" "$0" repeat "$N" "${REPEAT_GAP:-30}" echo "--- inducing and watching for the freeze ($(date +%T))" nohup python3 "$SD/heavy_read.py" 4000 2 cpu /tmp/heavy.log 2>&1 & "$0" watchonly "${WATCH_S:-900}" || { echo "NO FREEZE -- dist half not collected"; exit 1; } echo "--- $N FROZEN captures ($(date +%T))" exec "$0" frozenrepeat "$N" "${REPEAT_GAP2:-25}" fi