diff --git a/tools/re-capture/freeze_watch.sh b/tools/re-capture/freeze_watch.sh new file mode 100755 index 00000000..faf527b4 --- /dev/null +++ b/tools/re-capture/freeze_watch.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Wait for the in-mission freeze and capture the evidence at the moment it lands. +# +# Written because catching it by hand costs a tool call every 25 s and the freeze +# takes anywhere from ten seconds to never: roughly half the runs that reach +# flight freeze, and the other half do not +# (docs/re/mission-freeze-resume-spin.md). +# +# "Frozen" is two screenshots byte-identical (frozen.py); it is checked ONLY +# while the flight HUD is still on screen, because the GAME OVER screen animates +# and a run that ended there is not a freeze. +# +# On detection it snapshots what the stuck-wait probe has said, so the comparison +# against the healthy-run baseline (one thread polling one Event) is made from +# the same instant rather than reconstructed later. +set -u +export HOME=/sylph-home/re +DISP="${DISPLAY:-:98}" +SD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOG="${1:-/sylph-home/re/shots/probe-canary.stdout}" +OUT="${2:-/sylph-home/re/logs/freeze-evidence.txt}" +DEADLINE=$(( SECONDS + ${3:-1800} )) +: > "$OUT" + +while [ $SECONDS -lt $DEADLINE ]; do + if ! pgrep -x xenia_canary >/dev/null; then + echo "EMULATOR GONE at ${SECONDS}s" | tee -a "$OUT"; exit 4 + fi + if python3 "$SD/frozen.py" 5 >/dev/null; then + if python3 -c "import sys; sys.path.insert(0,'$SD'); import frozen; sys.exit(0 if frozen.in_flight() else 1)"; then + { + echo "FROZEN IN FLIGHT at ${SECONDS}s" + echo + echo "=== stuck-wait probe: (thread, object) pairs ===" + grep -oE 'thread [0-9A-F]+ has timed out [0-9]+ times in a row on object [0-9A-F]+ \(type [0-9]+\)' "$LOG" \ + | sed -E 's/has timed out [0-9]+ times in a row //' | sort | uniq -c | sort -rn + echo + echo "=== last 12 probe lines ===" + grep 'wait-probe' "$LOG" | tail -12 + } | tee -a "$OUT" + exit 0 + fi + echo "frozen but NOT in flight (mission over) at ${SECONDS}s" | tee -a "$OUT" + exit 5 + fi + sleep 20 +done +echo "NO FREEZE within ${3:-1800}s" | tee -a "$OUT"; exit 1