Sylpheed/tools/port/verify-menu-audio

#!/usr/bin/env bash
# Does the port actually MAKE SOUND on the P5 walk, and the RIGHT sound?
#
#   tools/port/verify-menu-audio            # assert
#   tools/port/verify-menu-audio --control  # can it fail?
#
# 🔴 FOR WEEKS THIS COULD NOT FAIL. It computed the verdict, printed a red line
# when a cue was silent -- and the python had NO EXIT PATH, so it returned 0
# every time while `check-all` registered it `must-pass`. A cue could stop
# sounding and the suite would print the failure and stay green.
#
# That is this project's recurring defect one level up: not an instrument that
# sits below the thing under test, but an instrument that SEES the failure and
# does not report it. Ask of any check: what would this still report if the
# feature were absent -- AND what would it EXIT?
#
# This is the P6 gate check. P6's gate is "sound on the P5 gate", and until this
# existed the only evidence for it was that `audio.play("move")` appears in
# boot.gd -- which is evidence that a call is written, not that a sound reaches
# the Master bus. Those differ: the black hold was implemented, called, and
# emitted nothing for five milestones.
#
# It needs NO SOUND CARD. Godot records the Master bus to a WAV under the Dummy
# driver (docs/port/AUDIO-VERIFICATION.md section 2).
#
# WHAT IT CONCLUDES, and what it must not be read as:
#
#  * ✅ that a cue REACHES THE BUS when a press does something;
#  * ✅ that a press bound to NOTHING is silent, byte for byte;
#  * ✅ that two presses of the same action play the SAME cue;
#  * 🔴 NOT that the cue is the one the GAME plays. That binding is HANDOFF Q8,
#    measured by the Decoder, and nothing here re-measures it. This tool cannot
#    tell a correct cue from a confidently wrong one.
#
# ⚠️ Cue LENGTH is deliberately not asserted. The audible part of a cue is much
# shorter than its wave -- the music bed masks the tail -- so "elevated for
# 0.13 s" is a fact about the bed, not about the cue, and an assertion built on
# it would fail whenever the bed changes.
set -euo pipefail
cd "${PROJECT_DIR:-/work}"
export DISPLAY="${DISPLAY:-:97}"
OUT="${OUT:-${TMPDIR:-/tmp}/verify-menu-audio}"
CONTROL=0; [ "${1:-}" = "--control" ] && CONTROL=1
mkdir -p "$OUT"
[ -d port/.godot ] || godot --headless --path port --import >/dev/null 2>&1

run() { # name, script
  timeout 300 godot --path port --resolution 1280x720 -- \
    --menu=main_menu "--script=$2" "--audio=$OUT/$1.wav" >"$OUT/$1.log" 2>&1 || true
  [ -s "$OUT/$1.wav" ] || { echo "no audio written for $1 -- see $OUT/$1.log" >&2; exit 2; }
}

# THE WALK, and TWO CONTROLS. The controls are the point: a run that makes noise
# proves nothing on its own, because the music bed makes noise too.
#
#  `wait`  -- the bed alone, nothing pressed.
#  `left`  -- five presses that REACH _unhandled_input and are bound to nothing
#             (HANDOFF Q5: left/right do nothing). If these differ from `wait`,
#             the port is making a sound the game does not.
run walk down,down,accept,cancel,up
run ctrl wait,wait,wait,wait,wait
run noop left,left,left,left,left

# 🔴 AND A PER-CUE KNOWN NEGATIVE, because the bed-only control could not settle
# what it was being asked. `move` reported NOT FOUND on three consecutive runs at
# margins 0.109/0.120/0.131 against a 0.15 line that a documented earlier run had
# cleared at 0.185. Two readings fit that -- the cue stopped playing, or the
# threshold sits above the quietest cue's true signal -- and A MARGIN CANNOT
# SEPARATE THEM, because both produce a small number.
#
# So each cue now gets its own negative: the SAME walk, with only that cue's .ogg
# replaced by silence through the mod tree. Silencing a cue that is playing must
# collapse its correlation and leave the other two alone, which is a 3x3 matrix
# with six off-diagonal controls rather than one number to compare against a
# threshold.
for c in move confirm back; do
  d="$OUT/sup_$c"; mkdir -p "$d/audio/se"
  ffmpeg -v error -f lavfi -i anullsrc=r=44100:cl=stereo \
    -t "$(ffprobe -v error -show_entries format=duration -of csv=p=0 export/audio/se/$c.ogg)" \
    -c:a libvorbis "$d/audio/se/$c.ogg" -y
  SYLPHEED_MODS="$d" run "sup_$c" down,down,accept,cancel,up
  grep -q "^mod: audio/se/$c.ogg" "$OUT/sup_$c.log" || {
    echo "the $c override was never read -- the matrix below would be meaningless" >&2
    exit 2; }
done

# THE CONTROL. Replace the walk with the run that already had `move` silenced, so
# the cue is genuinely missing from the baseline. Silencing it again can then
# remove nothing, the diagonal cannot drop, and the check MUST fail. Built from
# the tool's OWN suppression machinery rather than a second mechanism -- a
# control built a different way tests the control, not the check.
if [ $CONTROL -eq 1 ]; then
  cp "$OUT/sup_move.wav" "$OUT/walk.wav"
  echo "control: analysing a walk in which \`move\` never sounded"
fi

rc=0
python3 - "$OUT" <<'PYEOF' || rc=$?
import array, math, subprocess, sys
O = sys.argv[1]; SR = 44100
def dec(src, dst):
    subprocess.run(["ffmpeg","-v","error","-i",src,"-f","s16le","-ac","1",
                    "-ar",str(SR),dst,"-y"], check=True)
    a = array.array('h'); a.frombytes(open(dst,'rb').read()); return a
walk = dec(f"{O}/walk.wav", f"{O}/walk.raw")
ctrl = dec(f"{O}/ctrl.wav", f"{O}/ctrl.raw")
noop = dec(f"{O}/noop.wav", f"{O}/noop.raw")

# 1. A press bound to nothing must be SILENT, and silent still means IDENTICAL --
#    but aligned to a WHOLE AUDIO BUFFER, because the recording is not
#    sample-deterministic across runs and never was.
#
#    🔴 This check compared the two byte streams directly and passed for weeks.
#    It then began failing, and the cause is not the port: three IDENTICAL
#    invocations produce two distinct outcomes, 1.207438 s and 1.300317 s,
#    differing by 0.092879 s = **exactly 4096 samples**, one mixing buffer. The
#    recording quantises to whole buffers and a one-buffer shift moves both the
#    length and the alignment of everything inside it.
#
#    So the old premise -- cross-run bit-determinism -- was never guaranteed. It
#    held while the run's timing sat away from a buffer boundary, and a larger
#    export (three voice streams instead of one) moved it onto one. A test that
#    passes by luck reports the luck running out as a regression in the code.
#
#    The fix keeps the strength that mattered: still EXACT equality, still no
#    threshold to tune. It only allows the comparison to slide by whole buffers,
#    which is the one degree of freedom the recorder actually has.
BUF = 4096
best = None
for k in (0, BUF, -BUF, 2*BUF, -2*BUF):
    a, b = (ctrl[k:], noop) if k >= 0 else (ctrl, noop[-k:])
    n = min(len(a), len(b))
    if n < BUF:
        continue
    if a[:n].tobytes() == b[:n].tobytes():
        best = (k, n)
        break
if best:
    print("no-op presses vs bed alone : IDENTICAL -- silent (%d samples, %+d buffer shift)"
          % (best[1], best[0] // BUF))
else:
    n = min(len(ctrl), len(noop))
    print("no-op presses vs bed alone : DIFFER at every whole-buffer alignment "
          "-- the port sounds a dead press (%d samples)" % n)

# 2. Is the RIGHT CUE on the bus? Match each EXPORTED cue wave against the
#    recording by normalised cross-correlation over the whole file.
#
#    This replaced a burst-counter that thresholded the envelope at a multiple
#    of the bed level. That counter reported 4 cues on one run and 0 on the next
#    from the SAME script, because its answer was set by two hand-picked
#    constants -- the multiple and a minimum run length -- and the bed level is
#    not constant across a run. It was nearly shipped. A tool whose headline
#    number moves with its own tuning cannot detect anything.
#
#    This has no such constant. The cue file is its own template, the search is
#    over the whole recording, and the verdict is a MARGIN over the same
#    template matched against the bed-only control.
def slide(tpl, hay, step=16):
    t = [float(v) for v in tpl]; bt = math.sqrt(sum(v*v for v in t))
    if bt == 0: return (0.0, 0.0)
    best = (-2.0, 0.0)
    for i in range(0, len(hay)-len(t), step):
        seg = hay[i:i+len(t)]
        bs = math.sqrt(sum(float(v)*v for v in seg))
        if bs:
            r = sum(a*float(b) for a, b in zip(t, seg))/(bt*bs)
            if r > best[0]: best = (r, i/SR)
    return best

found = []
tpls = {}
for cue in ("move", "confirm", "back"):
    tpl = dec("export/audio/se/%s.ogg" % cue, "%s/%s.raw" % (O, cue))[:int(0.15*SR)]
    tpls[cue] = tpl
    rw, tw = slide(tpl, walk)
    rc, _  = slide(tpl, ctrl)
    # 🔴 NO VERDICT ON THIS LINE ANY MORE. It used to print PRESENT/NOT FOUND on
    # `margin > 0.15`, and it called `move` NOT FOUND on three consecutive runs at
    # 0.109/0.120/0.131 while the cue was DEMONSTRABLY SOUNDING -- silencing its
    # .ogg collapses it to the bed floor. The bed-only control is a DIFFERENT RUN,
    # so its margin carries every difference between two runs; the threshold that
    # once cleared 0.185 was never a property of the cue. The number is still worth
    # printing. The verdict now comes from the suppression matrix below.
    hit = rw - rc > 0.15
    found.append((cue, tw, hit))
    print("%-8s walk r=%.3f at %5.2fs | bed-only r=%.3f | margin %+.3f"
          % (cue, rw, tw, rc, rw-rc))

# 3. The ORDER is the strongest evidence here and it is free: the correlator is
#    never told where to look, so three templates landing in script order --
#    move (step 1) before confirm (step 3) before back (step 4) -- is three
#    independent searches agreeing with the log.
# 3b. THE SUPPRESSION MATRIX. Row = the cue silenced, column = the template
#     searched for. The diagonal is the only cell that should move.
sup = {c: dec("%s/sup_%s.wav" % (O, c), "%s/sup_%s.raw" % (O, c))
       for c in ("move", "confirm", "back")}
base = {c: slide(tpls[c], walk)[0] for c in tpls}
print("\nsuppression matrix -- drop in r when one cue's .ogg is silenced")
print("            " + "".join("%9s" % c for c in ("move", "confirm", "back")))
ok = True
for row in ("move", "confirm", "back"):
    drops = {col: base[col] - slide(tpls[col], sup[row])[0] for col in ("move", "confirm", "back")}
    print("  silence %-6s" % row + "".join("%+9.3f" % drops[c] for c in ("move", "confirm", "back")))
    if drops[row] <= 0.05:
        ok = False
        print("      🔴 silencing %s did not remove %s -- that cue is NOT SOUNDING" % (row, row))
print("  => %s" % ("all three cues SOUND: silencing each one collapses its own signal"
                   if ok else "at least one cue is not sounding"))
# 🔴 THE VERDICT EXITS. Everything below this line is REPORTED, not asserted, and
# deliberately so: the no-op-silence line and the cue-order line both carry
# DOCUMENTED cross-run instability (whole-buffer recording shifts; a 0.15 margin
# this file's own comments show going to 0.109 on a sounding cue). Making either
# binding would produce red on correct audio, which is how a suite gets ignored.
# The diagonal has no threshold to drift: silencing a cue either removes its own
# signal or it was never there.
VERDICT_FAILED = not ok
# 🔴 THE VERDICT IS THE DIAGONAL ONLY, and the first version of this asserted the
# off-diagonal too -- "silencing a cue must not move the others". That failed, and
# the material is why: `confirm` lands at 1.12 s and `back` at 1.21 s, 0.09 s apart
# under a 0.15 s template, so the two windows OVERLAP. Silencing `confirm` raises
# `back` by 0.468 because confirm was masking it. That is a fact about two cues the
# game plays 90 ms apart, not a fault, and an assertion that calls it one would
# fail forever on correct audio.
print("     (off-diagonal is MASKING between overlapping cues, not an error --")
print("      confirm at 1.12 s and back at 1.21 s share a 0.15 s window)")

times = [t for _, t, hit in found if hit]
print("cue order vs script order  : %s"
      % ("CONSISTENT" if times == sorted(times) and len(times) == 3
         else "check %s" % [(c, round(t, 2)) for c, t, _ in found]))
raise SystemExit(1 if VERDICT_FAILED else 0)
PYEOF

if [ $CONTROL -eq 1 ]; then
  if [ $rc -eq 0 ]; then
    echo
    echo "  🔴 CONTROL FAILED -- the check passed a walk with \`move\` silenced, so it"
    echo "     cannot detect a cue that stops sounding."
    exit 1
  fi
  echo
  echo "the check rejects a run with a cue missing (rc=$rc)"
  exit 0
fi
exit $rc