Sylpheed/tools/port/verify-screen

#!/usr/bin/env bash
# Diff Godot's drawing of an exported screen against `sylpheed-cli screen
# render` of the same build.
#
# WHAT THIS IS, AND WHAT IT IS NOT.
#
# It is a CONSISTENCY check between two decoders that share their assumptions,
# and a REGRESSION detector: "did anything move since last commit". It is NOT a
# correctness check and agreement here is NOT evidence of correctness.
#
# `sylpheed-cli` is not the oracle. The oracle is the Xenia Canary capture and
# the game. Reborn is an explorer and extraction CLI for verifying decodes, and
# it can be wrong -- this corpus has been bitten three times by both renderers
# agreeing and both being wrong: pteff05 (the menu background, missing from
# both), scale-0, and rest(). Each time the capture caught it and neither
# renderer could have.
#
# So: a DIFFERS row means "we moved apart, go find out which of us moved". It
# does not mean the port is wrong. Where a capture and this tool disagree, the
# capture wins. Use `tools/verify-capture` for the correctness question.
#
#   tools/verify-screen                 # every screen in the manifest
#   tools/verify-screen main_menu title # named screens
#
# Writes <screen>.godot.png, <screen>.ref.png and <screen>.diff.png into
# $OUT (default: a directory under /tmp) and prints, per screen, the largest
# per-channel difference anywhere in the frame.
#
# The two renderers are held to the same inputs on purpose:
#
#  * the COMPARISON CLI is the one built by `build-reference-cli`, from the same
#    `sylpheed-formats` revision the exporter is pinned to. /reborn's own
#    target/ is a live mount of the other agent's checkout and moves mid-run; a
#    pixel disagreement against a moving decoder proves nothing.
#  * `--black` because Godot clears to black and the screen carries its own
#    background. The CLI's default dim slate stands in for a 3D scene behind an
#    in-mission screen, which is not this screen.
#  * `--primitives --animated` because those are what make the CLI draw the same
#    element set. `--focus` is NOT passed: nothing is focused at rest (HANDOFF
#    Q5 measured initial focus as unstable boot to boot, so choosing one is
#    P5's decision).
#  * `--pose=rest` on the Godot side. Since P2 the port's DEFAULT is to play the
#    timeline, and the settled timeline is deliberately NOT what `rest` says --
#    the export's `rest` misses `ptframe1`/`ptframe2` on the main menu, and the
#    running game shows them (docs/DECISIONS.md). Both renderers read `rest`
#    through the same decoder, so asking for it here keeps this a test of the
#    PORT against the reference. It is not the test of whether `rest` is right;
#    that one is the oracle capture, and the port already departs from it.
#
# A difference here is not automatically the port's fault, and it is not
# automatically a fault at all. Say which renderer moved and why -- do not tune
# until they match.
set -euo pipefail
cd "${PROJECT_DIR:-/work}"

# THE REFERENCE IS THE WORKSPACE'S OWN `sylpheed-cli`, and that is a change.
#
# It used to be a binary built per PINNED REVISION into `reference-cli/<rev>/`,
# because `sylpheed-formats` was a git dependency and /reborn's target/ was a
# live mount of the other agent's checkout that moved mid-run. A pixel
# disagreement against a moving decoder has a free variable in it.
#
# The monorepo merge (`65cefa7`) removed that problem by construction:
# `crates/sylpheed-export/Cargo.toml` now says
# `sylpheed-formats = { path = "../sylpheed-formats" }`, so the exporter, this
# reference and the port all read ONE decoder -- the working tree's.
#
# 🔴 It also silently broke the old machinery, and this script did not notice.
# `build-reference-cli` greps Cargo.toml for `Syplheed-Reborn.git", rev = "..."`;
# that line no longer exists, so the script exits 1 and the binary at
# `reference-cli/sylpheed-cli` is whatever was last built before the merge --
# here, three hours older than the sources and from a revision nothing points
# at any more. Running the diff against it would have compared the port to a
# decoder from another era and called the result a regression check. This
# corpus has already been bitten by a stale reference renderer three times.
#
# So: build it from the workspace. `SYLPHEED_CLI` still overrides, for anyone
# who does want to pin one deliberately.
CLI="${SYLPHEED_CLI:-}"
if [ -z "$CLI" ]; then
  CLI="${CARGO_TARGET_DIR:-/sylph-home/port/target-container}/release/sylpheed-cli"
  cargo build --release -p sylpheed-cli >/dev/null 2>&1 || true
fi
DISC="${SYLPHEED_DISC:-/disc}"
OUT="${OUT:-${TMPDIR:-/tmp}/verify-screen}"
export DISPLAY="${DISPLAY:-:97}"

[ -x "$CLI" ] || { echo "no reference CLI at $CLI -- \`cargo build --release -p sylpheed-cli\` failed?" >&2; exit 2; }
[ -f export/manifest.json ] || { echo "no export/manifest.json -- run build-export --run" >&2; exit 2; }
mkdir -p "$OUT"

# Godot needs one scan to register the `class_name` globals; without it every
# script fails to parse and the run dies with no frame drawn.
[ -d port/.godot ] || godot --headless --path port --import >/dev/null 2>&1

screens=("$@")
if [ ${#screens[@]} -eq 0 ]; then
  mapfile -t screens < <(python3 -c '
import json; print("\n".join(s["name"] for s in json.load(open("export/manifest.json"))["screens"]))')
fi

status=0
for name in "${screens[@]}"; do
  build=$(python3 -c '
import json,sys
m=json.load(open("export/manifest.json"))
f=next(s["file"] for s in m["screens"] if s["name"]==sys.argv[1])
print(json.load(open("export/"+f))["source"]["build"])' "$name")

  # `--all` because the exporter now addresses by PAK ENTRY INDEX, which is the
  # numbering `--all` uses; without it the CLI enumerates only the 12 bundles
  # `is_build` accepts and `--build 10` would land on entry 12. `--all` widens
  # the list, it does not change how any one bundle composites.
  "$CLI" screen render "$DISC/dat/GP_TITLE.pak" "$OUT/$name.ref.png" \
    --build "$build" --all --black --primitives --animated >/dev/null

  godot --path port --resolution 1280x720 -- \
    "--screen=$name" --pose=rest "--capture=$OUT/$name.godot.png" >"$OUT/$name.log" 2>&1

  convert "$OUT/$name.godot.png" "$OUT/$name.ref.png" \
    -compose difference -composite -colorspace Gray -auto-level "$OUT/$name.diff.png"
  read -r max mean <<<"$(convert "$OUT/$name.godot.png" "$OUT/$name.ref.png" \
    -compose difference -composite -format "%[fx:maxima*255] %[fx:mean*255]" info:)"

  # HOW MANY pixels are over the bar, not just how far the worst one is. A
  # single `max` cannot tell 2 pixels from 25 444, and this run produced both:
  # `main_menu` trips the threshold on TWO pixels out of 921 600 while
  # `title_jp` trips it on 2.8 % of the frame. Reporting only the max made those
  # the same verdict, which is how a real disagreement hides behind a rounding
  # one. The bar itself is NOT raised -- tuning a threshold until things match
  # is the failure this script's own header warns about.
  over=$(convert "$OUT/$name.godot.png" "$OUT/$name.ref.png" \
    -compose difference -composite -colorspace Gray -threshold $((3*65535/255)) \
    -format "%[fx:int(mean*w*h)]" info:)

  # BOTH FRAMES BLANK IS NOT AGREEMENT, AND THIS SCRIPT USED TO SAY IT WAS.
  #
  # `build_12` and `build_15` -- the two dressed loading screens -- render as
  # pure black in BOTH renderers, mean 0 and max 0, so the difference is 0 and
  # the row read `max 0  over3 0  OK`. Two of the sixteen rows in the committed
  # baseline were comparing nothing against nothing and reporting the strongest
  # verdict this script has.
  #
  # That is worse than a missing test: it is a test that reports a pass. The
  # screens are black because `pgloading_eff00` is a full-frame opaque black
  # quad whose `rest.t` (38) sits inside its own opening black hold, and
  # `--pose=rest` freezes it there -- see docs/port/DECISIONS.md. Whether that
  # is the port's bug or the decoders' reading of `rest` is open; what is not
  # open is that a blank pair may not be scored.
  #
  # So blankness is checked FIRST and reported as its own verdict. It is not a
  # failure -- the port may legitimately have nothing to draw -- but it is not a
  # pass either, and `status` is left alone so an unrelated screen's DIFFERS is
  # still what fails the run.
  ink=$(convert "$OUT/$name.godot.png" "$OUT/$name.ref.png" \
    -evaluate-sequence max -colorspace Gray -format "%[fx:maxima*255]" info:)
  verdict=OK
  if awk "BEGIN{exit !($ink <= 0)}"; then
    verdict="BLANK -- both renderers drew nothing; this row proves nothing"
  else
    # 🔴 THE VERDICT USES `over3`, NOT `max` ALONE, AND FOR YEARS IT DID NOT.
    #
    # This script computed `over3` precisely because "a single `max` cannot tell
    # 2 pixels from 25 444" -- its own words, a few lines up -- and then decided
    # the verdict on `max` regardless. So `main_menu` (max 4, over3 **0**) read
    # DIFFERS while `extras` (max 3, over3 0) read OK: one unit on one pixel,
    # separating two frames that are pixel-for-pixel equivalent at the bar.
    #
    # ⚠️ This is NOT raising the bar, which this file rightly warns against. The
    # bar is still 3. What changes is that a frame with NO pixel over it gets a
    # verdict of its own instead of being lumped in with a real disagreement --
    # the distinction the statistic was added to make and was never given.
    if awk "BEGIN{exit !($over > 0)}"; then
      verdict=DIFFERS; status=1
    elif awk "BEGIN{exit !($max > 3)}"; then
      verdict="ROUNDING -- max $max but NO pixel over the bar"
    fi
  fi
  printf '%-17s build %-3s max %-5s mean %-8s over3 %-7s %s\n' \
    "$name" "$build" "$max" "${mean:0:6}" "$over" "$verdict"
done
echo "artifacts in $OUT"
exit $status