Merge pull request 'fix(port): check-citations was red, and its selftest BROKEN, on a clean tree' (#66) from fix/citations-generated-tree into main

Reviewed-on: #66

docker/decoder/bin/build-canary

@@ -10,11 +10,22 @@
 # AVAILABLE MEMORY as well as core count — a full-parallel build of this tree
 # has OOM-killed the host outright.
 #
-#   build-canary [Release|Debug]  [extra cmake --build args]
+# `Checked` is the default, and should stay it: it is what the host builds and
+# what `run-canary` picks first, so the two agree by construction. Building
+# `Release` here instead leaves a second binary that `run-canary` will not use
+# -- hours of CPU for something nothing runs. `Checked` is optimised, with
+# assertions left in.
+#
+#   build-canary [Checked|Release|Debug]  [extra cmake --build args]
 set -euo pipefail
 
-CONFIG="${1:-Release}"; shift || true
-SRC="${PROJECT_DIR:-/work}/xenia-canary"
+CONFIG="${1:-Checked}"; shift || true
+# $XENIA_SRC FIRST. In this container the Canary source is bind-mounted at
+# `/canary`, and `$PROJECT_DIR/xenia-canary` does not exist -- so the old default
+# made this script unusable here, and the documented workaround was to symlink
+# `/canary` into the repository and remember to delete it again (CONTAINER-NOTES
+# §"The toolchain is real"). The launcher already exports the right path; read it.
+SRC="${XENIA_SRC:-${PROJECT_DIR:-/work}/xenia-canary}"
 BUILD="${XENIA_BUILD_DIR:-/sylph-home/re/canary-build}"
 JOBS="${SYLPH_JOBS:-2}"
 

docker/decoder/bin/run-canary

@@ -48,14 +48,25 @@ fi
 PROJECT_DIR="${PROJECT_DIR:-/work}"
 
 # ── Binary ───────────────────────────────────────────────────────────────────
+# `Checked` FIRST. All three configurations can be built, but `Checked` is the
+# one this project actually builds, so it is the one carrying our
+# instrumentation; the `Release/` and `Debug/` binaries beside it are months-old
+# leftovers that still run and still boot the game, which is exactly what makes
+# them dangerous -- a probe run against one reports zero hits and reads as a
+# finding about the game. Measured on this box 2026-09-21: `Checked` has
+# `audit_61_branch_probe_pcs`, `Release` (Aug 28) and `Debug` (Jul 19) do not.
 pick_bin() {
   [ -n "${XENIA_BIN:-}" ] && { echo "$XENIA_BIN"; return; }
-  for c in \
-      "${XENIA_BUILD_DIR:-/sylph-home/re/canary-build}/bin/Linux/Release/xenia_canary" \
-      "${XENIA_BUILD_DIR:-/sylph-home/re/canary-build}/bin/Linux/Debug/xenia_canary" \
-      "$PROJECT_DIR/xenia-canary/build/bin/Linux/Release/xenia_canary" \
-      "$PROJECT_DIR/xenia-canary/build/bin/Linux/Debug/xenia_canary"; do
-    [ -x "$c" ] && { echo "$c"; return; }
+  # Configuration is the OUTER loop, location the inner one: a `Checked` build
+  # anywhere beats a `Release` build anywhere. The other order picks a stale
+  # container `Release` over a fresh instrumented repo `Checked`, which is the
+  # exact mistake this is here to stop.
+  for cfg in Checked Release Debug; do
+    for c in \
+        "${XENIA_BUILD_DIR:-/sylph-home/re/canary-build}/bin/Linux/$cfg/xenia_canary" \
+        "$PROJECT_DIR/xenia-canary/build/bin/Linux/$cfg/xenia_canary"; do
+      [ -x "$c" ] && { echo "$c"; return; }
+    done
   done
 }
 BIN="$(pick_bin)"
@@ -64,6 +75,13 @@ if [ -z "${BIN:-}" ]; then
   exit 1
 fi
 
+# Say what is in the binary before the run, not after reading an empty log.
+# A missing probe is a build that predates it, never a quiet game.
+for sym in audit_61_branch_probe_pcs RE-DRAW; do
+  grep -aqm1 -- "$sym" "$BIN" \
+    || echo "run-canary: ⚠ $BIN has NO '$sym' -- it predates that instrumentation. Rebuild with: build-canary" >&2
+done
+
 # ── ISO ──────────────────────────────────────────────────────────────────────
 ISO="${SYLPH_ISO:-}"
 if [ -z "$ISO" ]; then

docker/decoder/sylph-decoder

@@ -123,7 +123,16 @@ docker_args() {
     # emulator and scrape /dev/shm to get at it. An earlier belief that this
     # file was STALE was tested and refuted -- it is current.
     -v "${SYLPH_PE:-$PROJECT/Project Sylpheed - Arc of Deception (USA, Europe) (En,Ja).pe}:/image/sylpheed.pe:ro"
-    -e "SYLPHEED_DB=/xenia-rs/sylpheed.db"
+    # 🔴 THE DISASSEMBLY DATABASE. This used to be read from the xenia-rs clone
+    # (`SYLPHEED_DB=/xenia-rs/sylpheed.db`). That repo is retired, its directory
+    # is gone, and the mount that served it was removed -- so the variable named
+    # a path that did not exist and `zq.py` had nothing to open. The database now
+    # lives beside the repo as a build artefact; mounted at the container's repo
+    # root, which is where `zq.py` looks when `$SYLPHEED_DB` is unset.
+    #
+    # Read-only on purpose: the host owns it, DuckDB takes an exclusive lock to
+    # write, and two agents plus the human sharing one file would corrupt it.
+    # Regenerating means writing elsewhere and pointing `$SYLPHEED_DB` at it.
     -e "SYLPHEED_PE=/image/sylpheed.pe"
     -e "SYLPHEED_IMAGE_BASE=0x82000000"
     # 🔴 EVIDENCE IS SHARED, NOT COMMITTED (issue #49). Each agent works in its
@@ -276,6 +285,20 @@ docker_args() {
     echo "==> NOTE: no ISO at $_iso -- Canary has nothing to boot." >&2
   fi
 
+  # The disassembly database. `zq.py` looks at the repo root when $SYLPHEED_DB
+  # is unset, so mounting it there needs no environment variable at all -- which
+  # is what the dead `/xenia-rs/sylpheed.db` variable taught: a path in an env
+  # var and a path in a mount drift apart silently.
+  _db="${SYLPH_DB:-$PROJECT/Sylpheed/sylpheed.db}"
+  if [ -f "$_db" ]; then
+    _out+=(-v "$_db:/work/sylpheed.db:ro")
+  else
+    echo "==> NOTE: no sylpheed.db at $_db -- /sylph-dis and zq.py cannot run." >&2
+    echo "    build one:  cargo run --release -p sylpheed-xexdb --bin sylph-xexdb -- \\" >&2
+    echo "                  dis \"\$SYLPH_ISO\" --db sylpheed.db --analyze sql --quiet" >&2
+    echo "    then re-apply the RE names with tools/apply_re_symbols.sql" >&2
+  fi
+
   # ── GPU ──
   # Three distinct cases, and conflating them is how you end up believing you
   # have hardware Vulkan while actually running llvmpipe:

docs/agents/CONTAINER-NOTES.md

@@ -12,13 +12,20 @@ already went wrong once.
 * **The toolchain is real.** `tools/re-capture/rebuild_canary.sh` exists because
   the old box had no cmake/ninja/clang and only runtime sonames, so it hand-
   relinked object files. **Do not use it here.** Use `build-canary`.
-  🔴 **But `build-canary` does not work in this container as it stands
+  ✅ **FIXED 2026-09-21 — `build-canary` now reads `$XENIA_SRC` first**, which the
+  launcher already sets to `/canary`, so there is no symlink to make and none to
+  forget. The two paragraphs below are kept because they are how the defect was
+  found and refuted, but **neither describes the script any more.**
+
+  🔴 ~~**`build-canary` does not work in this container as it stands
   (2026-08-29).** It builds `${PROJECT_DIR:-/work}/xenia-canary`, which **does
   not exist here** — the Canary source is at **`/canary`** (`$XENIA_SRC`). The
   warm 235 MB tree at `/sylph-home/re/canary-build` is configured with
   `CMAKE_HOME_DIRECTORY=/work/xenia-canary`, also missing, and its
   `build-Release.ninja` carries **no per-file rules** — it wants to re-run CMake
-  first, which would fail on the absent source root.
+  first, which would fail on the absent source root.~~ The warm tree is gone too:
+  every `sylph-*` volume was removed in the 2026-09-18 cleanup, so the next build
+  configures from scratch against `/canary` and caches the right source root.
 
   ✅ **The conclusion drawn from all that is REFUTED (2026-08-31).** The note went
   on to say *"any Canary change is a full reconfigure against `/canary` plus a full
@@ -38,9 +45,10 @@ already went wrong once.
   warm 235 MB tree is otherwise intact and the ninja re-run resolves its rules
   from the symlink.
 
-  ⚠️ **Remove the symlink when you are done.** `/work` is the repository, and
-  `xenia-canary` is not in `.gitignore`, so it shows up as untracked and can be
-  swept into a `git add -A`.
+  ⚠️ ~~**Remove the symlink when you are done.**~~ No longer needed — but if you
+  ever make one by hand, note that `/work` is the repository and `xenia-canary`
+  is not in `.gitignore`, so it shows up as untracked and can be swept into a
+  `git add -A`.
 * **numpy and Pillow are installed.** `entities2.py`, `flight_probe.py` and the
   image oracles work. Their absence used to look like a logic bug.
 

docs/agents/decoder-loop.md

@@ -56,11 +56,28 @@ rather than left in a document you might not reach.
 | path | what | env |
 |---|---|---|
 | `/image/sylpheed.pe` | the decompressed executable image | `SYLPHEED_PE` |
-| `/xenia-rs/sylpheed.db` | a disassembly database, 586 MB | `SYLPHEED_DB` |
+| `sylpheed.db` at the repo root | the disassembly database, 337 MB | — (`zq.py` finds it; `$SYLPHEED_DB` only to override) |
+| `/canary/build/bin/Linux/Checked/xenia_canary` | the built oracle, inside the `/canary` mount | — |
 | `/disc` | the extracted disc | `SYLPHEED_DISC` |
 | `/iso/game.iso` | the retail ISO Canary boots | `SYLPH_ISO` |
 | `/canary` | the Canary source, read-write | `XENIA_SRC` |
 
+⚠️ **The database is read-only and is NOT in git** — it is a build artefact, and
+`docs/re/captures/` is the same (issue #49). If either is missing the launcher
+says so and prints the command that builds it. Query the database with
+`python3 tools/zq.py …`, never by reading a disassembly dump.
+
+⚠️ **Take the oracle from `Checked/`, not `Release/`.** `Checked` is what this
+tree actually builds (optimised, with assertions); the `Release/` binary beside
+it is a stale August build with no `audit_61` probe in it. `strings` on the two
+is how that was found, and is how to check any binary before quoting a run from
+it.
+
+The `Checked` build carries all three instrumentations — `RE-INPUT`, `RE-DRAW`
+and the `audit_61` guest-PC branch probe (fork PR #2 brought the probe onto
+`sylpheed-re`; before it, the probe lived only on `phase-a-tracing`). Verified
+by `strings` on the binary, not assumed.
+
 **The `.pe` is a flat VA dump**: file offset = `VA - 0x82000000`. Reading
 `0x820A1630` is `seek(0xA1630)`. No XEX decrypt, no LZX, **no booted emulator** —
 dumping guest memory works but makes the whole static corpus depend on a running

docs/re/upstream-baseline.md

@@ -19,12 +19,13 @@ The instrumented branch stops at the Stage 02 briefing under a storm of
 behind `upstream/canary_experimental` (`a5a18f5c7`); our branch carries 50 of its
 own.
 
-* **`version.h` is never generated.** The build fails on
-  `trace_writer.cc:17: fatal error: 'version.h' file not found`. Upstream's
-  `xenia-build.py` writes it from git HEAD; the container's `build-canary`
-  wrapper does not invoke it, and our tree only builds because a stale copy from
-  an old `sylpheed-re` build sits in the build directory. Regenerated by hand in
-  exactly the format that script emits.
+* ~~**`version.h` is never generated.**~~ ✅ **FIXED — `CMakeLists.txt` now
+  generates it at configure time**, calling `xenia-build.py`'s
+  `generate_version_h()` and falling back to a stub if that fails, so a
+  CMake-direct build no longer depends on a stale copy in the build directory.
+  It used to fail on `trace_writer.cc:17: fatal error: 'version.h' file not
+  found`, and the fix had to exist before the 2026-09-18 cleanup deleted the
+  build volume that was carrying that stale copy.
 * **`build-canary` reports success on a failed build.** The harness recorded
   "completed (exit code 0)" while ninja had stopped with `1 error generated`.
   Only the missing binary gave it away.

tools/port/check-citations

@@ -36,7 +36,60 @@ CITE = re.compile(
     r"`?((?:docs|crates|port|tools|authored|export)/[\w./-]+"
     r"\.(?:md|rs|gd|json|txt|py|tsv|csv))`?"
 )
-PEER_REFS = ("origin/auto/frame-blend-draw-path", "origin/main")
+# ⚠️ `origin/auto/frame-blend-draw-path` used to head this list and NO LONGER
+# EXISTS -- the consolidation merged it and the branch was removed. The class is
+# kept because it is about the workflow, not about that one branch: the next
+# topic branch that lands a finding recreates the condition exactly. What must
+# not happen again is the selftest asserting the class against a fixture path
+# that has since become an ordinary local file, which is how it came to print
+# 🔴 BROKEN on a correct checkout. It now finds its own fixture, or says the
+# condition does not exist here.
+PEER_REFS = ("origin/main",)
+
+
+def a_peer_only_path() -> str | None:
+    """A path carried by a PEER_REF but absent from this working tree.
+
+    The selftest needs a REAL one: a hardcoded fixture silently stops testing
+    the moment that file lands locally, and then reports the scanner broken
+    instead of itself. On a clean, up-to-date checkout there is usually no such
+    path at all -- which is not a failure, it is the class being empty here.
+    """
+    for ref in PEER_REFS:
+        out = subprocess.run(["git", "ls-tree", "-r", "--name-only", ref],
+                             capture_output=True, text=True)
+        for f in out.stdout.splitlines():
+            if f.endswith((".md", ".rs", ".gd", ".json", ".txt", ".py",
+                           ".tsv", ".csv")) and not os.path.exists(f):
+                return f
+    return None
+
+
+def gitignored(path: str) -> bool:
+    """Is this path deliberately untracked? Pattern match -- existence not needed."""
+    return subprocess.run(["git", "check-ignore", "-q", path],
+                          capture_output=True).returncode == 0
+
+
+def unbuilt(path: str) -> bool:
+    """A citation of GENERATED output whose tree has not been built here.
+
+    🔴 THE THIRD ABSENCE, AND IT IS NOT AN ERROR. `export/` is the exporter's
+    output and is gitignored (`.gitignore` `/export*/`). A checkout where nobody
+    has run the exporter has no `export/` at all, so the four `DECISIONS.md` and
+    `BLOCKED.md` citations of `export/manifest.json` and `export/screens/...`
+    were counted as "resolve NOWHERE" and this check was red on a clean tree --
+    for a state no edit can fix, which is precisely the shape its own docstring
+    says it exists to avoid.
+
+    `check-capture-citations` already learned this for `docs/re/captures/`.
+    Same rule here: absent BECAUSE UNBUILT is reported; absent while the tree
+    IS built is a real broken citation and still fails.
+    """
+    if not gitignored(path):
+        return False
+    root = path.split("/")[0]
+    return not os.path.exists(root)
 
 
 def on_a_ref(path: str) -> str | None:
@@ -49,7 +102,7 @@ def on_a_ref(path: str) -> str | None:
 
 
 def scan(files):
-    resolves, peer, nowhere = 0, {}, {}
+    resolves, peer, nowhere, ungenerated = 0, {}, {}, {}
     for p in files:
         try:
             text = open(p, encoding="utf-8").read()
@@ -58,11 +111,13 @@ def scan(files):
         for m in sorted(set(CITE.findall(text))):
             if os.path.exists(m):
                 resolves += 1
+            elif unbuilt(m):
+                ungenerated.setdefault(m, p)
             elif (ref := on_a_ref(m)):
                 peer.setdefault(m, (p, ref))
             else:
                 nowhere.setdefault(m, p)
-    return resolves, peer, nowhere
+    return resolves, peer, nowhere, ungenerated
 
 
 def main() -> int:
@@ -82,28 +137,51 @@ def main() -> int:
         # exists) and does not resolve here (the reader still gets nothing), and
         # a scanner that collapsed it into either would make the flag meaningless
         # while still passing the two checks above.
+        peer_probe = a_peer_only_path()
         peerfile = os.path.join(tmp, "peer.md")
-        open(peerfile, "w").write("see `docs/re/f5-a-press-snaps-the-plate.md`\n")
-        rp, pp, np_ = scan([peerfile])
+        open(peerfile, "w").write("see `%s`\n" % (peer_probe or "docs/port/PORT-MISSION.md"))
+        # The FOURTH class: generated output whose tree is not built here.
+        # It has to be told apart from "resolves nowhere", which is the whole
+        # point -- a scanner that lumped them together is what made this check
+        # red on a clean checkout.
+        genfile = os.path.join(tmp, "gen.md")
+        open(genfile, "w").write("see `export/screens/title/main_menu.json`\n")
 
-        _, _, nb = scan([bad])
-        r, _, ng = scan([good])
+        rp, pp, np_, gp = scan([peerfile])
+
+        _, _, nb, _ = scan([bad])
+        r, _, ng, _ = scan([good])
+        rg, pg, ng2, gg = scan([genfile])
         caught   = len(nb) == 1
         passed   = len(ng) == 0 and r == 1
-        peer_ok  = len(pp) == 1 and rp == 0 and len(np_) == 0
-        ok = caught and passed and peer_ok
+        # None == the class is empty in this checkout, not that it is broken.
+        peer_ok = (len(pp) == 1 and rp == 0 and len(np_) == 0 and len(gp) == 0) \
+                  if peer_probe else None
+        # Only meaningful while `export/` is absent; if someone ran the exporter
+        # in this checkout the citation legitimately resolves instead.
+        gen_ok = (len(gg) == 1 and len(ng2) == 0 and len(pg) == 0) \
+                 if not os.path.exists("export") else (rg == 1)
+        ok = caught and passed and gen_ok and peer_ok is not False
         print("selftest: planted dangling caught=%s, real citation passed=%s, "
-              "peer-branch classed separately=%s -> %s"
-              % (caught, passed, peer_ok, "ok" if ok else "🔴 BROKEN"))
-        if not peer_ok:
+              "peer-branch classed separately=%s, unbuilt-generated classed "
+              "separately=%s -> %s"
+              % (caught, passed,
+                 "n/a (no peer-only path exists here)" if peer_ok is None
+                 else peer_ok,
+                 gen_ok, "ok" if ok else "🔴 BROKEN"))
+        if not gen_ok:
+            print("   🔴 a citation of unbuilt generated output must NOT be "
+                  "dangling; got resolves=%d peer=%d nowhere=%d ungenerated=%d"
+                  % (rg, len(pg), len(ng2), len(gg)))
+        if peer_ok is False:
             print("   🔴 --for-merge cannot mean anything if the peer class is "
                   "not distinguished; got resolves=%d peer=%d nowhere=%d"
                   % (rp, len(pp), len(np_)))
         return 0 if ok else 2
 
     files = sorted(glob.glob("docs/port/*.md"))
-    resolves, peer, nowhere = scan(files)
-    total = resolves + len(peer) + len(nowhere)
+    resolves, peer, nowhere, ungenerated = scan(files)
+    total = resolves + len(peer) + len(nowhere) + len(ungenerated)
     print("citations of repo paths in docs/port/*.md: %d" % total)
     print("  resolve here                : %d" % resolves)
     # 🔴 --for-merge TURNS THE PEER CLASS INTO A FAILURE.
@@ -130,6 +208,13 @@ def main() -> int:
         print("   After this merges they resolve NOWHERE -- the reader gets a dead")
         print("   path. Land the finding first and make it a dependency of this PR.")
         return 1
+    if ungenerated:
+        print("  not built in this checkout  : %d  (reported, not failed)"
+              % len(ungenerated))
+        for m, src in sorted(ungenerated.items()):
+            print("      %-52s <- %s" % (m, os.path.basename(src)))
+        print("      run the exporter and these resolve; a path still missing")
+        print("      afterwards IS dangling and fails below.")
     if nowhere:
         print("  🔴 resolve NOWHERE          : %d" % len(nowhere))
         for m, src in sorted(nowhere.items()):

tools/re/check-capture-citations

@@ -82,10 +82,16 @@ def committed() -> tuple[set[str], set[str]]:
             if fn == ".gitignore":
                 continue
             files.add(os.path.join(dirpath, fn).replace(os.sep, "/"))
+    # 🔴 START AT 3, NOT 4 — 3 is `docs/re/captures` ITSELF. Starting at 4 built
+    # every sub-directory and never the root, so the eight pages that cite the
+    # directory as a whole ("evidence lives in `docs/re/captures/`") each looked
+    # like a citation of a file that does not exist, and the gate was red on a
+    # clean, correct tree. The selftest could not see it either: it took an
+    # arbitrary member of `dirs`, which is always a sub-directory.
     dirs = set()
     for f in files:
         parts = f.split("/")
-        for i in range(4, len(parts)):          # docs/re/captures/<dir>/...
+        for i in range(3, len(parts)):          # docs/re/captures[/<dir>/...]
             dirs.add("/".join(parts[:i]) + "/")
     return files, dirs
 
@@ -182,6 +188,9 @@ def selftest() -> int:
         ("real file passed", real_file, True),
         ("directory reference passed", real_dir, True),
         ("directory without slash passed", real_dir.rstrip("/"), True),
+        # The root is the case the arbitrary `real_dir` above can never be.
+        ("captures root itself passed", ROOT, True),
+        ("captures root without slash passed", ROOT.rstrip("/"), True),
         ("sentence punctuation stripped", real_file + ".", True),
     ]
     # 🔴 THE SELFTEST USED TO PASS WHILE THE SCAN RETURNED NOTHING. It exercised