Merge pull request 'docs(re): give docs/re/ the citation check docs/port/ has always had, and prune' (#33) from fix/capture-citations into main

Reviewed-on: #33

tools/re/check-capture-citations

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""Do the captures and the pages that cite them agree?
+
+    tools/re/check-capture-citations              # assert
+    tools/re/check-capture-citations --selftest   # can it fail?
+    tools/re/check-capture-citations --orphans    # list the unreferenced files
+
+`tools/port/check-citations` has checked `docs/port/*.md` since it was written.
+**`docs/re/` has never had one**, and `docs/re/captures/` is 118 MB — the
+largest thing in the repository, and the one place a file can be added, never
+cited, and never noticed.
+
+Two failures, which are opposites and must not be conflated:
+
+  * a page cites a capture that **is not committed** — a reader following it
+    gets nothing. That is an error, exactly as in `check-citations`.
+  * a capture that **no page cites** — not an error. It may be evidence a page
+    should have cited, and deleting on that basis would silently ratify the
+    omission. Reported, counted, never failed on.
+
+⚠️ THE NAIVE VERSION OF THIS CHECK REPORTS 10 DANGLING AND IS WRONG ABOUT 9.
+A first pass with a plain path regex claimed ten; the real number is one. Eight
+were **directory** references — `docs/re/captures/ui-layout/` — which resolve
+perfectly well and are absent only from `git ls-tree`, which lists files. One
+was a path at the end of a sentence, with the full stop pulled into the match.
+Both are handled below, and both are in the selftest, because a checker that
+cries wolf nine times out of ten is worse than none: it teaches you to ignore it.
+"""
+import os
+import re
+import subprocess
+import sys
+
+ROOT = "docs/re/captures/"
+# A capture path, optionally with the `docs/re/` prefix, optionally a directory.
+CITE = re.compile(r"(?:docs/re/)?captures/[A-Za-z0-9._/-]+")
+# ⚠️ `git grep -E` is POSIX ERE: it has no `(?:`, and a pattern carrying one
+# matches NOTHING while exiting 0. That silence read as "no page cites any
+# capture" — 257 orphans, 0 citations — which is the same shape as the export
+# table that returned empty and let the build succeed.
+CITE_ERE = r"(docs/re/)?captures/[A-Za-z0-9._/-]+"
+# Where a citation may live. Prose AND code: tests and tools open these files.
+SEARCH = ["docs", "crates", "tools", "port", "authored"]
+
+
+def git(*args: str) -> str:
+    return subprocess.run(["git", *args], capture_output=True, text=True).stdout
+
+
+def committed() -> tuple[set[str], set[str]]:
+    """Every capture in the WORKING TREE, and every directory they imply.
+
+    ⚠️ This read `git ls-tree HEAD` first, and that made the check useless for
+    the thing it is for: restoring a missing capture left it still reporting the
+    citation as dangling, because the fix was in the index and the check was
+    looking at the last commit. A pre-merge check must see the change being
+    proposed. `tools/port/check-citations` uses the working tree for exactly
+    this reason; so does this.
+    """
+    files = {l for l in git("ls-files", ROOT).splitlines() if l}
+    dirs = set()
+    for f in files:
+        parts = f.split("/")
+        for i in range(4, len(parts)):          # docs/re/captures/<dir>/...
+            dirs.add("/".join(parts[:i]) + "/")
+    return files, dirs
+
+
+def cited() -> set[str]:
+    raw = git("grep", "-rhoE", CITE_ERE, "--", *SEARCH).splitlines()
+    out = set()
+    for line in raw:
+        c = line.split(":")[-1]
+        # ⚠️ A path at the end of a sentence takes the punctuation with it.
+        c = c.rstrip(".,;:)`")
+        if not c.startswith("docs/"):
+            c = "docs/re/" + c
+        out.add(c)
+    return out
+
+
+def resolves(c: str, files: set[str], dirs: set[str]) -> bool:
+    if c in files:
+        return True
+    # A directory reference, written with or without its trailing slash.
+    return c.rstrip("/") + "/" in dirs
+
+
+def named_bare() -> set[str]:
+    """Basenames that appear anywhere outside the capture tree itself.
+
+    🔴 A CAPTURE CAN BE USED WITHOUT ITS PATH. `tools/re-capture/screen_match.py`
+    opens `movie-frame-attract-a.png` and `-b.png` by **filename**, building the
+    directory separately. A path-only scan calls both orphans, and deleting them
+    on that basis breaks the tool — which is what nearly happened. Anything
+    named at all counts as used.
+    """
+    out = set()
+    for line in git("grep", "-rhoE",
+                    r"[A-Za-z0-9._-]+\.(png|jpg|csv|log|txt|json|jsonl|bin|tsv)",
+                    "--", *SEARCH).splitlines():
+        out.add(line.split(":")[-1].strip())
+    return out
+
+
+def scan():
+    files, dirs = committed()
+    refs = cited()
+    bare = named_bare()
+    dangling = sorted(c for c in refs if not resolves(c, files, dirs))
+    # 🔴 A FILE INSIDE A CITED DIRECTORY IS CITED. A page that says "see
+    # `docs/re/captures/hud-runtime/`" is citing the whole directory, so every
+    # file in it is referenced and none is an orphan. Without this rule the
+    # first prune deleted the entire contents of two such directories and the
+    # check then went red on the very citations that made them evidence — it
+    # caught its own damage, which is the only reason this rule exists.
+    cited_dirs = {c.rstrip("/") + "/" for c in refs if resolves(c, files, dirs) and c not in files}
+    orphans = sorted(
+        f for f in files
+        if f not in refs
+        and os.path.basename(f) not in bare
+        and not any(f.startswith(d) for d in cited_dirs)
+    )
+    return files, refs, dangling, orphans
+
+
+def selftest() -> int:
+    """🔴 A CHECK THAT CANNOT FAIL IS NOT A CHECK — and this one has three ways
+    to be wrong, not one. It must catch a real dangling citation, and it must
+    NOT flag a directory reference or a path ending a sentence, which is how
+    the naive version produced nine false alarms."""
+    files, dirs = committed()
+    real_dir = next(iter(dirs), None)
+    real_file = next(iter(files), None)
+    if not real_dir or not real_file:
+        print("selftest: 🔴 no captures committed — nothing to test against")
+        return 2
+    cases = [
+        ("planted dangling caught", "docs/re/captures/no-such-file-anywhere.png", False),
+        ("real file passed", real_file, True),
+        ("directory reference passed", real_dir, True),
+        ("directory without slash passed", real_dir.rstrip("/"), True),
+        ("sentence punctuation stripped", real_file + ".", True),
+    ]
+    # 🔴 THE SELFTEST USED TO PASS WHILE THE SCAN RETURNED NOTHING. It exercised
+    # `resolves()` and never the gathering, so an ERE the grep could not compile
+    # produced "0 cited, 257 orphans" and five green ticks above it. A check
+    # whose selftest cannot see the failure that actually happened is decoration.
+    refs = cited()
+    gathering_ok = len(refs) > 50 and any(r.startswith(ROOT) for r in refs)
+    print("  %-34s %s  (%d citations found)"
+          % ("citation gathering works", "ok" if gathering_ok else "🔴 FAILED", len(refs)))
+
+    ok = gathering_ok
+    for name, path, want in cases:
+        got = resolves(path.rstrip(".,;:)`"), files, dirs)
+        mark = "ok" if got == want else "🔴 FAILED"
+        if got != want:
+            ok = False
+        print("  %-34s %s" % (name, mark))
+    print("selftest: %s" % ("ok" if ok else "🔴 BROKEN"))
+    return 0 if ok else 2
+
+
+def main() -> int:
+    if "--selftest" in sys.argv:
+        return selftest()
+    files, refs, dangling, orphans = scan()
+    if "--orphans" in sys.argv:
+        for f in orphans:
+            print(f)
+        return 0
+
+    print("captures committed          : %d" % len(files))
+    print("  cited by a page or a tool : %d" % (len(files) - len(orphans)))
+    print("  cited by nothing          : %d  (reported, not failed —" % len(orphans))
+    print("                               an orphan may be evidence a page owes)")
+    if dangling:
+        print("  🔴 cited but NOT committed: %d" % len(dangling))
+        for d in dangling:
+            print("      %s" % d)
+        print("\n🔴 a reader following those gets nothing. Commit the capture, fix the")
+        print("   path, or drop the citation.")
+        return 1
+    print("  🔴 cited but NOT committed: 0")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())