Sylpheed/tools/re/check-capture-citations

#!/usr/bin/env python3
"""Do the captures and the pages that cite them agree?

    tools/re/check-capture-citations              # assert
    tools/re/check-capture-citations --selftest   # can it fail?
    tools/re/check-capture-citations --orphans    # list the unreferenced files

`tools/port/check-citations` has checked `docs/port/*.md` since it was written.
**`docs/re/` has never had one**, and `docs/re/captures/` is 118 MB — the
largest thing in the repository, and the one place a file can be added, never
cited, and never noticed.

⚠️ 2026-09-19, issue #49: **game assets are no longer committed.** Screenshots
and savegame blobs live in the working tree and are gitignored, so the pages'
relative links still resolve on a machine that has them while nothing ships.
That inverts half of this check — "is it committed?" became "is it PRESENT?",
and a NEW failure appeared: an asset that IS tracked. Both are below.

Three failures, which are opposites and must not be conflated:

  * a page cites a capture that **is not present** — a reader following it
    gets nothing. That is an error, exactly as in `check-citations`.
  * a game asset that **is tracked by git** — issue #49 says it must not be.
  * a capture that **no page cites** — not an error. It may be evidence a page
    should have cited, and deleting on that basis would silently ratify the
    omission. Reported, counted, never failed on.

⚠️ THE NAIVE VERSION OF THIS CHECK REPORTS 10 DANGLING AND IS WRONG ABOUT 9.
A first pass with a plain path regex claimed ten; the real number is one. Eight
were **directory** references — `docs/re/captures/ui-layout/` — which resolve
perfectly well and are absent only from `git ls-tree`, which lists files. One
was a path at the end of a sentence, with the full stop pulled into the match.
Both are handled below, and both are in the selftest, because a checker that
cries wolf nine times out of ten is worse than none: it teaches you to ignore it.
"""
import os
import re
import subprocess
import sys

ROOT = "docs/re/captures/"
# A capture path, optionally with the `docs/re/` prefix, optionally a directory.
CITE = re.compile(r"(?:docs/re/)?captures/[A-Za-z0-9._/-]+")
# ⚠️ `git grep -E` is POSIX ERE: it has no `(?:`, and a pattern carrying one
# matches NOTHING while exiting 0. That silence read as "no page cites any
# capture" — 257 orphans, 0 citations — which is the same shape as the export
# table that returned empty and let the build succeed.
CITE_ERE = r"(docs/re/)?captures/[A-Za-z0-9._/-]+"
# Where a citation may live. Prose AND code: tests and tools open these files.
SEARCH = ["docs", "crates", "tools", "port", "authored"]
# 🔴 AND NOT THIS FILE. `SEARCH` includes `tools`, so the scan read *itself* --
# and `selftest()`'s planted fixture below is a literal capture path. The check
# therefore reported a dangling citation to a file that will never exist, and
# exited 1 on a clean checkout, for ever. A gate that is red before anyone
# changes anything teaches people to ignore it.
# Excluding only this one file keeps every other tool in scope; a real citation
# belongs in a page or a tool that opens the capture, never in the checker.
SELF = ":!tools/re/check-capture-citations"


def git(*args: str) -> str:
    return subprocess.run(["git", *args], capture_output=True, text=True).stdout


def committed() -> tuple[set[str], set[str]]:
    """Every capture in the WORKING TREE, and every directory they imply.

    ⚠️ This read `git ls-tree HEAD` first, and that made the check useless for
    the thing it is for: restoring a missing capture left it still reporting the
    citation as dangling, because the fix was in the index and the check was
    looking at the last commit. A pre-merge check must see the change being
    proposed. `tools/port/check-citations` uses the working tree for exactly
    this reason; so does this.
    """
    # 🔴 NOT `git ls-files`: since #49 the assets are deliberately untracked, so
    # the index no longer knows they exist. Presence is a question about the
    # working tree, and asking git would report every screenshot as missing and
    # fail on all 203 citations.
    files = set()
    for dirpath, _dirnames, filenames in os.walk(ROOT):
        for fn in filenames:
            if fn == ".gitignore":
                continue
            files.add(os.path.join(dirpath, fn).replace(os.sep, "/"))
    # 🔴 START AT 3, NOT 4 — 3 is `docs/re/captures` ITSELF. Starting at 4 built
    # every sub-directory and never the root, so the eight pages that cite the
    # directory as a whole ("evidence lives in `docs/re/captures/`") each looked
    # like a citation of a file that does not exist, and the gate was red on a
    # clean, correct tree. The selftest could not see it either: it took an
    # arbitrary member of `dirs`, which is always a sub-directory.
    dirs = set()
    for f in files:
        parts = f.split("/")
        for i in range(3, len(parts)):          # docs/re/captures[/<dir>/...]
            dirs.add("/".join(parts[:i]) + "/")
    return files, dirs


ASSET_SUFFIXES = (".png", ".jpg", ".jpeg", ".bin")


def tracked_assets() -> list[str]:
    """Game assets that are committed — forbidden since issue #49.

    The repo carries code, tooling and docs. A screenshot that sneaks back in
    is invisible in review (a binary shows as "Bin 0 -> 1234567 bytes") and is
    permanent once merged, because removing it later needs a history rewrite.
    So this is the half of the check that has to be loud.
    """
    out = []
    for l in git("ls-files").splitlines():
        if l.lower().endswith(ASSET_SUFFIXES):
            out.append(l)
    return out


def cited() -> set[str]:
    raw = git("grep", "-rhoE", CITE_ERE, "--", *SEARCH, SELF).splitlines()
    out = set()
    for line in raw:
        c = line.split(":")[-1]
        # ⚠️ A path at the end of a sentence takes the punctuation with it.
        c = c.rstrip(".,;:)`")
        if not c.startswith("docs/"):
            c = "docs/re/" + c
        out.add(c)
    return out


def resolves(c: str, files: set[str], dirs: set[str]) -> bool:
    if c in files:
        return True
    # A directory reference, written with or without its trailing slash.
    return c.rstrip("/") + "/" in dirs


def named_bare() -> set[str]:
    """Basenames that appear anywhere outside the capture tree itself.

    🔴 A CAPTURE CAN BE USED WITHOUT ITS PATH. `tools/re-capture/screen_match.py`
    opens `movie-frame-attract-a.png` and `-b.png` by **filename**, building the
    directory separately. A path-only scan calls both orphans, and deleting them
    on that basis breaks the tool — which is what nearly happened. Anything
    named at all counts as used.
    """
    out = set()
    for line in git("grep", "-rhoE",
                    r"[A-Za-z0-9._-]+\.(png|jpg|csv|log|txt|json|jsonl|bin|tsv)",
                    "--", *SEARCH, SELF).splitlines():
        out.add(line.split(":")[-1].strip())
    return out


def scan():
    files, dirs = committed()
    refs = cited()
    bare = named_bare()
    dangling = sorted(c for c in refs if not resolves(c, files, dirs))
    # 🔴 A FILE INSIDE A CITED DIRECTORY IS CITED. A page that says "see
    # `docs/re/captures/hud-runtime/`" is citing the whole directory, so every
    # file in it is referenced and none is an orphan. Without this rule the
    # first prune deleted the entire contents of two such directories and the
    # check then went red on the very citations that made them evidence — it
    # caught its own damage, which is the only reason this rule exists.
    cited_dirs = {c.rstrip("/") + "/" for c in refs if resolves(c, files, dirs) and c not in files}
    orphans = sorted(
        f for f in files
        if f not in refs
        and os.path.basename(f) not in bare
        and not any(f.startswith(d) for d in cited_dirs)
    )
    return files, refs, dangling, orphans


def selftest() -> int:
    """🔴 A CHECK THAT CANNOT FAIL IS NOT A CHECK — and this one has three ways
    to be wrong, not one. It must catch a real dangling citation, and it must
    NOT flag a directory reference or a path ending a sentence, which is how
    the naive version produced nine false alarms."""
    files, dirs = committed()
    real_dir = next(iter(dirs), None)
    real_file = next(iter(files), None)
    if not real_dir or not real_file:
        print("selftest: 🔴 no captures present — nothing to test against")
        return 2
    cases = [
        ("planted dangling caught", "docs/re/captures/no-such-file-anywhere.png", False),
        ("real file passed", real_file, True),
        ("directory reference passed", real_dir, True),
        ("directory without slash passed", real_dir.rstrip("/"), True),
        # The root is the case the arbitrary `real_dir` above can never be.
        ("captures root itself passed", ROOT, True),
        ("captures root without slash passed", ROOT.rstrip("/"), True),
        ("sentence punctuation stripped", real_file + ".", True),
    ]
    # 🔴 THE SELFTEST USED TO PASS WHILE THE SCAN RETURNED NOTHING. It exercised
    # `resolves()` and never the gathering, so an ERE the grep could not compile
    # produced "0 cited, 257 orphans" and five green ticks above it. A check
    # whose selftest cannot see the failure that actually happened is decoration.
    refs = cited()
    gathering_ok = len(refs) > 50 and any(r.startswith(ROOT) for r in refs)
    print("  %-34s %s  (%d citations found)"
          % ("citation gathering works", "ok" if gathering_ok else "🔴 FAILED", len(refs)))

    # 🔴 REGRESSION GUARD FOR `SELF`. The planted fixture above is a literal
    # capture path living in a file `SEARCH` covers. Before the exclusion
    # existed the scan counted it as a real citation, so the check reported a
    # dangling capture and exited 1 on a clean tree — permanently. If anyone
    # drops `SELF`, this goes red here instead of in everybody else's run.
    fixture_counted = cases[0][1] in refs
    print("  %-34s %s"
          % ("own fixtures not counted", "🔴 FAILED" if fixture_counted else "ok"))

    # 🔴 THE #49 RULE NEEDS ITS OWN TICK. Presence now comes from the working
    # tree, so a checkout with the captures present looks identical whether or
    # not they are tracked — only this assertion can tell the difference.
    stowaways = tracked_assets()
    print("  %-34s %s%s"
          % ("no game asset tracked", "ok" if not stowaways else "🔴 FAILED",
             "" if not stowaways else "  (%d tracked)" % len(stowaways)))

    ok = gathering_ok and not fixture_counted and not stowaways
    for name, path, want in cases:
        got = resolves(path.rstrip(".,;:)`"), files, dirs)
        mark = "ok" if got == want else "🔴 FAILED"
        if got != want:
            ok = False
        print("  %-34s %s" % (name, mark))
    print("selftest: %s" % ("ok" if ok else "🔴 BROKEN"))
    return 0 if ok else 2


def main() -> int:
    if "--selftest" in sys.argv:
        return selftest()
    files, refs, dangling, orphans = scan()
    if "--orphans" in sys.argv:
        for f in orphans:
            print(f)
        return 0

    stowaways = tracked_assets()
    print("captures present on disk    : %d" % len(files))
    print("  cited by a page or a tool : %d" % (len(files) - len(orphans)))
    print("  cited by nothing          : %d  (reported, not failed —" % len(orphans))
    print("                               an orphan may be evidence a page owes)")
    # 🔴 A BARE CLONE HAS NO CAPTURES AT ALL, and that is not a defect.
    # Since #49 the assets are never committed, so a fresh clone, a worktree or
    # a CI checkout legitimately has none — and the naive check calls all 134
    # citations dangling and exits 1. A gate that is red on every clean checkout
    # is one people learn to ignore, which is how the last wrong-by-default
    # check in this file cost a session. Distinguish "none here" from "this one
    # is missing": with some assets present, a gap is real and still fails.
    assets_here = sum(1 for f in files if f.lower().endswith(ASSET_SUFFIXES))
    if assets_here == 0 and dangling:
        print("  ⓘ no captures on this checkout: %d citations unresolved" % len(dangling))
        print("     Expected — captures are local-only (#49) and a fresh clone has none.")
        print("     Copy them in, or read the evidence on the issue it is attached to.")
        print("  🔴 game assets TRACKED    : %d" % len(stowaways))
        return 1 if stowaways else 0

    rc = 0
    if dangling:
        print("  🔴 cited but NOT present  : %d" % len(dangling))
        for d in dangling:
            print("      %s" % d)
        print("\n🔴 a reader following those gets nothing. Restore the capture, fix the")
        print("   path, or drop the citation.")
        rc = 1
    else:
        print("  🔴 cited but NOT present  : 0")

    if stowaways:
        print("  🔴 game assets TRACKED    : %d  (issue #49 — code, tooling, docs only)" % len(stowaways))
        for a in stowaways[:10]:
            print("      %s" % a)
        if len(stowaways) > 10:
            print("      … and %d more" % (len(stowaways) - 10))
        print("\n🔴 run `git rm --cached` on those; they stay on disk and stay ignored.")
        rc = 1
    else:
        print("  🔴 game assets TRACKED    : 0")
    return rc


if __name__ == "__main__":
    raise SystemExit(main())