Sylpheed/tools/re/check-capture-citations

#!/usr/bin/env python3
"""Do the captures and the pages that cite them agree?

    tools/re/check-capture-citations              # assert
    tools/re/check-capture-citations --selftest   # can it fail?
    tools/re/check-capture-citations --orphans    # list the unreferenced files

`tools/port/check-citations` has checked `docs/port/*.md` since it was written.
**`docs/re/` has never had one**, and `docs/re/captures/` is 118 MB — the
largest thing in the repository, and the one place a file can be added, never
cited, and never noticed.

Two failures, which are opposites and must not be conflated:

  * a page cites a capture that **is not committed** — a reader following it
    gets nothing. That is an error, exactly as in `check-citations`.
  * a capture that **no page cites** — not an error. It may be evidence a page
    should have cited, and deleting on that basis would silently ratify the
    omission. Reported, counted, never failed on.

⚠️ THE NAIVE VERSION OF THIS CHECK REPORTS 10 DANGLING AND IS WRONG ABOUT 9.
A first pass with a plain path regex claimed ten; the real number is one. Eight
were **directory** references — `docs/re/captures/ui-layout/` — which resolve
perfectly well and are absent only from `git ls-tree`, which lists files. One
was a path at the end of a sentence, with the full stop pulled into the match.
Both are handled below, and both are in the selftest, because a checker that
cries wolf nine times out of ten is worse than none: it teaches you to ignore it.
"""
import os
import re
import subprocess
import sys

ROOT = "docs/re/captures/"
# A capture path, optionally with the `docs/re/` prefix, optionally a directory.
CITE = re.compile(r"(?:docs/re/)?captures/[A-Za-z0-9._/-]+")
# ⚠️ `git grep -E` is POSIX ERE: it has no `(?:`, and a pattern carrying one
# matches NOTHING while exiting 0. That silence read as "no page cites any
# capture" — 257 orphans, 0 citations — which is the same shape as the export
# table that returned empty and let the build succeed.
CITE_ERE = r"(docs/re/)?captures/[A-Za-z0-9._/-]+"
# Where a citation may live. Prose AND code: tests and tools open these files.
SEARCH = ["docs", "crates", "tools", "port", "authored"]
# 🔴 AND NOT THIS FILE. `SEARCH` includes `tools`, so the scan read *itself* --
# and `selftest()`'s planted fixture below is a literal capture path. The check
# therefore reported a dangling citation to a file that will never exist, and
# exited 1 on a clean checkout, for ever. A gate that is red before anyone
# changes anything teaches people to ignore it.
# Excluding only this one file keeps every other tool in scope; a real citation
# belongs in a page or a tool that opens the capture, never in the checker.
SELF = ":!tools/re/check-capture-citations"


def git(*args: str) -> str:
    return subprocess.run(["git", *args], capture_output=True, text=True).stdout


def committed() -> tuple[set[str], set[str]]:
    """Every capture in the WORKING TREE, and every directory they imply.

    ⚠️ This read `git ls-tree HEAD` first, and that made the check useless for
    the thing it is for: restoring a missing capture left it still reporting the
    citation as dangling, because the fix was in the index and the check was
    looking at the last commit. A pre-merge check must see the change being
    proposed. `tools/port/check-citations` uses the working tree for exactly
    this reason; so does this.
    """
    files = {l for l in git("ls-files", ROOT).splitlines() if l}
    dirs = set()
    for f in files:
        parts = f.split("/")
        for i in range(4, len(parts)):          # docs/re/captures/<dir>/...
            dirs.add("/".join(parts[:i]) + "/")
    return files, dirs


def cited() -> set[str]:
    raw = git("grep", "-rhoE", CITE_ERE, "--", *SEARCH, SELF).splitlines()
    out = set()
    for line in raw:
        c = line.split(":")[-1]
        # ⚠️ A path at the end of a sentence takes the punctuation with it.
        c = c.rstrip(".,;:)`")
        if not c.startswith("docs/"):
            c = "docs/re/" + c
        out.add(c)
    return out


def resolves(c: str, files: set[str], dirs: set[str]) -> bool:
    if c in files:
        return True
    # A directory reference, written with or without its trailing slash.
    return c.rstrip("/") + "/" in dirs


def named_bare() -> set[str]:
    """Basenames that appear anywhere outside the capture tree itself.

    🔴 A CAPTURE CAN BE USED WITHOUT ITS PATH. `tools/re-capture/screen_match.py`
    opens `movie-frame-attract-a.png` and `-b.png` by **filename**, building the
    directory separately. A path-only scan calls both orphans, and deleting them
    on that basis breaks the tool — which is what nearly happened. Anything
    named at all counts as used.
    """
    out = set()
    for line in git("grep", "-rhoE",
                    r"[A-Za-z0-9._-]+\.(png|jpg|csv|log|txt|json|jsonl|bin|tsv)",
                    "--", *SEARCH, SELF).splitlines():
        out.add(line.split(":")[-1].strip())
    return out


def scan():
    files, dirs = committed()
    refs = cited()
    bare = named_bare()
    dangling = sorted(c for c in refs if not resolves(c, files, dirs))
    # 🔴 A FILE INSIDE A CITED DIRECTORY IS CITED. A page that says "see
    # `docs/re/captures/hud-runtime/`" is citing the whole directory, so every
    # file in it is referenced and none is an orphan. Without this rule the
    # first prune deleted the entire contents of two such directories and the
    # check then went red on the very citations that made them evidence — it
    # caught its own damage, which is the only reason this rule exists.
    cited_dirs = {c.rstrip("/") + "/" for c in refs if resolves(c, files, dirs) and c not in files}
    orphans = sorted(
        f for f in files
        if f not in refs
        and os.path.basename(f) not in bare
        and not any(f.startswith(d) for d in cited_dirs)
    )
    return files, refs, dangling, orphans


def selftest() -> int:
    """🔴 A CHECK THAT CANNOT FAIL IS NOT A CHECK — and this one has three ways
    to be wrong, not one. It must catch a real dangling citation, and it must
    NOT flag a directory reference or a path ending a sentence, which is how
    the naive version produced nine false alarms."""
    files, dirs = committed()
    real_dir = next(iter(dirs), None)
    real_file = next(iter(files), None)
    if not real_dir or not real_file:
        print("selftest: 🔴 no captures committed — nothing to test against")
        return 2
    cases = [
        ("planted dangling caught", "docs/re/captures/no-such-file-anywhere.png", False),
        ("real file passed", real_file, True),
        ("directory reference passed", real_dir, True),
        ("directory without slash passed", real_dir.rstrip("/"), True),
        ("sentence punctuation stripped", real_file + ".", True),
    ]
    # 🔴 THE SELFTEST USED TO PASS WHILE THE SCAN RETURNED NOTHING. It exercised
    # `resolves()` and never the gathering, so an ERE the grep could not compile
    # produced "0 cited, 257 orphans" and five green ticks above it. A check
    # whose selftest cannot see the failure that actually happened is decoration.
    refs = cited()
    gathering_ok = len(refs) > 50 and any(r.startswith(ROOT) for r in refs)
    print("  %-34s %s  (%d citations found)"
          % ("citation gathering works", "ok" if gathering_ok else "🔴 FAILED", len(refs)))

    # 🔴 REGRESSION GUARD FOR `SELF`. The planted fixture above is a literal
    # capture path living in a file `SEARCH` covers. Before the exclusion
    # existed the scan counted it as a real citation, so the check reported a
    # dangling capture and exited 1 on a clean tree — permanently. If anyone
    # drops `SELF`, this goes red here instead of in everybody else's run.
    fixture_counted = cases[0][1] in refs
    print("  %-34s %s"
          % ("own fixtures not counted", "🔴 FAILED" if fixture_counted else "ok"))

    ok = gathering_ok and not fixture_counted
    for name, path, want in cases:
        got = resolves(path.rstrip(".,;:)`"), files, dirs)
        mark = "ok" if got == want else "🔴 FAILED"
        if got != want:
            ok = False
        print("  %-34s %s" % (name, mark))
    print("selftest: %s" % ("ok" if ok else "🔴 BROKEN"))
    return 0 if ok else 2


def main() -> int:
    if "--selftest" in sys.argv:
        return selftest()
    files, refs, dangling, orphans = scan()
    if "--orphans" in sys.argv:
        for f in orphans:
            print(f)
        return 0

    print("captures committed          : %d" % len(files))
    print("  cited by a page or a tool : %d" % (len(files) - len(orphans)))
    print("  cited by nothing          : %d  (reported, not failed —" % len(orphans))
    print("                               an orphan may be evidence a page owes)")
    if dangling:
        print("  🔴 cited but NOT committed: %d" % len(dangling))
        for d in dangling:
            print("      %s" % d)
        print("\n🔴 a reader following those gets nothing. Commit the capture, fix the")
        print("   path, or drop the citation.")
        return 1
    print("  🔴 cited but NOT committed: 0")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())