Sylpheed/tools/port/blocked-provenance

#!/usr/bin/env python3
"""Date every open row in BLOCKED.md from history, instead of guessing.

`BLOCKED.md` is required to record the HANDOFF commit each row derives from, and
none of the rows in the two open tables do. The file itself says why: nobody
knows when most of them were written, and inventing a sha would be worse than
admitting there is none.

But git does know. A row's derivation is not a memory, it is the commit that
introduced the row -- recoverable with a pickaxe over the file's own history.
This prints, per row:

  introduced   the oldest commit whose diff added the row's key phrase
  HANDOFF@     `git log -1 -- docs/port/HANDOFF.md` as of that commit
  unread       commits touching docs/re/ ON ANY REF that are not ancestors of
               that commit -- decoding the row has never been read against

`--all`, not my own ancestry, and that distinction is the whole finding. Counted
against my checkout every row scores ZERO, which is true and useless: the
Decoder's live decoding sits on `origin/auto/no-disc-and-menu-captures`, `main`
is a hundred-odd commits behind it, and HANDOFF has not moved in four
milestones. So a row can be derived from the newest HANDOFF there is and still
be a day behind the decoding -- and the instruction to record the HANDOFF sha
CANNOT DETECT THAT, because the sha it asks for is constant.

That is the rot mechanism the 2026-08-30 audit found three instances of, and it
is not the one the header of BLOCKED.md describes.

Nothing here is authored. Every field is read out of git, and a row whose key
phrase has been rewritten since it was introduced reports `?` rather than a
plausible-looking sha.
"""
import re, subprocess, sys

DOC = "docs/port/BLOCKED.md"


def git(*a):
    return subprocess.run(["git", *a], capture_output=True, text=True).stdout.strip()


TOP = 3


def idf_of(commits):
    """log(N / how many subjects use the word) -- rarity, from the corpus itself."""
    import collections, math
    df = collections.Counter()
    for _, subj in commits:
        df.update(tokens(subj))
    n = len(commits)
    return collections.defaultdict(lambda: math.log(n), {w: math.log(n / c) for w, c in df.items()})


def key_of(cell):
    """The longest markdown-free fragment -- what to pickaxe for.

    Cells get struck through and re-emphasised as they are resolved, so the cell
    as it stands today is not what was committed. The inner text survives that.
    """
    frags = [f.strip(" ?.") for f in re.split(r"[*~`]+", cell)]
    frags = [f for f in frags if len(f) >= 20]
    return max(frags, key=len) if frags else None


def rows():
    """Every table row in the open sections, in file order."""
    open_only, out = False, []
    for line in open(DOC, encoding="utf-8"):
        if line.startswith("## "):
            open_only = line.startswith("## Still open")
            continue
        if not open_only or not line.startswith("| "):
            continue
        cells = [c.strip() for c in line.strip().strip("|").split(" | ")]
        if len(cells) < 4 or cells[0] in ("Milestone", "---"):
            continue
        out.append(cells)
    return out


STOP = set("""this that with from what which when does than the and are was were
have has been will would could should port game screen menu audio does not any
each only its it's whether where else same both very more most into onto over
under about after before still open blocked answered measured wrong right first
second third disc file files commit branch docs main head sha row rows table
mission handoff decoder agent claim claims""".split())


def stem(w):
    """Crudest possible stemmer, and it earns its place with a control.

    Without it `looping` does not match `loop` and the P6 row whose answer is
    sitting in an unread commit scores zero -- which is what happened.
    """
    for suf in ("ping", "ing", "ted", "ed", "es", "s"):
        if w.endswith(suf) and len(w) - len(suf) >= 4:
            return w[: -len(suf)]
    return w


def tokens(text):
    ws = re.findall(r"[a-z0-9_]{4,}", text.lower())
    return {stem(w) for w in ws if w not in STOP}


def rank(rt, commits, idf):
    """Score every unread commit against one row, rarest words first.

    A COUNT of shared words is the wrong instrument: `menu` and `loop` shared
    scores the same as `plate` and `pulse`, and in this corpus almost everything
    says `menu`. Weighting each shared stem by log(N / commits containing it)
    lets one rare word outrank two common ones -- and it removes the threshold,
    which was the part that could be tuned. The list is RANKED, fixed length,
    so nothing is decided by a cutoff nobody can justify.
    """
    out = []
    for sha, subj in commits:
        shared = rt & tokens(subj)
        if shared:
            out.append((sum(idf[w] for w in shared), sha, subj, shared))
    return sorted(out, reverse=True)


def overlap(rs):
    """Which unread commits NAME something an open row is about.

    Crude on purpose, and it says so: word overlap between a row and a commit
    SUBJECT, ranked by rarity, top few printed with the words that earned the
    rank so the reader judges rather than trusting the match. It cannot tell
    relevance from coincidence -- it narrows 196 commits to a short list worth
    opening, and nothing more.
    """
    log = git("log", "--all", "--not", "HEAD", "--format=%h\t%s", "--", "docs/re/")
    commits = [l.split("\t", 1) for l in log.splitlines() if "\t" in l]
    print(f"  {len(commits)} unread docs/re/ commit(s) exist on other refs.")
    print("  Crude word overlap with the open rows -- a reading list, not a verdict:\n")
    idf = idf_of(commits)
    hits = struck = dropped = 0
    for cells in rs:
        if cells[0].startswith("~~"):
            struck += 1       # already struck; re-reading it settles nothing
            continue
        scored = rank(tokens(cells[0] + " " + cells[1]), commits, idf)
        dropped += max(0, len(scored) - TOP)
        for score, sha, subj, shared in scored[:TOP]:
            hits += 1
            print(f"  {re.sub(r'[*~`]', '', cells[0])[:36]:<36} {sha} {score:5.1f}  {subj[:58]}")
            print(f"  {'':<36} {'':<8}       via {', '.join(sorted(shared))}")
    if not hits:
        print("  (no row shares a word with any unread commit)")
    # Every discard, counted. A detector that can drop a candidate in silence
    # has an unfalsifiable clean run -- which is how the P6 looping row stayed
    # marked open for a day while its answer sat in `712cac8`, and how the same
    # class of miss went unnoticed in the Decoder's checker on the same day.
    print(f"\n  suppressed: {struck} struck row(s) not scanned; {dropped} scoring")
    print(f"  pair(s) ranked below top-{TOP} and not shown; {len(STOP)} word(s)")
    print("  stoplisted and unable to match at any rank.")
    print()


def control():
    """Known positive: the row whose answer is demonstrably in an unread commit.

    `P6 looping` asks where the menu loop restarts. `712cac8` measures it at
    9.44 s and the port has since shipped that value, so the pair MUST match. It
    did not, until stemming -- the check exists so that regression is loud.
    """
    log = git("log", "--all", "--not", "HEAD", "--format=%h\t%s", "--", "docs/re/")
    commits = [l.split("\t", 1) for l in log.splitlines() if "\t" in l]
    scored = rank(tokens("P6 looping where a menu loop restarts"), commits, idf_of(commits))
    at = next((i for i, r in enumerate(scored) if r[1].startswith("712cac8")), None)
    ok = at is not None and at < TOP
    print(f"  control: P6-looping vs 712cac8 -> rank {at} of {len(scored)} scoring "
          f"{'✅' if ok else f'🔴 OUTSIDE TOP-{TOP}, THE KNOWN POSITIVE IS MISSED'}")
    return ok


def main():
    if "--control" in sys.argv:
        sys.exit(0 if control() else 1)
    rs = rows()
    if not rs:
        sys.exit(f"{DOC}: no rows found under a '## Still open' heading")
    head_handoff = git("log", "-1", "--format=%h", "--", "docs/port/HANDOFF.md")
    print(f"  {DOC}: {len(rs)} rows in the open tables")
    print(f"  HANDOFF is at {head_handoff} today\n")
    print(f"  {'row':<44} {'introduced':<12} {'date':<11} {'HANDOFF@':<9} unread")
    unknown = 0
    for cells in rs:
        milestone, needs = cells[0], cells[1]
        label = re.sub(r"[*~`]", "", milestone)[:43]
        key = key_of(needs) or key_of(milestone)
        sha = date = handoff = "?"
        since = "-"
        if key:
            # oldest commit whose diff changed the number of occurrences
            log = git("log", "--format=%h %ad", "--date=short", "-S", key, "--", DOC)
            if log:
                sha, date = log.splitlines()[-1].split()
                handoff = git("log", "-1", "--format=%h", sha, "--", "docs/port/HANDOFF.md")
                unread = git("log", "--all", "--not", sha, "--format=%h", "--", "docs/re/")
                since = str(len(unread.splitlines())) if unread else "0"
        if sha == "?":
            unknown += 1
        flag = ""
        if since not in ("-", "0") and not milestone.startswith("~~"):
            flag = f"  <- never read against {since} docs/re/ commit(s)"
        print(f"  {label:<44} {sha:<12} {date:<11} {handoff:<9} {since:>3}{flag}")
    print()
    overlap(rs)
    if unknown:
        print(f"  ⚠️  {unknown} row(s) could not be dated: the key phrase has been")
        print("      rewritten since it was introduced, so history cannot place it.")
    print("  Not a staleness verdict. A high `unread` is not a wrong row -- most of")
    print("  that decoding is irrelevant to most rows. It is the size of the surface")
    print("  nobody has looked at, and it is what the HANDOFF sha was supposed to be.")


main()