diff --git a/docs/re/METHOD.md b/docs/re/METHOD.md index cf7e73ea..15c0965c 100644 --- a/docs/re/METHOD.md +++ b/docs/re/METHOD.md @@ -2310,3 +2310,33 @@ a regression, and it is the same term I measured from the capture side as a phase-locked shutter โ€” arrived at independently, from the other end of the pipeline. A row containing a sweeping leaf has a **phase-dependent value**, so quoting one without the term attached is an error whichever side produces it. + +## ๐Ÿ”ด `check_refuted.py`'s clean run was not a pass โ€” measured, and it is my instrument + +`sylpheed-port` found their hook's cost is **per-mention, not per-correction**, and +that mentions multiply exactly when writing *about* the mechanism. Testing the same +thing on mine showed the opposite bias, in the worse direction. + +**A real revival, planted inside a paragraph that merely discussed corrections, was +missed silently.** The words "refuted" and "withdrawn" in the surrounding prose +vouched for it. Then the reach: **8 of 8** mentions of a registered claim in this +corpus are suppressed by marker language โ€” **100 %**. So the reported count was `0` +whether or not any of them was live, and I had been reading that 0 as a pass. + +โš ๏ธ **Their token over-reports; this one under-reported.** Over-reporting is the safe +direction โ€” it costs attention. Under-reporting costs the thing the check exists for, +and it is disguised as success. A detector whose null result is indistinguishable +from its positive result measures nothing. + +**Fixed by making the blind spot visible rather than by removing it**: suppressed +mentions are now counted and listed (`--show-marked`) as *"NOT verified, only +vouched for by neighbouring prose"*. The planted revival moves the suppressed count +8 โ†’ 9 and appears in the listing, so it is surfaced rather than silently absorbed. +Marker language stays โ€” dropping it re-creates the header false positives โ€” but it +now downgrades a hit instead of erasing it. + +๐Ÿ“Œ **The general rule: never let a check's suppression path be silent.** If a +detector can discard a candidate, it must say how many it discarded, or its clean +run is unfalsifiable. Both of us reached the same structural conclusion from +opposite failures within a day โ€” theirs by over-reporting loudly, mine by passing +quietly, which is why mine went unnoticed and theirs did not. diff --git a/tools/re-capture/check_refuted.py b/tools/re-capture/check_refuted.py index ab003d1c..664f007e 100755 --- a/tools/re-capture/check_refuted.py +++ b/tools/re-capture/check_refuted.py @@ -52,6 +52,7 @@ for line in ref.read_text().splitlines(): print(f"{len(claims)} quoted claims in REFUTED.md\n") hits = 0 +suppressed = [] for c in claims: needle = c.strip() for f in root.rglob("*.md"): @@ -73,10 +74,32 @@ for c in claims: hdr = lines[j] break near = "\n".join(lines[lo:hi]) + "\n" + hdr - if not any(m.lower() in near.lower() for m in MARKERS) and (f, i) not in seen_report: + marked = any(m.lower() in near.lower() for m in MARKERS) + if marked: + suppressed.append((f, i + 1, needle, l.strip())) + continue + if (f, i) not in seen_report: seen_report.add((f, i)) hits += 1 print(f"๐Ÿ”ด {f}:{i+1}") print(f' claim: "{needle[:80]}"') print(f" line : {l.strip()[:110]}\n") print(f"{hits} unmarked assertion(s) of a refuted claim") + +# ๐Ÿ”ด A CLEAN RUN IS NOT A PASS, and until 2026-08-30 it was reported as though it +# were. Measured: 8 of 8 mentions of a registered claim in this corpus are +# suppressed by marker language, so the count above was 0 whether or not any of +# them was a live revival. A planted REAL revival, written into a paragraph that +# merely discussed corrections, was missed silently -- the marker words in the +# surrounding prose vouched for it. +# +# sylpheed-port's token-based hook has the opposite bias: it OVER-reports on +# well-written corrections, which is the safe direction. This one under-reports, +# which is not. So the suppressed set is printed rather than hidden. +print(f"{len(suppressed)} mention(s) suppressed by nearby marker language " + f"-- NOT verified, only vouched for by neighbouring prose") +if "--show-marked" in sys.argv: + for f, ln, c, l in suppressed: + print(f" ยท {f}:{ln}\n claim: \"{c[:70]}\"\n line : {l[:100]}") +elif suppressed: + print(" re-run with --show-marked to read them")