Hankan/tools/dict/assert-roadmap.mjs

/* Every roadmap word must resolve. Build-time, blocking.

   REVIEW.md §2: 167 of the 371 words the curriculum tells the tutor it may
   use had no lexicon entry, so the word rail silently showed nothing. The
   dictionary import is what fixes that; this is what stops it regressing.

   Runs against the committed band files, so CI needs no network and no
   vendored dictionary. Exits non-zero on any miss.

   Run: npm run dict:assert */

import { readFile } from "node:fs/promises";
import { gunzipSync } from "node:zlib";
import { fileURLToPath } from "node:url";

import { flatten } from "../../lib/gate.js";

const ROOT = fileURLToPath(new URL("../../", import.meta.url));
const DICT = `${ROOT}app/public/dict/`;

const readJson = async (p) => JSON.parse(await readFile(p, "utf8"));

async function loadLexicon() {
  let manifest;
  try {
    manifest = await readJson(`${DICT}manifest.json`);
  } catch {
    console.error("No dictionary found. Run `npm run dict:build` first.");
    process.exit(2);
  }

  const lemmas = new Map(); // headword -> [pos]
  const surfaces = new Map(); // form -> analysis
  const cards = new Map(); // roadmap word -> [{ unit, source }], from unit_id

  for (const band of manifest.bands) {
    const raw = gunzipSync(await readFile(DICT + band.file));
    const data = JSON.parse(raw.toString("utf8"));
    // Columns are positional to keep the files small; see build.mjs.
    const L = data.columns.lemma;
    const S = data.columns.surface;
    const hw = L.indexOf("headword");
    const pos = L.indexOf("pos");
    const unit = L.indexOf("unit_id");
    const source = L.indexOf("source");
    const form = S.indexOf("form");
    const analysis = S.indexOf("analysis");

    for (const row of data.lemmas) {
      const w = row[hw];
      if (!lemmas.has(w)) lemmas.set(w, []);
      lemmas.get(w).push(row[pos]);
      if (unit !== -1 && row[unit]) {
        if (!cards.has(w)) cards.set(w, []);
        cards.get(w).push({ unit: row[unit], source: row[source] });
      }
    }
    for (const row of data.surfaces) {
      if (!surfaces.has(row[form])) surfaces.set(row[form], row[analysis]);
    }
  }

  return { manifest, lemmas, surfaces, cards };
}

const resolves = (lex, word) => lex.lemmas.has(word) || lex.surfaces.has(word);

async function main() {
  const lex = await loadLexicon();
  const curriculum = await readJson(`${ROOT}data/curriculum.json`);
  const deck = await readJson(`${ROOT}data/deck.json`);
  const units = flatten(curriculum);

  const failures = [];
  const check = (kind, where, word) => {
    if (!resolves(lex, word)) failures.push({ kind, where, word });
  };

  /* 1. the roadmap itself — the assertion REVIEW.md asked for */
  let roadmapWords = 0;
  for (const u of units) {
    for (const w of u.words ?? []) {
      roadmapWords++;
      check("roadmap", u.id, w);
    }
  }

  /* 2. the spiral targets, which the gate renders as their own instruction */
  let revisits = 0;
  for (const u of units) {
    for (const r of u.revisits ?? []) {
      revisits++;
      check("revisit", u.id, r.word);
    }
  }

  /* 3. the curated deck, which the vocabulary tab renders directly */
  const deckWords = Object.values(deck.topics).flat();
  for (const [w] of deckWords) check("deck", "deck.json", w);

  /* 4. every roadmap word is exactly ONE reviewable card, tagged with the
        unit that introduces it — the recall evidence, the phase review and
        the gate's met words all hang off that card */
  const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]);
  let cardWords = 0;
  for (const u of units) {
    for (const w of u.words ?? []) {
      cardWords++;
      const tagged = lex.cards.get(w) ?? [];
      if (tagged.length !== 1 || tagged[0].unit !== u.id || !REVIEWABLE.has(tagged[0].source)) {
        failures.push({ kind: "card", where: u.id, word: `${w} (${tagged.length} tagged)` });
      }
    }
  }

  /* ── report ── */
  const src = lex.manifest.builtWith.dictionary;
  console.log(`Dictionary: ${src} — ${lex.manifest.totals.lemmas} lemmas, ` +
    `${lex.manifest.totals.surfaces} surface forms`);
  console.log(`Resolvable: ${lex.lemmas.size} headwords, ${lex.surfaces.size} forms\n`);

  const byKind = (k) => failures.filter((f) => f.kind === k);
  const report = (label, total, kind) => {
    const bad = byKind(kind);
    console.log(`${label}: ${total - bad.length}/${total} resolve` + (bad.length ? "  ✗" : "  ✓"));
    const grouped = {};
    for (const f of bad) (grouped[f.where] ??= []).push(f.word);
    for (const [where, words] of Object.entries(grouped)) {
      console.log(`   ${where}: ${words.join(" · ")}`);
    }
  };

  report("Roadmap words", roadmapWords, "roadmap");
  report("Spiral targets", revisits, "revisit");
  report("Deck words", deckWords.length, "deck");
  report("Roadmap words as one card each", cardWords, "card");

  if (failures.length) {
    console.log(`\nFAIL — ${failures.length} words cannot be glossed.`);
    console.log("Add them to tools/dict/grammar-lexicon.json, or check the dictionary source.");
    process.exit(1);
  }

  console.log("\nPASS — every roadmap word, spiral target and deck word resolves.");
}

await main();