feat(dict): build pipeline, grammar lexicon, and the shipped band files

Closes REVIEW.md §2. 167 of 371 roadmap words had no lexicon entry, so the
word rail silently showed nothing. Now:

  Roadmap words: 371/371 resolve
  Spiral targets: 35/35 resolve
  Deck words: 386/386 resolve

and npm run dict:assert makes it a blocking build failure, not a silent
empty rail.

No runtime morphological analyser ships. lib/conjugation.js surfaceForms()
runs at BUILD time over every verb and adjective, so looking up a conjugated
form is an index hit on the surface table.

The frequency join had to be inverted. A subtitle frequency list holds
surface forms; a dictionary holds lemmas whose -다 citation form barely
occurs in running text, so joining on headword gives verbs a frequency of
roughly zero. Expanding each lemma into the forms it plausibly takes and
summing recovers 하다 from 118 to 89,041. Forms claimed by more than one
lemma are dropped rather than split, so homographs don't inherit each
other's mass. Those expansions score frequency only — the surface table
itself stays strictly surfaceForms() output plus the headword.

Bands are one per curriculum phase. Phase 1 admits no frequency band at
all: during the writing-system phase every word must be phonologically
legal for the unit reached, and a rank ceiling would hand the learner a
겹받침 during unit 1.4. shared/phonology.mjs lifts validate.mjs's own
feature ladder to enforce that; it agrees with the validator on all 371
words.

Sources are chosen automatically — KRDICT when vendored, otherwise the
kaikki.org extract. KRDICT's download is a JS form behind anti-bot
protection, so it cannot be fetched by CI; the derived band files are
committed instead, which the app needs offline regardless. Attribution and
the share-alike terms are in NOTICE.md.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

test/domain/bands.test.ts

@@ -0,0 +1,129 @@
+/* The vocabulary bands and the phonological ladder — the two things that
+   decide what the tutor is allowed to reach for. */
+
+import { describe, it, expect } from "vitest";
+import { BANDS, REFERENCE_BAND, bandForPhase, bandForUnit, bandOf, ceilingForBand } from "@shared/bands.mjs";
+import { featureLevel, isReadableAt, phonologyViolations, LADDER_COMPLETE } from "@shared/phonology.mjs";
+import { flatten } from "@lib/gate.js";
+import curriculum from "@data/curriculum.json";
+import type { Curriculum } from "@lib/gate.js";
+
+const units = flatten(curriculum as unknown as Curriculum);
+const indexOf = (id: string) => units.findIndex((u) => u.id === id);
+
+describe("bands", () => {
+  it("gives one band per curriculum phase, widening as it goes", () => {
+    expect(BANDS).toHaveLength(6);
+    const ceilings = BANDS.map((b) => b.maxFreq);
+    for (let i = 1; i < ceilings.length; i++) {
+      expect(ceilings[i]!, `band ${i} must not be narrower than band ${i - 1}`).toBeGreaterThan(
+        ceilings[i - 1]!,
+      );
+    }
+  });
+
+  it("maps phases and unit ids onto bands", () => {
+    expect(bandForPhase(1)).toBe(0);
+    expect(bandForPhase(6)).toBe(5);
+    expect(bandForUnit("1.1")).toBe(0);
+    expect(bandForUnit("3.4")).toBe(2);
+    expect(bandForUnit("6.8")).toBe(5);
+  });
+
+  it("clamps a phase outside the curriculum rather than throwing", () => {
+    expect(bandForPhase(0)).toBe(0);
+    expect(bandForPhase(99)).toBe(5);
+  });
+
+  it("admits phase 1 to curated words only — no frequency band at all", () => {
+    expect(ceilingForBand(0)).toBe(0);
+    // Even the single most frequent word in Korean is not admitted by rank.
+    expect(bandOf({ source: "kaikki", freqRank: 1, level: null })).toBeGreaterThan(0);
+    // But a curated word is always available.
+    expect(bandOf({ source: "curated", freqRank: null, level: null })).toBe(0);
+    expect(bandOf({ source: "grammar", freqRank: null, level: null })).toBe(0);
+  });
+
+  it("puts a common word in an early band and a rare one late", () => {
+    const common = bandOf({ source: "kaikki", freqRank: 200, level: null });
+    const mid = bandOf({ source: "kaikki", freqRank: 4000, level: null });
+    const rare = bandOf({ source: "kaikki", freqRank: 12000, level: null });
+    expect(common).toBeLessThan(mid);
+    expect(mid).toBeLessThan(rare);
+  });
+
+  it("lets the curated level override raw frequency", () => {
+    // Same rank, different graded level — the graded one lands later.
+    const beginner = bandOf({ source: "krdict", freqRank: 900, level: "초급" });
+    const advanced = bandOf({ source: "krdict", freqRank: 900, level: "고급" });
+    expect(beginner).toBeLessThan(advanced);
+  });
+
+  it("sends unranked and very rare words to the reference band", () => {
+    expect(bandOf({ source: "kaikki", freqRank: null, level: null })).toBe(REFERENCE_BAND);
+    expect(bandOf({ source: "kaikki", freqRank: 999_999, level: null })).toBe(REFERENCE_BAND);
+  });
+
+  it("keeps the reference band above every gated band, so it can never leak", () => {
+    for (const b of BANDS) expect(REFERENCE_BAND).toBeGreaterThan(b.band);
+  });
+});
+
+describe("the phonological ladder", () => {
+  it("agrees with validate.mjs — no curriculum word breaks its own unit", () => {
+    const violations: string[] = [];
+    units.forEach((u, i) => {
+      const level = featureLevel(i, indexOf);
+      for (const w of u.words ?? []) {
+        const v = phonologyViolations(w, level);
+        if (v.length) violations.push(`${u.id} ${w}: ${v.join(", ")}`);
+      }
+    });
+    expect(violations).toEqual([]);
+  });
+
+  it("climbs monotonically through phase 1", () => {
+    const levels = ["1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8"].map((id) =>
+      featureLevel(indexOf(id), indexOf),
+    );
+    for (let i = 1; i < levels.length; i++) {
+      expect(levels[i]!, `unit 1.${i + 1}`).toBeGreaterThan(levels[i - 1]!);
+    }
+  });
+
+  it("blocks a sound the learner has not reached", () => {
+    const at11 = featureLevel(indexOf("1.1"), indexOf);
+    // Unit 1.1 is batchim-free and uses only the ten basic vowels.
+    expect(isReadableAt("가", at11)).toBe(true);
+    expect(isReadableAt("밥", at11)).toBe(false); // final consonant
+    expect(isReadableAt("개", at11)).toBe(false); // compound vowel
+    expect(isReadableAt("까", at11)).toBe(false); // tense consonant
+  });
+
+  it("admits a double batchim only once 1.7 has been reached", () => {
+    const before = featureLevel(indexOf("1.6"), indexOf);
+    const after = featureLevel(indexOf("1.7"), indexOf);
+    expect(isReadableAt("값", before)).toBe(false);
+    expect(isReadableAt("값", after)).toBe(true);
+  });
+
+  it("admits a liaison context only once 1.5 has been reached", () => {
+    const before = featureLevel(indexOf("1.4"), indexOf);
+    const after = featureLevel(indexOf("1.5"), indexOf);
+    expect(phonologyViolations("음악", before)).toContain("liaison context");
+    expect(isReadableAt("음악", after)).toBe(true);
+  });
+
+  it("is a no-op once the sound phase is over", () => {
+    const at61 = featureLevel(indexOf("6.1"), indexOf);
+    expect(at61).toBe(LADDER_COMPLETE);
+    for (const w of ["괜찮다", "값", "읽었어", "많이"]) {
+      expect(isReadableAt(w, at61), w).toBe(true);
+    }
+  });
+
+  it("ignores non-Hangul characters rather than choking on them", () => {
+    expect(phonologyViolations("ABC 123", 0)).toEqual([]);
+    expect(phonologyViolations("", 0)).toEqual([]);
+  });
+});

test/domain/freq-forms.test.ts

@@ -0,0 +1,116 @@
+/* The inverted frequency join.
+
+   A subtitle frequency list holds surface forms; the dictionary holds
+   lemmas. Joining them on the headword gives verbs a frequency of roughly
+   zero, because the -다 citation form barely occurs in running text. These
+   tests pin the expansion that fixes it. */
+
+import { describe, it, expect } from "vitest";
+import { frequencyForms, hasBatchim, rankByFrequency } from "../../tools/dict/freq-forms.mjs";
+
+describe("hasBatchim", () => {
+  it("detects a final consonant", () => {
+    expect(hasBatchim("밥")).toBe(true);
+    expect(hasBatchim("학교")).toBe(false);
+    expect(hasBatchim("값")).toBe(true);
+    expect(hasBatchim("나")).toBe(false);
+  });
+});
+
+describe("frequencyForms", () => {
+  it("expands a verb into the forms it actually appears as", () => {
+    const forms = frequencyForms("먹다", "verb");
+    // The tested generator's output must be in there.
+    for (const f of ["먹어", "먹어요", "먹었어"]) expect(forms.has(f), f).toBe(true);
+    // Plus the high-yield endings the citation form hides.
+    for (const f of ["먹고", "먹지", "먹으면", "먹는", "먹습니다"]) expect(forms.has(f), f).toBe(true);
+    expect(forms.has("먹다")).toBe(true);
+  });
+
+  it("picks the right allomorph for an open stem", () => {
+    const forms = frequencyForms("가다", "verb");
+    expect(forms.has("간다")).toBe(true); // 가 + ㄴ다, fused into one block
+    expect(forms.has("갑니다")).toBe(true); // 가 + ㅂ니다
+    expect(forms.has("가면")).toBe(true); // no 으 after a vowel
+  });
+
+  it("attaches particles to a noun, allomorph by 받침", () => {
+    const withFinal = frequencyForms("밥", "noun");
+    expect(withFinal.has("밥이")).toBe(true);
+    expect(withFinal.has("밥을")).toBe(true);
+    expect(withFinal.has("밥은")).toBe(true);
+    expect(withFinal.has("밥가")).toBe(false);
+
+    const openStem = frequencyForms("학교", "noun");
+    expect(openStem.has("학교가")).toBe(true);
+    expect(openStem.has("학교를")).toBe(true);
+    expect(openStem.has("학교는")).toBe(true);
+    expect(openStem.has("학교이")).toBe(false);
+  });
+
+  it("leaves a particle or an ending alone — they take nothing", () => {
+    expect([...frequencyForms("은", "particle")]).toEqual(["은"]);
+    expect([...frequencyForms("습니다", "ending")]).toEqual(["습니다"]);
+  });
+});
+
+describe("rankByFrequency", () => {
+  it("recovers a verb that the naive headword join would score at zero", () => {
+    const counts = new Map([
+      ["하다", 1], // the citation form barely occurs …
+      ["해", 5000], // … while its real mass sits here
+      ["했어", 3000],
+      ["하고", 2000],
+      ["밥", 400],
+    ]);
+    const ranks = rankByFrequency(
+      [
+        { headword: "하다", pos: "verb" },
+        { headword: "밥", pos: "noun" },
+      ],
+      counts,
+    );
+    // 하다 sums to ~10,000 against 밥's 400, so it must rank first.
+    expect(ranks.get("하다 verb")).toBe(1);
+    expect(ranks.get("밥 noun")).toBe(2);
+  });
+
+  it("drops a form two lemmas both claim, rather than double-counting it", () => {
+    // A real collision: 가다's 해체 form is 가, which is also the noun 가.
+    // Neither lemma may claim that huge count.
+    const counts = new Map([
+      ["가", 100_000], // claimed by both — must be ignored
+      ["갔어", 700], // 가다 alone
+      ["가를", 3], // the noun alone
+    ]);
+    const ranks = rankByFrequency(
+      [
+        { headword: "가다", pos: "verb" },
+        { headword: "가", pos: "noun" },
+      ],
+      counts,
+    );
+    // Each scores only on its unambiguous forms, so the verb wins on 700
+    // rather than either of them inheriting 100,000.
+    expect(ranks.get("가다 verb")).toBe(1);
+    expect(ranks.get("가 noun")).toBe(2);
+  });
+
+  it("omits a lemma that matches nothing at all", () => {
+    const ranks = rankByFrequency([{ headword: "없는말", pos: "noun" }], new Map([["밥", 10]]));
+    expect(ranks.size).toBe(0);
+  });
+
+  it("ranks densely from 1, so band ceilings mean what they say", () => {
+    const counts = new Map([
+      ["가", 300],
+      ["나", 200],
+      ["다", 100],
+    ]);
+    const ranks = rankByFrequency(
+      ["가", "나", "다"].map((headword) => ({ headword, pos: "adv" })),
+      counts,
+    );
+    expect([...ranks.values()].sort()).toEqual([1, 2, 3]);
+  });
+});