feat(dict): build pipeline, grammar lexicon, and the shipped band files

Closes REVIEW.md §2. 167 of 371 roadmap words had no lexicon entry, so the word rail silently showed nothing. Now: Roadmap words: 371/371 resolve Spiral targets: 35/35 resolve Deck words: 386/386 resolve and npm run dict:assert makes it a blocking build failure, not a silent empty rail. No runtime morphological analyser ships. lib/conjugation.js surfaceForms() runs at BUILD time over every verb and adjective, so looking up a conjugated form is an index hit on the surface table. The frequency join had to be inverted. A subtitle frequency list holds surface forms; a dictionary holds lemmas whose -다 citation form barely occurs in running text, so joining on headword gives verbs a frequency of roughly zero. Expanding each lemma into the forms it plausibly takes and summing recovers 하다 from 118 to 89,041. Forms claimed by more than one lemma are dropped rather than split, so homographs don't inherit each other's mass. Those expansions score frequency only — the surface table itself stays strictly surfaceForms() output plus the headword. Bands are one per curriculum phase. Phase 1 admits no frequency band at all: during the writing-system phase every word must be phonologically legal for the unit reached, and a rank ceiling would hand the learner a 겹받침 during unit 1.4. shared/phonology.mjs lifts validate.mjs's own feature ladder to enforce that; it agrees with the validator on all 371 words. Sources are chosen automatically — KRDICT when vendored, otherwise the kaikki.org extract. KRDICT's download is a JS form behind anti-bot protection, so it cannot be fetched by CI; the derived band files are committed instead, which the app needs offline regardless. Attribution and the share-alike terms are in NOTICE.md. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-08 19:13:15 +02:00
parent bbe6302a9b
commit 7bd8507909
23 changed files with 1396 additions and 0 deletions
--- a/tools/dict/assert-roadmap.mjs
+++ b/tools/dict/assert-roadmap.mjs
@@ -0,0 +1,124 @@
+/* Every roadmap word must resolve. Build-time, blocking.
+
+   REVIEW.md §2: 167 of the 371 words the curriculum tells the tutor it may
+   use had no lexicon entry, so the word rail silently showed nothing. The
+   dictionary import is what fixes that; this is what stops it regressing.
+
+   Runs against the committed band files, so CI needs no network and no
+   vendored dictionary. Exits non-zero on any miss.
+
+   Run: npm run dict:assert */
+
+import { readFile } from "node:fs/promises";
+import { gunzipSync } from "node:zlib";
+import { fileURLToPath } from "node:url";
+
+import { flatten } from "../../lib/gate.js";
+
+const ROOT = fileURLToPath(new URL("../../", import.meta.url));
+const DICT = `${ROOT}app/public/dict/`;
+
+const readJson = async (p) => JSON.parse(await readFile(p, "utf8"));
+
+async function loadLexicon() {
+  let manifest;
+  try {
+    manifest = await readJson(`${DICT}manifest.json`);
+  } catch {
+    console.error("No dictionary found. Run `npm run dict:build` first.");
+    process.exit(2);
+  }
+
+  const lemmas = new Map(); // headword -> [pos]
+  const surfaces = new Map(); // form -> analysis
+
+  for (const band of manifest.bands) {
+    const raw = gunzipSync(await readFile(DICT + band.file));
+    const data = JSON.parse(raw.toString("utf8"));
+    // Columns are positional to keep the files small; see build.mjs.
+    const L = data.columns.lemma;
+    const S = data.columns.surface;
+    const hw = L.indexOf("headword");
+    const pos = L.indexOf("pos");
+    const form = S.indexOf("form");
+    const analysis = S.indexOf("analysis");
+
+    for (const row of data.lemmas) {
+      const w = row[hw];
+      if (!lemmas.has(w)) lemmas.set(w, []);
+      lemmas.get(w).push(row[pos]);
+    }
+    for (const row of data.surfaces) {
+      if (!surfaces.has(row[form])) surfaces.set(row[form], row[analysis]);
+    }
+  }
+
+  return { manifest, lemmas, surfaces };
+}
+
+const resolves = (lex, word) => lex.lemmas.has(word) || lex.surfaces.has(word);
+
+async function main() {
+  const lex = await loadLexicon();
+  const curriculum = await readJson(`${ROOT}data/curriculum.json`);
+  const deck = await readJson(`${ROOT}data/deck.json`);
+  const units = flatten(curriculum);
+
+  const failures = [];
+  const check = (kind, where, word) => {
+    if (!resolves(lex, word)) failures.push({ kind, where, word });
+  };
+
+  /* 1. the roadmap itself — the assertion REVIEW.md asked for */
+  let roadmapWords = 0;
+  for (const u of units) {
+    for (const w of u.words ?? []) {
+      roadmapWords++;
+      check("roadmap", u.id, w);
+    }
+  }
+
+  /* 2. the spiral targets, which the gate renders as their own instruction */
+  let revisits = 0;
+  for (const u of units) {
+    for (const r of u.revisits ?? []) {
+      revisits++;
+      check("revisit", u.id, r.word);
+    }
+  }
+
+  /* 3. the curated deck, which the vocabulary tab renders directly */
+  const deckWords = Object.values(deck.topics).flat();
+  for (const [w] of deckWords) check("deck", "deck.json", w);
+
+  /* ── report ── */
+  const src = lex.manifest.builtWith.dictionary;
+  console.log(`Dictionary: ${src} — ${lex.manifest.totals.lemmas} lemmas, ` +
+    `${lex.manifest.totals.surfaces} surface forms`);
+  console.log(`Resolvable: ${lex.lemmas.size} headwords, ${lex.surfaces.size} forms\n`);
+
+  const byKind = (k) => failures.filter((f) => f.kind === k);
+  const report = (label, total, kind) => {
+    const bad = byKind(kind);
+    console.log(`${label}: ${total - bad.length}/${total} resolve` + (bad.length ? "  ✗" : "  ✓"));
+    const grouped = {};
+    for (const f of bad) (grouped[f.where] ??= []).push(f.word);
+    for (const [where, words] of Object.entries(grouped)) {
+      console.log(`   ${where}: ${words.join(" · ")}`);
+    }
+  };
+
+  report("Roadmap words", roadmapWords, "roadmap");
+  report("Spiral targets", revisits, "revisit");
+  report("Deck words", deckWords.length, "deck");
+
+  if (failures.length) {
+    console.log(`\nFAIL — ${failures.length} words cannot be glossed.`);
+    console.log("Add them to tools/dict/grammar-lexicon.json, or check the dictionary source.");
+    process.exit(1);
+  }
+
+  console.log("\nPASS — every roadmap word, spiral target and deck word resolves.");
+}
+
+await main();
--- a/tools/dict/build.mjs
+++ b/tools/dict/build.mjs
--- a/tools/dict/fetch.mjs
+++ b/tools/dict/fetch.mjs
@@ -0,0 +1,92 @@
+/* Fetch what can be fetched into vendor/.
+
+   Two of the three sources are a plain download. The third is not:
+
+   KRDICT (한국어기초사전) is the preferred dictionary — it is the only source
+   that carries curated learner glosses in English AND a human-graded
+   difficulty level (초급/중급/고급), which is what the vocabulary bands are
+   built on. It is free and needs no login, but the download page is a
+   JavaScript form behind anti-bot protection, so a script cannot get at it.
+   Fetch it once by hand and drop the ZIP in vendor/; build.mjs picks it up
+   automatically and prefers it over kaikki from then on.
+
+       https://krdict.korean.go.kr/download/downloadPopup
+       → 사전 내려받기 → XML
+
+   Everything here is cached: a source already in vendor/ is left alone. */
+
+import { createWriteStream } from "node:fs";
+import { mkdir, stat } from "node:fs/promises";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGzip } from "node:zlib";
+import { fileURLToPath } from "node:url";
+
+const VENDOR = fileURLToPath(new URL("../../vendor/", import.meta.url));
+
+export const SOURCES = {
+  kaikki: {
+    file: "kaikki-Korean.jsonl.gz",
+    // The ENGLISH-edition Wiktionary, filtered to Korean words — 57,252 of
+    // them, with English glosses. Not to be confused with
+    // /dictionary/downloads/ko/, which is the Korean-EDITION Wiktionary:
+    // words of every language, glossed in Korean. Wrong direction entirely.
+    url: "https://kaikki.org/dictionary/Korean/kaikki.org-dictionary-Korean.jsonl",
+    // Served uncompressed at ~200 MB; gzipped on the way in.
+    gzip: true,
+    note: "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL",
+  },
+  frequency: {
+    file: "ko_full.txt",
+    // The full list, not ko_50k. Korean inflection scatters a lemma's mass
+    // across dozens of surface forms, so the long tail is where most of a
+    // lemma's count actually lives — with the 50k list only a quarter of the
+    // dictionary got any rank at all, and the upper bands came out empty.
+    url: "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/ko/ko_full.txt",
+    note: "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0",
+  },
+};
+
+const MB = (n) => `${(n / 1024 / 1024).toFixed(1)} MB`;
+
+async function sizeOf(path) {
+  try {
+    return (await stat(path)).size;
+  } catch {
+    return null;
+  }
+}
+
+export async function fetchSource(key) {
+  const src = SOURCES[key];
+  const dest = VENDOR + src.file;
+
+  const have = await sizeOf(dest);
+  if (have) {
+    console.log(`  ${src.file} — cached (${MB(have)})`);
+    return dest;
+  }
+
+  console.log(`  ${src.file} — downloading…`);
+  const res = await fetch(src.url);
+  if (!res.ok) throw new Error(`${src.url} → HTTP ${res.status}`);
+  const stages = [Readable.fromWeb(res.body)];
+  if (src.gzip) stages.push(createGzip());
+  stages.push(createWriteStream(dest));
+  await pipeline(...stages);
+  console.log(`  ${src.file} — ${MB(await sizeOf(dest))}  (${src.note})`);
+  return dest;
+}
+
+export async function ensureVendor() {
+  await mkdir(VENDOR, { recursive: true });
+  return VENDOR;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await ensureVendor();
+  console.log("Fetching dictionary sources into vendor/ …");
+  for (const key of Object.keys(SOURCES)) await fetchSource(key);
+  console.log("\nDone. KRDICT, if you want it, is a manual download — see the");
+  console.log("comment at the top of tools/dict/fetch.mjs.");
+}
--- a/tools/dict/freq-forms.mjs
+++ b/tools/dict/freq-forms.mjs
@@ -0,0 +1,195 @@
+/* Recovering lemma frequency from a surface-form frequency list.
+
+   THE PROBLEM
+
+   The frequency list is Korean as it is actually written; a dictionary is
+   keyed on lemmas. Korean is agglutinative with obligatory particles, and
+   the verb citation form (-다) essentially never occurs in running text. So
+   joining `frequency.word = lemma.headword` would:
+
+     - give ZERO frequency to every verb and adjective, the most important
+       class in a learner dictionary, because the citation form barely
+       appears while its mass is spread across a dozen inflected forms;
+     - understate every noun, whose count is split across the bare form and
+       each of its particle-attached forms.
+
+   THE FIX
+
+   Invert the join. Expand each lemma into the surface forms it plausibly
+   takes, then SUM the counts over them. No runtime analyser and no
+   build-image dependency — just the rules, applied at build time.
+
+   Two honest limitations, both of which cost recall and not correctness:
+
+     - the generated endings are regular, so an irregular stem produces some
+       forms that do not exist. A form that is not real simply matches
+       nothing in the frequency list.
+     - a form claimed by more than one lemma is DROPPED rather than split,
+       so homographs do not inherit each other's mass. Dropping is the
+       conservative choice.
+
+   These forms are for FREQUENCY SCORING ONLY. The `surface` table the app
+   actually queries is populated strictly from lib/conjugation.js
+   surfaceForms() plus the headword — see build.mjs. */
+
+import { surfaceForms, haeche, past, polite } from "../../lib/conjugation.js";
+
+/** True when the word ends in a consonant. Pure arithmetic on the syllable. */
+export function hasBatchim(word) {
+  const cp = word.codePointAt(word.length - 1);
+  if (cp === undefined) return false;
+  const i = cp - 0xac00;
+  return i >= 0 && i <= 11171 && i % 28 !== 0;
+}
+
+/** Particles that attach to a noun; allomorph chosen by the final consonant. */
+const NOUN_PARTICLES = [
+  ["이", "가"], // subject
+  ["을", "를"], // object
+  ["은", "는"], // topic
+  ["과", "와"], // and / with
+  ["이나", "나"], // or
+  ["이랑", "랑"], // and, casual
+  ["으로", "로"], // direction / means
+  ["아", "야"], // vocative
+];
+
+/** Particles with a single form, whatever the stem ends in. */
+const INVARIANT_PARTICLES = [
+  "에", // 에
+  "에서", // 에서
+  "도", // 도
+  "의", // 의
+  "만", // 만
+  "까지", // 까지
+  "부터", // 부터
+  "처럼", // 처럼
+  "보다", // 보다
+  "한테", // 한테
+  "에게", // 에게
+  "께", // 께
+  "마다", // 마다
+];
+
+/** Endings attached straight to a stem. Regular forms only. */
+const STEM_ENDINGS = [
+  "고", // 고
+  "지", // 지
+  "다가", // 다가
+  "면서", // 면서
+  "네", // 네
+  "자", // 자
+  "는", // 는, adnominal present
+];
+
+/** Endings whose shape depends on whether the stem has a final consonant. */
+const STEM_ENDINGS_BATCHIM = [
+  ["은", "ㄴ"], // 은 / ㄴ   adnominal past, adjectival
+  ["을", "ㄹ"], // 을 / ㄹ   adnominal prospective
+  ["으면", "면"], // 으면 / 면
+  ["으니까", "니까"], // 으니까 / 니까
+  ["습니다", "ㅂ니다"], // 습니다 / ㅂ니다
+  ["는다", "ㄴ다"], // 는다 / ㄴ다
+];
+
+const JONG = " ㄱㄲㄳㄴㄵㄶㄷㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅄㅅㅆㅇㅈㅊㅋㅌㅍㅎ";
+
+/** Glue a bare-jamo ending onto an open syllable, so the two become one block. */
+function fuseFinal(stem, ending) {
+  const idx = JONG.indexOf(ending[0]);
+  if (idx <= 0) return stem + ending;
+  const cp = stem.codePointAt(stem.length - 1);
+  const i = cp - 0xac00;
+  if (i < 0 || i > 11171 || i % 28 !== 0) return stem + ending; // needs an open syllable
+  return stem.slice(0, -1) + String.fromCodePoint(0xac00 + i + idx) + ending.slice(1);
+}
+
+const DICT_SUFFIX = "다"; // 다
+
+/**
+ * Every surface form this lemma plausibly takes, for summing frequency.
+ * Deliberately over-generates: a form that does not exist matches nothing.
+ */
+export function frequencyForms(headword, pos) {
+  const forms = new Set([headword]);
+
+  if (pos === "verb" || pos === "adj") {
+    // The real, tested generator first.
+    for (const s of surfaceForms(headword, "")) forms.add(s.form);
+    const present = haeche(headword);
+    if (present) {
+      forms.add(present);
+      forms.add(polite(present));
+      const p = past(present);
+      if (p) {
+        forms.add(p);
+        forms.add(polite(p));
+      }
+    }
+
+    if (headword.endsWith(DICT_SUFFIX)) {
+      const stem = headword.slice(0, -1);
+      if (stem) {
+        for (const e of STEM_ENDINGS) forms.add(stem + e);
+        const batchim = hasBatchim(stem);
+        for (const [withB, withoutB] of STEM_ENDINGS_BATCHIM) {
+          forms.add(batchim ? stem + withB : fuseFinal(stem, withoutB));
+        }
+      }
+    }
+    return forms;
+  }
+
+  // Nouns, pronouns, numerals, counters and determiners take particles.
+  if (["noun", "pron", "num", "counter", "det"].includes(pos)) {
+    const batchim = hasBatchim(headword);
+    for (const [withB, withoutB] of NOUN_PARTICLES) {
+      forms.add(headword + (batchim ? withB : withoutB));
+    }
+    for (const p of INVARIANT_PARTICLES) forms.add(headword + p);
+  }
+
+  return forms;
+}
+
+/**
+ * Rank every headword by summed frequency.
+ *
+ * @param entries  [{ headword, pos }]
+ * @param counts   Map<surfaceForm, count> from the frequency list
+ * @returns Map<"headword pos", rank> — rank 1 is the most frequent
+ */
+export function rankByFrequency(entries, counts) {
+  const claims = new Map(); // form -> the lemmas that generated it
+  const formsOf = new Map();
+
+  for (const e of entries) {
+    const key = `${e.headword} ${e.pos}`;
+    if (formsOf.has(key)) continue;
+    const forms = frequencyForms(e.headword, e.pos);
+    formsOf.set(key, forms);
+    for (const f of forms) {
+      let owners = claims.get(f);
+      if (!owners) claims.set(f, (owners = new Set()));
+      owners.add(key);
+    }
+  }
+
+  const totals = [];
+  for (const [key, forms] of formsOf) {
+    let total = 0;
+    for (const f of forms) {
+      const n = counts.get(f);
+      if (!n) continue;
+      // Ambiguous forms are dropped, not split — see the header.
+      if (claims.get(f).size > 1) continue;
+      total += n;
+    }
+    if (total > 0) totals.push([key, total]);
+  }
+
+  totals.sort((a, b) => b[1] - a[1]);
+  const ranks = new Map();
+  totals.forEach(([key], i) => ranks.set(key, i + 1));
+  return ranks;
+}
--- a/tools/dict/grammar-lexicon.json
+++ b/tools/dict/grammar-lexicon.json
@@ -0,0 +1,87 @@
+{
+  "note": "Grammatical detail for function words, layered ON TOP of data/gloss-extra.json. gloss-extra supplies a plain English gloss for all 167 roadmap words the deck does not cover; this file adds what a gloss cannot carry — a part of speech (particle / ending / contraction / bound noun), a Korean gloss, and the allomorph rule (\uc740 after a consonant, \ub294 after a vowel). Merged LAST, so these win for the (headword, pos) pairs they define. Entries always land in band 0: function words are available from the start because the curriculum introduces them explicitly.",
+  "license": "Written for Hankan. No third-party dictionary content.",
+  "entries": [
+    { "headword": "은", "pos": "particle", "gloss_en": "topic marker", "gloss_ko": "주제", "note": "after a consonant; 는 after a vowel" },
+    { "headword": "는", "pos": "particle", "gloss_en": "topic marker", "gloss_ko": "주제", "note": "after a vowel; 은 after a consonant" },
+    { "headword": "이", "pos": "particle", "gloss_en": "subject marker", "gloss_ko": "주격", "note": "after a consonant; 가 after a vowel" },
+    { "headword": "가", "pos": "particle", "gloss_en": "subject marker", "gloss_ko": "주격", "note": "after a vowel; 이 after a consonant" },
+    { "headword": "을", "pos": "particle", "gloss_en": "object marker", "gloss_ko": "목적격", "note": "after a consonant; 를 after a vowel" },
+    { "headword": "를", "pos": "particle", "gloss_en": "object marker", "gloss_ko": "목적격", "note": "after a vowel; 을 after a consonant" },
+    { "headword": "에", "pos": "particle", "gloss_en": "at, to, in — a place or a time", "gloss_ko": "장소·시간", "note": "where something IS or is going" },
+    { "headword": "에서", "pos": "particle", "gloss_en": "at, from — where an action happens", "gloss_ko": "행동의 장소", "note": "contrast with 에: 에서 is where you DO something" },
+    { "headword": "도", "pos": "particle", "gloss_en": "also, too, even", "gloss_ko": "역시", "note": "replaces 은/는 and 이/가 rather than stacking" },
+    { "headword": "만", "pos": "particle", "gloss_en": "only, just", "gloss_ko": "오직", "note": "" },
+    { "headword": "의", "pos": "particle", "gloss_en": "of, 's — possession", "gloss_ko": "소유", "note": "often dropped in speech" },
+    { "headword": "로", "pos": "particle", "gloss_en": "by, with, toward", "gloss_ko": "방향·수단", "note": "after a vowel or ㄹ; 으로 after another consonant" },
+    { "headword": "으로", "pos": "particle", "gloss_en": "by, with, toward", "gloss_ko": "방향·수단", "note": "after a consonant; 로 after a vowel or ㄹ" },
+    { "headword": "와", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a vowel; 과 after a consonant. Written register" },
+    { "headword": "과", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a consonant; 와 after a vowel. Written register" },
+    { "headword": "하고", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "spoken; same job as 와/과" },
+    { "headword": "랑", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "the most casual of 와/과 · 하고 · 랑; 이랑 after a consonant" },
+    { "headword": "이랑", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a consonant; 랑 after a vowel" },
+    { "headword": "부터", "pos": "particle", "gloss_en": "from — a starting point", "gloss_ko": "시작", "note": "pairs with 까지" },
+    { "headword": "까지", "pos": "particle", "gloss_en": "until, as far as, up to", "gloss_ko": "끝", "note": "pairs with 부터" },
+    { "headword": "처럼", "pos": "particle", "gloss_en": "like, as", "gloss_ko": "같이", "note": "" },
+    { "headword": "한테", "pos": "particle", "gloss_en": "to, for — a person", "gloss_ko": "사람에게", "note": "spoken; 에게 in writing" },
+    { "headword": "에게", "pos": "particle", "gloss_en": "to, for — a person", "gloss_ko": "사람에게", "note": "written; 한테 in speech" },
+    { "headword": "께", "pos": "particle", "gloss_en": "to — honorific", "gloss_ko": "높임", "note": "the honorific form of 한테 / 에게" },
+    { "headword": "보다", "pos": "particle", "gloss_en": "than", "gloss_ko": "비교", "note": "comparison; distinct from the verb 보다 'to see'" },
+    { "headword": "마다", "pos": "particle", "gloss_en": "every, each", "gloss_ko": "각각", "note": "" },
+    { "headword": "밖에", "pos": "particle", "gloss_en": "nothing but, only", "gloss_ko": "오직", "note": "always followed by a negative" },
+
+    { "headword": "습니다", "pos": "ending", "gloss_en": "formal declarative ending", "gloss_ko": "합쇼체", "note": "after a consonant stem; ㅂ니다 after a vowel" },
+    { "headword": "ㅂ니다", "pos": "ending", "gloss_en": "formal declarative ending", "gloss_ko": "합쇼체", "note": "after a vowel stem; 습니다 after a consonant" },
+    { "headword": "입니다", "pos": "ending", "gloss_en": "is, am, are — formal", "gloss_ko": "이다의 합쇼체", "note": "the formal form of 이다" },
+    { "headword": "그렇습니다", "pos": "phrase", "gloss_en": "that is so, yes — formal", "gloss_ko": "그렇다의 합쇼체", "note": "formal 그래" },
+    { "headword": "고", "pos": "ending", "gloss_en": "and — links two clauses", "gloss_ko": "연결", "note": "" },
+    { "headword": "지", "pos": "ending", "gloss_en": "isn't it, right? — also the base for 지 않다", "gloss_ko": "확인", "note": "" },
+    { "headword": "면", "pos": "ending", "gloss_en": "if, when", "gloss_ko": "조건", "note": "after a vowel; 으면 after a consonant" },
+    { "headword": "으면", "pos": "ending", "gloss_en": "if, when", "gloss_ko": "조건", "note": "after a consonant; 면 after a vowel" },
+    { "headword": "니까", "pos": "ending", "gloss_en": "because, since", "gloss_ko": "이유", "note": "after a vowel; 으니까 after a consonant" },
+    { "headword": "어서", "pos": "ending", "gloss_en": "and so, because", "gloss_ko": "이유·순서", "note": "아서 after a bright vowel" },
+    { "headword": "는데", "pos": "ending", "gloss_en": "but, and — sets up background", "gloss_ko": "배경", "note": "very common in dialogue" },
+    { "headword": "지만", "pos": "ending", "gloss_en": "but, although", "gloss_ko": "대조", "note": "" },
+    { "headword": "라고", "pos": "ending", "gloss_en": "quoting — \"that …\"", "gloss_ko": "인용", "note": "marks reported speech; 이라고 after a consonant" },
+    { "headword": "대", "pos": "ending", "gloss_en": "they say that …", "gloss_ko": "인용 축약", "note": "contracted from 다고 해" },
+    { "headword": "래", "pos": "ending", "gloss_en": "he says, she says — reported", "gloss_ko": "인용 축약", "note": "contracted from 라고 해" },
+    { "headword": "냬", "pos": "ending", "gloss_en": "asks whether — a reported question", "gloss_ko": "인용 축약", "note": "contracted from 냐고 해" },
+    { "headword": "재", "pos": "ending", "gloss_en": "suggests that — a reported proposal", "gloss_ko": "인용 축약", "note": "contracted from 자고 해" },
+
+    { "headword": "한다", "pos": "form", "gloss_en": "does — plain written style", "gloss_ko": "하다의 해라체", "note": "from 하다; the register of narration and manhwa captions" },
+    { "headword": "했다", "pos": "form", "gloss_en": "did — plain written style", "gloss_ko": "하다의 과거 해라체", "note": "from 하다" },
+    { "headword": "였다", "pos": "form", "gloss_en": "was — plain written style", "gloss_ko": "이다의 과거 해라체", "note": "from 이다; 이었다 after a consonant" },
+
+    { "headword": "것", "pos": "noun", "gloss_en": "thing, one — a bound noun", "gloss_ko": "사물", "note": "needs a modifier in front; 거 in speech" },
+    { "headword": "거", "pos": "noun", "gloss_en": "thing, one", "gloss_ko": "것의 준말", "note": "the spoken form of 것" },
+    { "headword": "수", "pos": "noun", "gloss_en": "way, possibility — as in 할 수 있다 \"can\"", "gloss_ko": "가능성", "note": "bound noun; almost always with 있다 / 없다" },
+    { "headword": "때", "pos": "noun", "gloss_en": "time, when", "gloss_ko": "시간", "note": "bound noun after a modifier" },
+    { "headword": "곳", "pos": "noun", "gloss_en": "place", "gloss_ko": "장소", "note": "bound noun after a modifier" },
+    { "headword": "적", "pos": "noun", "gloss_en": "occasion, the experience of — as in 한 적 있다", "gloss_ko": "경험", "note": "bound noun" },
+    { "headword": "뿐", "pos": "noun", "gloss_en": "only, nothing but", "gloss_ko": "오직", "note": "bound noun" },
+    { "headword": "줄", "pos": "noun", "gloss_en": "how to, the fact that — as in 할 줄 알다", "gloss_ko": "방법", "note": "bound noun" },
+
+    { "headword": "난", "pos": "contraction", "gloss_en": "I — 나 + 는", "gloss_ko": "나는", "note": "" },
+    { "headword": "넌", "pos": "contraction", "gloss_en": "you — 너 + 는", "gloss_ko": "너는", "note": "" },
+    { "headword": "건", "pos": "contraction", "gloss_en": "the thing — 것 + 은", "gloss_ko": "것은", "note": "" },
+    { "headword": "이건", "pos": "contraction", "gloss_en": "this thing — 이것 + 은", "gloss_ko": "이것은", "note": "" },
+    { "headword": "그건", "pos": "contraction", "gloss_en": "that thing — 그것 + 은", "gloss_ko": "그것은", "note": "" },
+    { "headword": "뭘", "pos": "contraction", "gloss_en": "what — 무엇 + 을", "gloss_ko": "무엇을", "note": "" },
+    { "headword": "절", "pos": "contraction", "gloss_en": "me — 저 + 를, humble", "gloss_ko": "저를", "note": "" },
+    { "headword": "날", "pos": "contraction", "gloss_en": "me — 나 + 를", "gloss_ko": "나를", "note": "" },
+    { "headword": "제가", "pos": "contraction", "gloss_en": "I — 저 + 가, humble", "gloss_ko": "저가", "note": "the humble counterpart of 내가" },
+    { "headword": "내가", "pos": "contraction", "gloss_en": "I — 나 + 가", "gloss_ko": "나가", "note": "" },
+
+    { "headword": "드리다", "pos": "verb", "gloss_en": "to give — humble", "gloss_ko": "주다의 겸양", "note": "used when the speaker gives to someone higher" },
+    { "headword": "계시다", "pos": "verb", "gloss_en": "to be, to stay — honorific", "gloss_ko": "있다의 높임", "note": "the honorific of 있다, for people" },
+    { "headword": "주무시다", "pos": "verb", "gloss_en": "to sleep — honorific", "gloss_ko": "자다의 높임", "note": "the honorific of 자다" },
+    { "headword": "앉히다", "pos": "verb", "gloss_en": "to seat someone, to sit someone down", "gloss_ko": "앉게 하다", "note": "causative of 앉다" },
+
+    { "headword": "나중에", "pos": "adv", "gloss_en": "later, afterwards", "gloss_ko": "이따가", "note": "" },
+    { "headword": "신경 쓰다", "pos": "phrase", "gloss_en": "to care about, to be bothered by", "gloss_ko": "마음을 쓰다", "note": "신경 'nerve' + 쓰다 'to use'" },
+    { "headword": "몇", "pos": "det", "gloss_en": "how many, a few", "gloss_ko": "얼마나", "note": "takes a counter after it" },
+    { "headword": "명", "pos": "counter", "gloss_en": "counter for people", "gloss_ko": "사람 세는 말", "note": "" },
+    { "headword": "몇 명", "pos": "phrase", "gloss_en": "how many people", "gloss_ko": "사람이 얼마나", "note": "몇 + the counter 명" },
+    { "headword": "신라", "pos": "noun", "gloss_en": "Silla — the ancient Korean kingdom", "gloss_ko": "옛 나라 이름", "note": "read [실라]; a standing example of ㄴ+ㄹ becoming ㄹㄹ" }
+  ]
+}
--- a/tools/dict/sources/frequency.mjs
+++ b/tools/dict/sources/frequency.mjs
@@ -0,0 +1,33 @@
+/* hermitdave/FrequencyWords — Korean, OpenSubtitles2018.
+
+   `word count` per line, descending. 50k lines, 695 KB, plain fetch.
+   Licence: CC BY-SA 4.0 for the list content. See NOTICE.md, and note the
+   deliberate decision recorded there to keep this data in its own column
+   rather than merging it into the dictionary content. */
+
+import { readFile } from "node:fs/promises";
+
+/**
+ * Surface form -> occurrence count. These are SURFACE forms from subtitles,
+ * not lemmas — see freq-forms.mjs for why that matters and how the ranks
+ * are recovered.
+ */
+export async function readFrequency(path) {
+  const text = await readFile(path, "utf8");
+  const counts = new Map();
+
+  for (const line of text.split("\n")) {
+    const sp = line.indexOf(" ");
+    if (sp < 1) continue;
+    const word = line.slice(0, sp);
+    const n = Number.parseInt(line.slice(sp + 1), 10);
+    if (!Number.isFinite(n)) continue;
+    // Later duplicates would only ever be smaller; keep the first.
+    if (!counts.has(word)) counts.set(word, n);
+  }
+
+  return counts;
+}
+
+export const FREQUENCY_ATTRIBUTION =
+  "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0";
--- a/tools/dict/sources/kaikki.mjs
+++ b/tools/dict/sources/kaikki.mjs
@@ -0,0 +1,125 @@
+/* kaikki.org adapter — English Wiktionary, filtered to Korean.
+
+   ~64k records / ~34k distinct Hangul headwords, with English glosses written
+   by and for English speakers. Automatable: a stable URL, no signup.
+
+   What it does NOT have is a curated learner level, so bands fall back to
+   frequency alone for entries from this source (see shared/bands.mjs). That
+   is the main reason KRDICT is preferred when it has been vendored.
+
+   Licence: CC BY-SA 3.0 + GFDL, inherited from Wiktionary. See NOTICE.md. */
+
+import { createReadStream } from "node:fs";
+import { createGunzip } from "node:zlib";
+import { createInterface } from "node:readline";
+
+/** kaikki part-of-speech → ours. Anything absent is skipped. */
+const POS = {
+  noun: "noun",
+  verb: "verb",
+  adj: "adj",
+  adv: "adv",
+  pron: "pron",
+  det: "det",
+  num: "num",
+  conj: "conj",
+  intj: "interj",
+  particle: "particle",
+  postp: "particle",
+  counter: "counter",
+  suffix: "suffix",
+  prefix: "prefix",
+  phrase: "phrase",
+  proverb: "phrase",
+  contraction: "contraction",
+};
+
+/* Hanja, syllable stubs, proper nouns and romanisations are not words the
+   learner is reading manhwa to decode. */
+const SKIP_POS = new Set([
+  "character",
+  "syllable",
+  "name",
+  "root",
+  "symbol",
+  "romanization",
+  "punct",
+  "affix",
+  "interfix",
+]);
+
+/** Senses worth ignoring when a better one exists. */
+const WEAK_TAGS = new Set([
+  "obsolete",
+  "archaic",
+  "rare",
+  "dialectal",
+  "North-Korea",
+  "dated",
+  "historical",
+]);
+
+const HANGUL_ONLY = /^[가-힣]+(?: [가-힣]+)*$/;
+const isEnglish = (s) => s && !/[가-힣]/.test(s);
+
+/** Pick the most useful English gloss, preferring a plain modern sense. */
+function bestGloss(senses) {
+  const usable = [];
+  for (const s of senses ?? []) {
+    const glosses = (s.glosses ?? []).filter(isEnglish);
+    if (!glosses.length) continue;
+    // "form-of" senses describe an inflected form; we generate those
+    // ourselves from surfaceForms(), so they add nothing here.
+    const tags = new Set(s.tags ?? []);
+    if (tags.has("form-of") || s.form_of) continue;
+    usable.push({ text: glosses.join("; "), weak: [...tags].some((t) => WEAK_TAGS.has(t)) });
+  }
+  if (!usable.length) return "";
+  const strong = usable.filter((u) => !u.weak);
+  const pick = (strong.length ? strong : usable).slice(0, 2);
+  return pick.map((p) => p.text).join("; ").slice(0, 240);
+}
+
+/**
+ * Yields { headword, pos, gloss_en, gloss_ko, level, source } for every
+ * usable Korean entry. Streams — the file does not fit comfortably in memory.
+ */
+export async function* readKaikki(path) {
+  const rl = createInterface({
+    input: createReadStream(path).pipe(createGunzip()),
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of rl) {
+    if (!line) continue;
+    let o;
+    try {
+      o = JSON.parse(line);
+    } catch {
+      continue; // a truncated tail line is not worth failing the build over
+    }
+
+    if (o.lang_code !== "ko") continue;
+    if (SKIP_POS.has(o.pos)) continue;
+    const pos = POS[o.pos];
+    if (!pos) continue;
+
+    const headword = o.word;
+    if (!headword || !HANGUL_ONLY.test(headword)) continue;
+
+    const gloss = bestGloss(o.senses);
+    if (!gloss) continue;
+
+    yield {
+      headword,
+      pos,
+      gloss_en: gloss,
+      gloss_ko: "",
+      level: null, // kaikki does not grade entries
+      source: "kaikki",
+    };
+  }
+}
+
+export const KAIKKI_ATTRIBUTION =
+  "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL";
--- a/tools/dict/sources/krdict.mjs
+++ b/tools/dict/sources/krdict.mjs
@@ -0,0 +1,204 @@
+/* KRDICT (한국어기초사전) adapter — the preferred dictionary source.
+
+   Why it is preferred over kaikki:
+
+     - curated learner glosses in English, written for exactly this audience;
+     - a human-graded difficulty level per entry (초급 / 중급 / 고급), which is
+       what shared/bands.mjs would rather build the vocabulary bands on than
+       a subtitle frequency count;
+     - Korean definitions as well as English ones, so the word rail can show
+       both.
+
+   ACQUISITION IS MANUAL, ON PURPOSE. The dataset is free and needs no login,
+   but the download page is a JavaScript form behind anti-bot protection, so
+   no build script can fetch it. Download once by hand and drop the result in
+   vendor/ — build.mjs then prefers it automatically and records the choice
+   in the manifest:
+
+       https://krdict.korean.go.kr/download/downloadPopup  →  사전 내려받기 → XML
+
+   Either the .zip or its unpacked .xml files will do; a .zip is unpacked
+   with the system `unzip`, which is the only external tool this needs.
+
+   Licence: CC BY-SA 2.0 KR. Attribution is required and share-alike applies
+   to the derived dictionary data — see NOTICE.md. The XML references audio
+   under dicmedia.korean.go.kr; that media is EXCLUDED from the open licence,
+   so the URLs are ignored here and nothing is mirrored.
+
+   NOTE: this adapter is written against the documented LMF structure but has
+   not been run — no KRDICT file has been vendored yet. The kaikki path is
+   the one currently exercised by the build and by CI. */
+
+import { createReadStream } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+import path from "node:path";
+
+/** KRDICT 품사 → ours. Anything absent is skipped. */
+const POS = {
+  명사: "noun",
+  동사: "verb",
+  형용사: "adj",
+  부사: "adv",
+  대명사: "pron",
+  관형사: "det",
+  수사: "num",
+  감탄사: "interj",
+  조사: "particle",
+  접사: "suffix",
+  의존명사: "counter",
+  어미: "ending",
+};
+
+const LEVELS = new Set(["초급", "중급", "고급"]);
+const ENGLISH = "영어";
+
+const tag = (xml, name) => {
+  const m = xml.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`));
+  return m ? decode(m[1].trim()) : "";
+};
+
+const attr = (xml, name, key) => {
+  const m = xml.match(new RegExp(`<${name}\\b[^>]*\\b${key}="([^"]*)"`));
+  return m ? decode(m[1]) : "";
+};
+
+function decode(s) {
+  return s
+    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&amp;/g, "&")
+    .replace(/<[^>]+>/g, "")
+    .trim();
+}
+
+/**
+ * Read a <feat att="..." val="..."/> style value, which is how LMF carries
+ * most of its fields. Falls back to an element of the same name.
+ */
+function feat(xml, att) {
+  const m = xml.match(new RegExp(`<feat\\b[^>]*\\batt="${att}"[^>]*\\bval="([^"]*)"`));
+  if (m) return decode(m[1]);
+  const alt = xml.match(new RegExp(`<feat\\b[^>]*\\bval="([^"]*)"[^>]*\\batt="${att}"`));
+  return alt ? decode(alt[1]) : tag(xml, att);
+}
+
+/** The English Equivalent block, if the entry has one. */
+function englishEquivalent(senseXml) {
+  const blocks = senseXml.match(/<Equivalent\b[\s\S]*?<\/Equivalent>/g) ?? [];
+  for (const b of blocks) {
+    const lang = feat(b, "language") || attr(b, "Equivalent", "language");
+    if (lang && lang !== ENGLISH) continue;
+    const lemma = feat(b, "lemma");
+    const definition = feat(b, "definition");
+    if (lemma || definition) return { lemma, definition };
+  }
+  return null;
+}
+
+function parseEntry(xml) {
+  const headword = feat(xml, "writtenForm") || tag(xml, "writtenForm");
+  if (!headword || !/^[가-힣]+(?: [가-힣]+)*$/.test(headword)) return null;
+
+  const posKo = feat(xml, "partOfSpeech");
+  const pos = POS[posKo];
+  if (!pos) return null;
+
+  const levelRaw = feat(xml, "vocabularyLevel");
+  const level = LEVELS.has(levelRaw) ? levelRaw : null;
+
+  const senses = xml.match(/<Sense\b[\s\S]*?<\/Sense>/g) ?? [];
+  let glossEn = "";
+  let glossKo = "";
+
+  for (const s of senses) {
+    if (!glossKo) glossKo = feat(s, "definition");
+    const eq = englishEquivalent(s);
+    if (eq) {
+      const text = [eq.lemma, eq.definition].filter(Boolean).join("; ");
+      glossEn = glossEn ? `${glossEn}; ${text}` : text;
+    }
+    if (glossEn.length > 200) break;
+  }
+
+  if (!glossEn && !glossKo) return null;
+
+  return {
+    headword,
+    pos,
+    gloss_en: glossEn.slice(0, 240),
+    gloss_ko: glossKo.slice(0, 240),
+    level,
+    source: "krdict",
+  };
+}
+
+/** Every .xml under a directory, or the file itself. */
+async function xmlFiles(target) {
+  const s = await stat(target);
+  if (s.isFile()) return [target];
+  const names = await readdir(target);
+  return names
+    .filter((n) => n.toLowerCase().endsWith(".xml"))
+    .sort()
+    .map((n) => path.join(target, n));
+}
+
+/** Line stream for either a plain .xml or a member of a .zip. */
+function lineStream(file) {
+  if (file.toLowerCase().endsWith(".zip")) {
+    const proc = spawn("unzip", ["-p", file], { stdio: ["ignore", "pipe", "inherit"] });
+    return createInterface({ input: proc.stdout, crlfDelay: Infinity });
+  }
+  return createInterface({ input: createReadStream(file), crlfDelay: Infinity });
+}
+
+/**
+ * Yields { headword, pos, gloss_en, gloss_ko, level, source } per entry.
+ * Accumulates one <LexicalEntry> at a time so a multi-hundred-MB file never
+ * lands in memory.
+ */
+export async function* readKrdict(target) {
+  const files = target.toLowerCase().endsWith(".zip") ? [target] : await xmlFiles(target);
+
+  for (const file of files) {
+    let buffer = "";
+    let inside = false;
+
+    for await (const line of lineStream(file)) {
+      if (!inside && line.includes("<LexicalEntry")) inside = true;
+      if (!inside) continue;
+      buffer += line + "\n";
+
+      if (line.includes("</LexicalEntry>")) {
+        const entry = parseEntry(buffer);
+        buffer = "";
+        inside = false;
+        if (entry) yield entry;
+      }
+    }
+  }
+}
+
+/** The .zip or directory to read, if the user has vendored one. */
+export async function findKrdict(vendorDir) {
+  let names;
+  try {
+    names = await readdir(vendorDir);
+  } catch {
+    return null;
+  }
+  const zip = names.find((n) => /krdict|기초사전/i.test(n) && n.toLowerCase().endsWith(".zip"));
+  if (zip) return path.join(vendorDir, zip);
+  const dir = names.find((n) => /krdict/i.test(n) && !n.includes("."));
+  if (dir) return path.join(vendorDir, dir);
+  const xml = names.find((n) => /krdict/i.test(n) && n.toLowerCase().endsWith(".xml"));
+  return xml ? path.join(vendorDir, xml) : null;
+}
+
+export const KRDICT_ATTRIBUTION =
+  "한국어기초사전, 국립국어원 (National Institute of Korean Language) — CC BY-SA 2.0 KR";