Hankan/tools/dict/build.mjs

/* The dictionary build.

     dictionary source  →  normalize
     + frequency list   →  ranks, via build-time surface expansion
     + data/            →  curated glosses that win over the dictionary
     + grammar lexicon  →  the function words a dictionary cannot supply
     + surfaceForms()   →  the surface index that replaces a runtime analyser
     ------------------------------------------------------------------
     →  app/public/dict/band-N.json.gz  +  manifest.json  +  seed.sqlite3

   Run: npm run dict:build

   Source choice is automatic. If a KRDICT file has been vendored it wins,
   because it carries curated learner glosses and a graded difficulty level;
   otherwise the kaikki extract is used. Either way the manifest records
   which one produced the files.

   The output is committed. KRDICT cannot be re-fetched by CI, and the app
   has to ship the data offline regardless, so the band files are artefacts
   of the repo rather than of the build machine. */

import { mkdir, writeFile, readFile, rm } from "node:fs/promises";
import { gzipSync } from "node:zlib";
import { createHash } from "node:crypto";
import { fileURLToPath } from "node:url";
import { DatabaseSync } from "node:sqlite";

import { surfaceForms, haeche, past } from "../../lib/conjugation.js";
import { flatten } from "../../lib/gate.js";
import { bandOf, bandForUnit, REFERENCE_BAND, BANDS } from "../../shared/bands.mjs";
import { lemmaId } from "../../shared/lemma-id.mjs";

import { readKaikki, KAIKKI_ATTRIBUTION } from "./sources/kaikki.mjs";
import { readKrdict, findKrdict, KRDICT_ATTRIBUTION } from "./sources/krdict.mjs";
import { readFrequency, FREQUENCY_ATTRIBUTION } from "./sources/frequency.mjs";
import { rankByFrequency } from "./freq-forms.mjs";
import { ensureVendor, fetchSource } from "./fetch.mjs";

const ROOT = fileURLToPath(new URL("../../", import.meta.url));
const VENDOR = `${ROOT}vendor/`;
const OUT = `${ROOT}app/public/dict/`;

const readJson = async (p) => JSON.parse(await readFile(p, "utf8"));
const log = (...a) => console.log(...a);

/** The dictionary-form suffix every Korean predicate ends in. */
const DICT_FORM = "\uB2E4";

/* ── merge ────────────────────────────────────────────────────────────
   One entry per (headword, pos). Later sources overwrite earlier ones,
   so the precedence order below is the precedence order of the glosses. */

class Lexicon {
  constructor() {
    this.byKey = new Map();
    this.byHeadword = new Map();
  }
  /** Every entry for a headword, whatever its part of speech. */
  forHeadword(headword) {
    return [...(this.byHeadword.get(headword) ?? [])].map((k) => this.byKey.get(k));
  }
  key(headword, pos) {
    /* Escaped, never a literal NUL: a raw one makes this file binary to
       git, grep and diff. A headword cannot contain it either way. */
    return `${headword}\u0000${pos}`;
  }
  add(entry) {
    const k = this.key(entry.headword, entry.pos);
    const prev = this.byKey.get(k);
    // A later source wins, but never by replacing a real gloss with nothing.
    if (prev && !entry.gloss_en && prev.gloss_en) entry = { ...entry, gloss_en: prev.gloss_en };
    if (prev && !entry.gloss_ko && prev.gloss_ko) entry = { ...entry, gloss_ko: prev.gloss_ko };
    if (prev && entry.level == null && prev.level != null) entry = { ...entry, level: prev.level };
    this.byKey.set(k, entry);
    let keys = this.byHeadword.get(entry.headword);
    if (!keys) this.byHeadword.set(entry.headword, (keys = new Set()));
    keys.add(k);
  }
  get size() {
    return this.byKey.size;
  }
  entries() {
    return [...this.byKey.values()];
  }
}

/* ── sources ──────────────────────────────────────────────────────── */

async function loadDictionary(lex) {
  const krdict = await findKrdict(VENDOR);
  if (krdict) {
    log(`  dictionary: KRDICT — ${krdict.replace(ROOT, "")}`);
    let n = 0;
    for await (const e of readKrdict(krdict)) {
      lex.add(e);
      n++;
    }
    log(`    ${n} entries`);
    return { name: "krdict", attribution: KRDICT_ATTRIBUTION, entries: n };
  }

  const file = await fetchSource("kaikki");
  log(`  dictionary: kaikki — ${file.replace(ROOT, "")}`);
  log("    (vendor a KRDICT download to prefer it — see tools/dict/sources/krdict.mjs)");
  let n = 0;
  for await (const e of readKaikki(file)) {
    lex.add(e);
    n++;
  }
  log(`    ${n} entries`);
  return { name: "kaikki", attribution: KAIKKI_ATTRIBUTION, entries: n };
}

/**
 * data/gloss-extra.json — curated glosses for exactly the roadmap words the
 * deck does not cover. Upstream calls these "gloss only, NOT SRS cards", so
 * they are applied as an OVERRIDE on whatever the dictionary already has
 * rather than as new entries: overwriting the gloss where the headword is
 * already known, and only adding a row where it is not. That keeps one row
 * per real word instead of shadowing every noun with a second, pos-less copy.
 */
async function loadGlossExtra(lex) {
  const extra = await readJson(`${ROOT}data/gloss-extra.json`);
  let overridden = 0;
  let added = 0;

  for (const e of extra.entries) {
    const existing = lex.forHeadword(e.ko);
    if (existing.length) {
      for (const prev of existing) {
        lex.add({ ...prev, gloss_en: e.en, note: e.note ?? "", source: "curated" });
      }
      overridden++;
    } else {
      // Unknown to the dictionary. A -다 headword is a predicate, and haeche()
      // treats verbs and adjectives alike, so "verb" is safe for generating
      // its surface forms; anything else is left unclassified.
      lex.add({
        headword: e.ko,
        pos: e.ko.endsWith(DICT_FORM) ? "verb" : "word",
        gloss_en: e.en,
        gloss_ko: "",
        note: e.note ?? "",
        level: null,
        source: "curated",
      });
      added++;
    }
  }

  log(`  gloss-extra: ${extra.entries.length} entries (${overridden} overrode a dictionary gloss, ${added} new)`);
  return extra.entries.length;
}

/** deck.json — 386 curated words. These glosses beat the dictionary's. */
async function loadDeck(lex, deckOrder) {
  const deck = await readJson(`${ROOT}data/deck.json`);
  let n = 0;
  for (const [topic, rows] of Object.entries(deck.topics)) {
    for (const [headword, , gloss, pos] of rows) {
      const key = lex.key(headword, pos === "phrase" ? "phrase" : pos);
      if (!deckOrder.has(key)) deckOrder.set(key, deckOrder.size);
      lex.add({
        headword,
        pos: pos === "phrase" ? "phrase" : pos,
        gloss_en: gloss,
        gloss_ko: "",
        level: null,
        source: "curated",
        topic,
      });
      n++;
    }
  }
  log(`  curated deck: ${n} entries`);
  return n;
}

/** Sentence chunks and sound words — glossed in context, so worth keeping. */
async function loadSentencesAndSfx(lex) {
  const sentences = await readJson(`${ROOT}data/sentences.json`);
  const sfx = await readJson(`${ROOT}data/sfx.json`);
  let n = 0;

  for (const s of sentences.sentences) {
    for (const [chunk, gloss] of s.parts) {
      lex.add({
        headword: chunk,
        pos: "chunk",
        gloss_en: gloss,
        gloss_ko: "",
        level: null,
        source: "sentence",
      });
      n++;
    }
  }
  for (const item of sfx.items) {
    lex.add({
      headword: item.ko,
      pos: "sfx",
      gloss_en: item.en,
      gloss_ko: "",
      level: null,
      source: "sfx",
      topic: "의성어·의태어",
    });
    n++;
  }
  log(`  sentence chunks + sound words: ${n} entries`);
  return n;
}

/** The hand-written function words. Merged last, so they win outright. */
async function loadGrammar(lex) {
  const g = await readJson(`${ROOT}tools/dict/grammar-lexicon.json`);
  for (const e of g.entries) {
    lex.add({
      headword: e.headword,
      pos: e.pos,
      gloss_en: e.gloss_en,
      gloss_ko: e.gloss_ko ?? "",
      note: e.note ?? "",
      level: null,
      source: "grammar",
      topic: "문법 Grammar",
    });
  }
  log(`  grammar lexicon: ${g.entries.length} entries`);
  return g.entries.length;
}

/* ── curriculum words are cards ────────────────────────────────────────
   Every word a unit introduces must be studiable: the recall evidence, the
   phase-review checklist, the practice set and the gate's "met words" all
   hang off a card. So each roadmap word gets exactly ONE reviewable lemma
   marked with the unit that introduces it.

   Sixteen roadmap words have no reviewable row — inflected forms (먹어,
   봤어) and words the dictionary only knows as something else (자 as
   "ruler", where unit 2.3 means the 반말 of 자다). Those get a lemma of
   their own, glossed from the curated verb they conjugate, else from the
   sentence that uses them, else from the dictionary. */

/** Sources the review deck draws from. */
const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]);

/** A deterministic preference among several reviewable rows for one word. */
const POS_ORDER = ["noun", "verb", "adj", "adv", "pron", "num", "det", "particle", "ending", "phrase", "word"];
const posRank = (pos) => {
  const i = POS_ORDER.indexOf(pos);
  return i === -1 ? POS_ORDER.length : i;
};

function markCurriculum(lex, units, deckOrder) {
  /* 반말 forms of the curated verbs: 봐 is 보다, 먹었어 is 먹다 in the past. */
  const formGloss = new Map();
  for (const e of lex.entries()) {
    if ((e.pos !== "verb" && e.pos !== "adj") || !REVIEWABLE.has(e.source) || !e.gloss_en) continue;
    const g = String(e.gloss_en).replace(/^to be /, "").replace(/^to /, "");
    const present = haeche(e.headword);
    if (!present) continue;
    if (!formGloss.has(present)) formGloss.set(present, { gloss: g, note: `반말, from ${e.headword}` });
    const was = past(present);
    if (was && !formGloss.has(was)) formGloss.set(was, { gloss: `${g} (past)`, note: `반말 past, from ${e.headword}` });
  }

  let chosen = 0;
  let made = 0;
  const seen = new Set();
  for (const u of units) {
    for (const w of u.words ?? []) {
      if (seen.has(w)) continue;
      seen.add(w);
      const topic = `수업 ${u.id} ${u.ko}`;
      const rows = lex.forHeadword(w);
      const reviewable = rows
        .filter((e) => REVIEWABLE.has(e.source))
        .sort(
          (a, b) =>
            (deckOrder.get(lex.key(a.headword, a.pos)) ?? Infinity) -
              (deckOrder.get(lex.key(b.headword, b.pos)) ?? Infinity) ||
            ["curated", "grammar", "sfx", "curriculum"].indexOf(a.source) -
              ["curated", "grammar", "sfx", "curriculum"].indexOf(b.source) ||
            posRank(a.pos) - posRank(b.pos) ||
            a.pos.localeCompare(b.pos),
        );

      if (reviewable.length) {
        const e = reviewable[0];
        lex.add({ ...e, unit_id: u.id, topic: e.topic ?? topic });
        chosen++;
        continue;
      }

      const form = formGloss.get(w);
      const chunk = rows.find((e) => e.source === "sentence" && e.gloss_en);
      const other = rows.find((e) => e.gloss_en);
      lex.add({
        headword: w,
        pos: form ? "form" : "word",
        gloss_en: form?.gloss ?? chunk?.gloss_en ?? other?.gloss_en ?? "",
        gloss_ko: "",
        note: form?.note ?? "",
        level: null,
        source: "curriculum",
        topic,
        unit_id: u.id,
      });
      made++;
    }
  }
  log(`  curriculum: ${chosen + made} roadmap words as cards (${made} given a lemma of their own)`);
}

/* ── build ────────────────────────────────────────────────────────── */

async function main() {
  log("Building the dictionary\n");
  await ensureVendor();
  await mkdir(OUT, { recursive: true });

  const lex = new Lexicon();
  const deckOrder = new Map();
  const dict = await loadDictionary(lex);
  await loadGlossExtra(lex);
  await loadDeck(lex, deckOrder);
  await loadSentencesAndSfx(lex);
  await loadGrammar(lex);

  const curriculum = await readJson(`${ROOT}data/curriculum.json`);
  const units = flatten(curriculum);
  markCurriculum(lex, units, deckOrder);
  log(`  merged: ${lex.size} distinct (headword, pos)\n`);

  /* Frequency, via the inverted join — see freq-forms.mjs. */
  const freqFile = await fetchSource("frequency");
  const counts = await readFrequency(freqFile);
  const entries = lex.entries();
  const ranks = rankByFrequency(entries, counts);
  log(`  frequency: ${counts.size} surface forms → ${ranks.size} lemmas ranked\n`);

  /* Which unit introduces a word, so curriculum vocabulary is admitted by
     the band of its own unit rather than by its raw frequency. */
  const introducedIn = new Map();
  for (const u of units) {
    for (const w of u.words ?? []) if (!introducedIn.has(w)) introducedIn.set(w, u.id);
  }

  /* Sorted, so a rebuild produces byte-identical output. The ids do NOT come
     from this order: an id is a hash of (headword, pos) — see
     shared/lemma-id.mjs — so a card keeps its word however the dictionary
     changes around it. */
  entries.sort((a, b) =>
    a.headword === b.headword ? a.pos.localeCompare(b.pos) : a.headword.localeCompare(b.headword),
  );

  const lemmas = [];
  const surfaces = [];
  const idOwner = new Map();

  for (const e of entries) {
    const id = lemmaId(e.headword, e.pos);
    const clash = idOwner.get(id);
    if (clash) {
      throw new Error(
        `lemma id collision: ${clash} and ${e.headword}/${e.pos} both hash to ${id}. ` +
          "Two words would share every card and review — change the hash before shipping.",
      );
    }
    idOwner.set(id, `${e.headword}/${e.pos}`);
    const rank = ranks.get(`${e.headword} ${e.pos}`) ?? null;
    let band = bandOf({ source: e.source, freqRank: rank, level: e.level });

    const unit = introducedIn.get(e.headword);
    if (unit) band = Math.min(band, bandForUnit(unit));

    lemmas.push({
      id,
      headword: e.headword,
      pos: e.pos,
      freq_rank: rank,
      level: e.level ?? null,
      gloss_en: e.gloss_en ?? "",
      gloss_ko: e.gloss_ko ?? "",
      unit_band: band,
      source: e.source,
      topic: e.topic ?? null,
      unit_id: e.unit_id ?? null,
    });

    /* The surface index. Strictly surfaceForms() plus the headword itself —
       the extra forms in freq-forms.mjs exist only to score frequency and
       are deliberately NOT stored, because they are not all real words. */
    surfaces.push({ form: e.headword, lemma_id: id, analysis: `headword, ${e.pos}` });
    if (e.pos === "verb" || e.pos === "adj") {
      for (const s of surfaceForms(e.headword, e.gloss_en ?? "")) {
        if (s.form) surfaces.push({ form: s.form, lemma_id: id, analysis: s.note });
      }
    }
  }

  log(`  lemmas: ${lemmas.length}`);
  log(`  surface forms: ${surfaces.length}\n`);

  /* ── emit ── */
  const byBand = new Map();
  for (const l of lemmas) {
    if (!byBand.has(l.unit_band)) byBand.set(l.unit_band, { lemmas: [], surfaces: [] });
    byBand.get(l.unit_band).lemmas.push(l);
  }
  const bandOfLemma = new Map(lemmas.map((l) => [l.id, l.unit_band]));
  for (const s of surfaces) {
    const b = bandOfLemma.get(s.lemma_id);
    byBand.get(b).surfaces.push(s);
  }

  const bands = [];
  const allBands = [...BANDS.map((b) => b.band), REFERENCE_BAND];

  for (const band of allBands) {
    const data = byBand.get(band) ?? { lemmas: [], surfaces: [] };
    const rowOf = new Map(data.lemmas.map((l, i) => [l.id, i]));
    /* Arrays, not objects: the key names would otherwise be ~60% of the file.

       Format 2 ships no ids at all. An id is lemmaId(headword, pos), which the
       app computes as it loads, and a surface names its lemma by row index in
       this same file — a surface always travels in its lemma's band. Hashed
       ids written out in full cost 0.5 MB of incompressible digits. */
    const payload = {
      band,
      format: 2,
      columns: {
        lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source", "topic", "unit_id"],
        surface: ["form", "lemma", "analysis"],
      },
      lemmas: data.lemmas.map((l) => [
        l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id,
      ]),
      surfaces: data.surfaces.map((s) => [s.form, rowOf.get(s.lemma_id), s.analysis]),
    };

    const gz = gzipSync(Buffer.from(JSON.stringify(payload), "utf8"), { level: 9 });
    const file = `band-${band}.json.gz`;
    await writeFile(OUT + file, gz);
    const sha = createHash("sha256").update(gz).digest("hex");

    bands.push({
      band,
      file,
      lemmas: data.lemmas.length,
      surfaces: data.surfaces.length,
      bytes: gz.length,
      sha256: sha,
      reference: band === REFERENCE_BAND,
    });
    log(
      `  band ${band}${band === REFERENCE_BAND ? " (reference)" : ""}: ` +
        `${String(data.lemmas.length).padStart(6)} lemmas, ` +
        `${String(data.surfaces.length).padStart(6)} surfaces, ` +
        `${(gz.length / 1024).toFixed(0)} KB`,
    );
  }

  /* A real SQLite file of band 0, for inspection and for anyone who wants to
     open the seed with a normal client. The app loads the band files. */
  await rm(`${OUT}seed.sqlite3`, { force: true });
  const seed = new DatabaseSync(`${OUT}seed.sqlite3`);
  seed.exec(`
    CREATE TABLE lemma (id INTEGER PRIMARY KEY, headword TEXT NOT NULL, pos TEXT NOT NULL,
      freq_rank INTEGER, level TEXT, gloss_en TEXT NOT NULL DEFAULT '',
      gloss_ko TEXT NOT NULL DEFAULT '', unit_band INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL,
      topic TEXT, unit_id TEXT);
    CREATE TABLE surface (form TEXT NOT NULL, lemma_id INTEGER NOT NULL, analysis TEXT NOT NULL,
      PRIMARY KEY (form, lemma_id)) WITHOUT ROWID;
    CREATE INDEX surface_form ON surface(form);
  `);
  const band0 = byBand.get(0) ?? { lemmas: [], surfaces: [] };
  const li = seed.prepare("INSERT INTO lemma VALUES (?,?,?,?,?,?,?,?,?,?,?)");
  const si = seed.prepare("INSERT OR IGNORE INTO surface VALUES (?,?,?)");
  seed.exec("BEGIN");
  for (const l of band0.lemmas)
    li.run(l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id);
  for (const s of band0.surfaces) si.run(s.form, s.lemma_id, s.analysis);
  seed.exec("COMMIT");
  seed.close();

  const manifest = {
    format: 2,
    builtWith: {
      dictionary: dict.name,
      dictionaryEntries: dict.entries,
      frequencyForms: counts.size,
    },
    totals: { lemmas: lemmas.length, surfaces: surfaces.length },
    bands,
    attribution: [dict.attribution, FREQUENCY_ATTRIBUTION],
    notice: "See NOTICE.md. Share-alike applies to this dictionary data, not to the app code.",
  };
  await writeFile(OUT + "manifest.json", JSON.stringify(manifest, null, 2) + "\n");

  const total = bands.reduce((a, b) => a + b.bytes, 0);
  log(`\n  total: ${(total / 1024 / 1024).toFixed(1)} MB across ${bands.length} band files`);
  log(`  written to ${OUT.replace(ROOT, "")}`);
  log("\nNow run: npm run dict:assert");
}

await main();