Hankan/tools/dict/sources/kaikki.mjs

/* kaikki.org adapter — English Wiktionary, filtered to Korean.

   ~64k records / ~34k distinct Hangul headwords, with English glosses written
   by and for English speakers. Automatable: a stable URL, no signup.

   What it does NOT have is a curated learner level, so bands fall back to
   frequency alone for entries from this source (see shared/bands.mjs). That
   is the main reason KRDICT is preferred when it has been vendored.

   Licence: CC BY-SA 3.0 + GFDL, inherited from Wiktionary. See NOTICE.md. */

import { createReadStream } from "node:fs";
import { createGunzip } from "node:zlib";
import { createInterface } from "node:readline";

/** kaikki part-of-speech → ours. Anything absent is skipped. */
const POS = {
  noun: "noun",
  verb: "verb",
  adj: "adj",
  adv: "adv",
  pron: "pron",
  det: "det",
  num: "num",
  conj: "conj",
  intj: "interj",
  particle: "particle",
  postp: "particle",
  counter: "counter",
  suffix: "suffix",
  prefix: "prefix",
  phrase: "phrase",
  proverb: "phrase",
  contraction: "contraction",
};

/* Hanja, syllable stubs, proper nouns and romanisations are not words the
   learner is reading manhwa to decode. */
const SKIP_POS = new Set([
  "character",
  "syllable",
  "name",
  "root",
  "symbol",
  "romanization",
  "punct",
  "affix",
  "interfix",
]);

/** Senses worth ignoring when a better one exists. */
const WEAK_TAGS = new Set([
  "obsolete",
  "archaic",
  "rare",
  "dialectal",
  "North-Korea",
  "dated",
  "historical",
]);

const HANGUL_ONLY = /^[가-힣]+(?: [가-힣]+)*$/;
const isEnglish = (s) => s && !/[가-힣]/.test(s);

/** Pick the most useful English gloss, preferring a plain modern sense. */
function bestGloss(senses) {
  const usable = [];
  for (const s of senses ?? []) {
    const glosses = (s.glosses ?? []).filter(isEnglish);
    if (!glosses.length) continue;
    // "form-of" senses describe an inflected form; we generate those
    // ourselves from surfaceForms(), so they add nothing here.
    const tags = new Set(s.tags ?? []);
    if (tags.has("form-of") || s.form_of) continue;
    usable.push({ text: glosses.join("; "), weak: [...tags].some((t) => WEAK_TAGS.has(t)) });
  }
  if (!usable.length) return "";
  const strong = usable.filter((u) => !u.weak);
  const pick = (strong.length ? strong : usable).slice(0, 2);
  return pick.map((p) => p.text).join("; ").slice(0, 240);
}

/**
 * Yields { headword, pos, gloss_en, gloss_ko, level, source } for every
 * usable Korean entry. Streams — the file does not fit comfortably in memory.
 */
export async function* readKaikki(path) {
  const rl = createInterface({
    input: createReadStream(path).pipe(createGunzip()),
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    if (!line) continue;
    let o;
    try {
      o = JSON.parse(line);
    } catch {
      continue; // a truncated tail line is not worth failing the build over
    }

    if (o.lang_code !== "ko") continue;
    if (SKIP_POS.has(o.pos)) continue;
    const pos = POS[o.pos];
    if (!pos) continue;

    const headword = o.word;
    if (!headword || !HANGUL_ONLY.test(headword)) continue;

    const gloss = bestGloss(o.senses);
    if (!gloss) continue;

    yield {
      headword,
      pos,
      gloss_en: gloss,
      gloss_ko: "",
      level: null, // kaikki does not grade entries
      source: "kaikki",
    };
  }
}

export const KAIKKI_ATTRIBUTION =
  "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL";