Hankan/tools/dict/sources/krdict.mjs

/* KRDICT (한국어기초사전) adapter — the preferred dictionary source.

   Why it is preferred over kaikki:

     - curated learner glosses in English, written for exactly this audience;
     - a human-graded difficulty level per entry (초급 / 중급 / 고급), which is
       what shared/bands.mjs would rather build the vocabulary bands on than
       a subtitle frequency count;
     - Korean definitions as well as English ones, so the word rail can show
       both.

   ACQUISITION IS MANUAL, ON PURPOSE. The dataset is free and needs no login,
   but the download page is a JavaScript form behind anti-bot protection, so
   no build script can fetch it. Download once by hand and drop the result in
   vendor/ — build.mjs then prefers it automatically and records the choice
   in the manifest:

       https://krdict.korean.go.kr/download/downloadPopup  →  사전 내려받기 → XML

   Either the .zip or its unpacked .xml files will do; a .zip is unpacked
   with the system `unzip`, which is the only external tool this needs.

   Licence: CC BY-SA 2.0 KR. Attribution is required and share-alike applies
   to the derived dictionary data — see NOTICE.md. The XML references audio
   under dicmedia.korean.go.kr; that media is EXCLUDED from the open licence,
   so the URLs are ignored here and nothing is mirrored.

   NOTE: this adapter is written against the documented LMF structure but has
   not been run — no KRDICT file has been vendored yet. The kaikki path is
   the one currently exercised by the build and by CI. */

import { createReadStream } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import { spawn } from "node:child_process";
import { createInterface } from "node:readline";
import path from "node:path";

/** KRDICT 품사 → ours. Anything absent is skipped. */
const POS = {
  명사: "noun",
  동사: "verb",
  형용사: "adj",
  부사: "adv",
  대명사: "pron",
  관형사: "det",
  수사: "num",
  감탄사: "interj",
  조사: "particle",
  접사: "suffix",
  의존명사: "counter",
  어미: "ending",
};

const LEVELS = new Set(["초급", "중급", "고급"]);
const ENGLISH = "영어";

const tag = (xml, name) => {
  const m = xml.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`));
  return m ? decode(m[1].trim()) : "";
};

const attr = (xml, name, key) => {
  const m = xml.match(new RegExp(`<${name}\\b[^>]*\\b${key}="([^"]*)"`));
  return m ? decode(m[1]) : "";
};

function decode(s) {
  return s
    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&apos;/g, "'")
    .replace(/&amp;/g, "&")
    .replace(/<[^>]+>/g, "")
    .trim();
}

/**
 * Read a <feat att="..." val="..."/> style value, which is how LMF carries
 * most of its fields. Falls back to an element of the same name.
 */
function feat(xml, att) {
  const m = xml.match(new RegExp(`<feat\\b[^>]*\\batt="${att}"[^>]*\\bval="([^"]*)"`));
  if (m) return decode(m[1]);
  const alt = xml.match(new RegExp(`<feat\\b[^>]*\\bval="([^"]*)"[^>]*\\batt="${att}"`));
  return alt ? decode(alt[1]) : tag(xml, att);
}

/** The English Equivalent block, if the entry has one. */
function englishEquivalent(senseXml) {
  const blocks = senseXml.match(/<Equivalent\b[\s\S]*?<\/Equivalent>/g) ?? [];
  for (const b of blocks) {
    const lang = feat(b, "language") || attr(b, "Equivalent", "language");
    if (lang && lang !== ENGLISH) continue;
    const lemma = feat(b, "lemma");
    const definition = feat(b, "definition");
    if (lemma || definition) return { lemma, definition };
  }
  return null;
}

function parseEntry(xml) {
  const headword = feat(xml, "writtenForm") || tag(xml, "writtenForm");
  if (!headword || !/^[가-힣]+(?: [가-힣]+)*$/.test(headword)) return null;

  const posKo = feat(xml, "partOfSpeech");
  const pos = POS[posKo];
  if (!pos) return null;

  const levelRaw = feat(xml, "vocabularyLevel");
  const level = LEVELS.has(levelRaw) ? levelRaw : null;

  const senses = xml.match(/<Sense\b[\s\S]*?<\/Sense>/g) ?? [];
  let glossEn = "";
  let glossKo = "";

  for (const s of senses) {
    if (!glossKo) glossKo = feat(s, "definition");
    const eq = englishEquivalent(s);
    if (eq) {
      const text = [eq.lemma, eq.definition].filter(Boolean).join("; ");
      glossEn = glossEn ? `${glossEn}; ${text}` : text;
    }
    if (glossEn.length > 200) break;
  }

  if (!glossEn && !glossKo) return null;

  return {
    headword,
    pos,
    gloss_en: glossEn.slice(0, 240),
    gloss_ko: glossKo.slice(0, 240),
    level,
    source: "krdict",
  };
}

/** Every .xml under a directory, or the file itself. */
async function xmlFiles(target) {
  const s = await stat(target);
  if (s.isFile()) return [target];
  const names = await readdir(target);
  return names
    .filter((n) => n.toLowerCase().endsWith(".xml"))
    .sort()
    .map((n) => path.join(target, n));
}

/** Line stream for either a plain .xml or a member of a .zip. */
function lineStream(file) {
  if (file.toLowerCase().endsWith(".zip")) {
    const proc = spawn("unzip", ["-p", file], { stdio: ["ignore", "pipe", "inherit"] });
    return createInterface({ input: proc.stdout, crlfDelay: Infinity });
  }
  return createInterface({ input: createReadStream(file), crlfDelay: Infinity });
}

/**
 * Yields { headword, pos, gloss_en, gloss_ko, level, source } per entry.
 * Accumulates one <LexicalEntry> at a time so a multi-hundred-MB file never
 * lands in memory.
 */
export async function* readKrdict(target) {
  const files = target.toLowerCase().endsWith(".zip") ? [target] : await xmlFiles(target);

  for (const file of files) {
    let buffer = "";
    let inside = false;

    for await (const line of lineStream(file)) {
      if (!inside && line.includes("<LexicalEntry")) inside = true;
      if (!inside) continue;
      buffer += line + "\n";

      if (line.includes("</LexicalEntry>")) {
        const entry = parseEntry(buffer);
        buffer = "";
        inside = false;
        if (entry) yield entry;
      }
    }
  }
}

/** The .zip or directory to read, if the user has vendored one. */
export async function findKrdict(vendorDir) {
  let names;
  try {
    names = await readdir(vendorDir);
  } catch {
    return null;
  }
  const zip = names.find((n) => /krdict|기초사전/i.test(n) && n.toLowerCase().endsWith(".zip"));
  if (zip) return path.join(vendorDir, zip);
  const dir = names.find((n) => /krdict/i.test(n) && !n.includes("."));
  if (dir) return path.join(vendorDir, dir);
  const xml = names.find((n) => /krdict/i.test(n) && n.toLowerCase().endsWith(".xml"));
  return xml ? path.join(vendorDir, xml) : null;
}

export const KRDICT_ATTRIBUTION =
  "한국어기초사전, 국립국어원 (National Institute of Korean Language) — CC BY-SA 2.0 KR";