Hankan/tools/dict/freq-forms.mjs

/* Recovering lemma frequency from a surface-form frequency list.

   THE PROBLEM

   The frequency list is Korean as it is actually written; a dictionary is
   keyed on lemmas. Korean is agglutinative with obligatory particles, and
   the verb citation form (-다) essentially never occurs in running text. So
   joining `frequency.word = lemma.headword` would:

     - give ZERO frequency to every verb and adjective, the most important
       class in a learner dictionary, because the citation form barely
       appears while its mass is spread across a dozen inflected forms;
     - understate every noun, whose count is split across the bare form and
       each of its particle-attached forms.

   THE FIX

   Invert the join. Expand each lemma into the surface forms it plausibly
   takes, then SUM the counts over them. No runtime analyser and no
   build-image dependency — just the rules, applied at build time.

   Two honest limitations, both of which cost recall and not correctness:

     - the generated endings are regular, so an irregular stem produces some
       forms that do not exist. A form that is not real simply matches
       nothing in the frequency list.
     - a form claimed by more than one lemma is DROPPED rather than split,
       so homographs do not inherit each other's mass. Dropping is the
       conservative choice.

   These forms are for FREQUENCY SCORING ONLY. The `surface` table the app
   actually queries is populated strictly from lib/conjugation.js
   surfaceForms() plus the headword — see build.mjs. */

import { surfaceForms, haeche, past, polite } from "../../lib/conjugation.js";

/** True when the word ends in a consonant. Pure arithmetic on the syllable. */
export function hasBatchim(word) {
  const cp = word.codePointAt(word.length - 1);
  if (cp === undefined) return false;
  const i = cp - 0xac00;
  return i >= 0 && i <= 11171 && i % 28 !== 0;
}

/** Particles that attach to a noun; allomorph chosen by the final consonant. */
const NOUN_PARTICLES = [
  ["이", "가"], // subject
  ["을", "를"], // object
  ["은", "는"], // topic
  ["과", "와"], // and / with
  ["이나", "나"], // or
  ["이랑", "랑"], // and, casual
  ["으로", "로"], // direction / means
  ["아", "야"], // vocative
];

/** Particles with a single form, whatever the stem ends in. */
const INVARIANT_PARTICLES = [
  "에", // 에
  "에서", // 에서
  "도", // 도
  "의", // 의
  "만", // 만
  "까지", // 까지
  "부터", // 부터
  "처럼", // 처럼
  "보다", // 보다
  "한테", // 한테
  "에게", // 에게
  "께", // 께
  "마다", // 마다
];

/** Endings attached straight to a stem. Regular forms only. */
const STEM_ENDINGS = [
  "고", // 고
  "지", // 지
  "다가", // 다가
  "면서", // 면서
  "네", // 네
  "자", // 자
  "는", // 는, adnominal present
];

/** Endings whose shape depends on whether the stem has a final consonant. */
const STEM_ENDINGS_BATCHIM = [
  ["은", "ㄴ"], // 은 / ㄴ   adnominal past, adjectival
  ["을", "ㄹ"], // 을 / ㄹ   adnominal prospective
  ["으면", "면"], // 으면 / 면
  ["으니까", "니까"], // 으니까 / 니까
  ["습니다", "ㅂ니다"], // 습니다 / ㅂ니다
  ["는다", "ㄴ다"], // 는다 / ㄴ다
];

const JONG = " ㄱㄲㄳㄴㄵㄶㄷㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅄㅅㅆㅇㅈㅊㅋㅌㅍㅎ";

/** Glue a bare-jamo ending onto an open syllable, so the two become one block. */
function fuseFinal(stem, ending) {
  const idx = JONG.indexOf(ending[0]);
  if (idx <= 0) return stem + ending;
  const cp = stem.codePointAt(stem.length - 1);
  const i = cp - 0xac00;
  if (i < 0 || i > 11171 || i % 28 !== 0) return stem + ending; // needs an open syllable
  return stem.slice(0, -1) + String.fromCodePoint(0xac00 + i + idx) + ending.slice(1);
}

const DICT_SUFFIX = "다"; // 다

/**
 * Every surface form this lemma plausibly takes, for summing frequency.
 * Deliberately over-generates: a form that does not exist matches nothing.
 */
export function frequencyForms(headword, pos) {
  const forms = new Set([headword]);

  if (pos === "verb" || pos === "adj") {
    // The real, tested generator first.
    for (const s of surfaceForms(headword, "")) forms.add(s.form);
    const present = haeche(headword);
    if (present) {
      forms.add(present);
      forms.add(polite(present));
      const p = past(present);
      if (p) {
        forms.add(p);
        forms.add(polite(p));
      }
    }

    if (headword.endsWith(DICT_SUFFIX)) {
      const stem = headword.slice(0, -1);
      if (stem) {
        for (const e of STEM_ENDINGS) forms.add(stem + e);
        const batchim = hasBatchim(stem);
        for (const [withB, withoutB] of STEM_ENDINGS_BATCHIM) {
          forms.add(batchim ? stem + withB : fuseFinal(stem, withoutB));
        }
      }
    }
    return forms;
  }

  // Nouns, pronouns, numerals, counters and determiners take particles.
  if (["noun", "pron", "num", "counter", "det"].includes(pos)) {
    const batchim = hasBatchim(headword);
    for (const [withB, withoutB] of NOUN_PARTICLES) {
      forms.add(headword + (batchim ? withB : withoutB));
    }
    for (const p of INVARIANT_PARTICLES) forms.add(headword + p);
  }

  return forms;
}

/**
 * Rank every headword by summed frequency.
 *
 * @param entries  [{ headword, pos }]
 * @param counts   Map<surfaceForm, count> from the frequency list
 * @returns Map<"headword pos", rank> — rank 1 is the most frequent
 */
export function rankByFrequency(entries, counts) {
  const claims = new Map(); // form -> the lemmas that generated it
  const formsOf = new Map();

  for (const e of entries) {
    const key = `${e.headword} ${e.pos}`;
    if (formsOf.has(key)) continue;
    const forms = frequencyForms(e.headword, e.pos);
    formsOf.set(key, forms);
    for (const f of forms) {
      let owners = claims.get(f);
      if (!owners) claims.set(f, (owners = new Set()));
      owners.add(key);
    }
  }

  const totals = [];
  for (const [key, forms] of formsOf) {
    let total = 0;
    for (const f of forms) {
      const n = counts.get(f);
      if (!n) continue;
      // Ambiguous forms are dropped, not split — see the header.
      if (claims.get(f).size > 1) continue;
      total += n;
    }
    if (total > 0) totals.push([key, total]);
  }

  totals.sort((a, b) => b[1] - a[1]);
  const ranks = new Map();
  totals.forEach(([key], i) => ranks.set(key, i + 1));
  return ranks;
}