Hankan/test/domain/freq-forms.test.ts

/* The inverted frequency join.

   A subtitle frequency list holds surface forms; the dictionary holds
   lemmas. Joining them on the headword gives verbs a frequency of roughly
   zero, because the -다 citation form barely occurs in running text. These
   tests pin the expansion that fixes it. */

import { describe, it, expect } from "vitest";
import { frequencyForms, hasBatchim, rankByFrequency } from "../../tools/dict/freq-forms.mjs";

describe("hasBatchim", () => {
  it("detects a final consonant", () => {
    expect(hasBatchim("밥")).toBe(true);
    expect(hasBatchim("학교")).toBe(false);
    expect(hasBatchim("값")).toBe(true);
    expect(hasBatchim("나")).toBe(false);
  });
});

describe("frequencyForms", () => {
  it("expands a verb into the forms it actually appears as", () => {
    const forms = frequencyForms("먹다", "verb");
    // The tested generator's output must be in there.
    for (const f of ["먹어", "먹어요", "먹었어"]) expect(forms.has(f), f).toBe(true);
    // Plus the high-yield endings the citation form hides.
    for (const f of ["먹고", "먹지", "먹으면", "먹는", "먹습니다"]) expect(forms.has(f), f).toBe(true);
    expect(forms.has("먹다")).toBe(true);
  });

  it("picks the right allomorph for an open stem", () => {
    const forms = frequencyForms("가다", "verb");
    expect(forms.has("간다")).toBe(true); // 가 + ㄴ다, fused into one block
    expect(forms.has("갑니다")).toBe(true); // 가 + ㅂ니다
    expect(forms.has("가면")).toBe(true); // no 으 after a vowel
  });

  it("attaches particles to a noun, allomorph by 받침", () => {
    const withFinal = frequencyForms("밥", "noun");
    expect(withFinal.has("밥이")).toBe(true);
    expect(withFinal.has("밥을")).toBe(true);
    expect(withFinal.has("밥은")).toBe(true);
    expect(withFinal.has("밥가")).toBe(false);

    const openStem = frequencyForms("학교", "noun");
    expect(openStem.has("학교가")).toBe(true);
    expect(openStem.has("학교를")).toBe(true);
    expect(openStem.has("학교는")).toBe(true);
    expect(openStem.has("학교이")).toBe(false);
  });

  it("leaves a particle or an ending alone — they take nothing", () => {
    expect([...frequencyForms("은", "particle")]).toEqual(["은"]);
    expect([...frequencyForms("습니다", "ending")]).toEqual(["습니다"]);
  });
});

describe("rankByFrequency", () => {
  it("recovers a verb that the naive headword join would score at zero", () => {
    const counts = new Map([
      ["하다", 1], // the citation form barely occurs …
      ["해", 5000], // … while its real mass sits here
      ["했어", 3000],
      ["하고", 2000],
      ["밥", 400],
    ]);
    const ranks = rankByFrequency(
      [
        { headword: "하다", pos: "verb" },
        { headword: "밥", pos: "noun" },
      ],
      counts,
    );
    // 하다 sums to ~10,000 against 밥's 400, so it must rank first.
    expect(ranks.get("하다 verb")).toBe(1);
    expect(ranks.get("밥 noun")).toBe(2);
  });

  it("drops a form two lemmas both claim, rather than double-counting it", () => {
    // A real collision: 가다's 해체 form is 가, which is also the noun 가.
    // Neither lemma may claim that huge count.
    const counts = new Map([
      ["가", 100_000], // claimed by both — must be ignored
      ["갔어", 700], // 가다 alone
      ["가를", 3], // the noun alone
    ]);
    const ranks = rankByFrequency(
      [
        { headword: "가다", pos: "verb" },
        { headword: "가", pos: "noun" },
      ],
      counts,
    );
    // Each scores only on its unambiguous forms, so the verb wins on 700
    // rather than either of them inheriting 100,000.
    expect(ranks.get("가다 verb")).toBe(1);
    expect(ranks.get("가 noun")).toBe(2);
  });

  it("omits a lemma that matches nothing at all", () => {
    const ranks = rankByFrequency([{ headword: "없는말", pos: "noun" }], new Map([["밥", 10]]));
    expect(ranks.size).toBe(0);
  });

  it("ranks densely from 1, so band ceilings mean what they say", () => {
    const counts = new Map([
      ["가", 300],
      ["나", 200],
      ["다", 100],
    ]);
    const ranks = rankByFrequency(
      ["가", "나", "다"].map((headword) => ({ headword, pos: "adv" })),
      counts,
    );
    expect([...ranks.values()].sort()).toEqual([1, 2, 3]);
  });
});