/* hermitdave/FrequencyWords — Korean, OpenSubtitles2018.

   `word count` per line, descending. 50k lines, 695 KB, plain fetch.
   Licence: CC BY-SA 4.0 for the list content. See NOTICE.md, and note the
   deliberate decision recorded there to keep this data in its own column
   rather than merging it into the dictionary content. */

import { readFile } from "node:fs/promises";

/**
 * Surface form -> occurrence count. These are SURFACE forms from subtitles,
 * not lemmas — see freq-forms.mjs for why that matters and how the ranks
 * are recovered.
 */
export async function readFrequency(path) {
  const text = await readFile(path, "utf8");
  const counts = new Map();

  for (const line of text.split("\n")) {
    const sp = line.indexOf(" ");
    if (sp < 1) continue;
    const word = line.slice(0, sp);
    const n = Number.parseInt(line.slice(sp + 1), 10);
    if (!Number.isFinite(n)) continue;
    // Later duplicates would only ever be smaller; keep the first.
    if (!counts.has(word)) counts.set(word, n);
  }

  return counts;
}

export const FREQUENCY_ATTRIBUTION =
  "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0";