Hankan/tools/dict/fetch.mjs

/* Fetch what can be fetched into vendor/.

   Two of the three sources are a plain download. The third is not:

   KRDICT (한국어기초사전) is the preferred dictionary — it is the only source
   that carries curated learner glosses in English AND a human-graded
   difficulty level (초급/중급/고급), which is what the vocabulary bands are
   built on. It is free and needs no login, but the download page is a
   JavaScript form behind anti-bot protection, so a script cannot get at it.
   Fetch it once by hand and drop the ZIP in vendor/; build.mjs picks it up
   automatically and prefers it over kaikki from then on.

       https://krdict.korean.go.kr/download/downloadPopup
       → 사전 내려받기 → XML

   Everything here is cached: a source already in vendor/ is left alone. */

import { createWriteStream } from "node:fs";
import { mkdir, stat } from "node:fs/promises";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import { createGzip } from "node:zlib";
import { fileURLToPath } from "node:url";

const VENDOR = fileURLToPath(new URL("../../vendor/", import.meta.url));

export const SOURCES = {
  kaikki: {
    file: "kaikki-Korean.jsonl.gz",
    // The ENGLISH-edition Wiktionary, filtered to Korean words — 57,252 of
    // them, with English glosses. Not to be confused with
    // /dictionary/downloads/ko/, which is the Korean-EDITION Wiktionary:
    // words of every language, glossed in Korean. Wrong direction entirely.
    url: "https://kaikki.org/dictionary/Korean/kaikki.org-dictionary-Korean.jsonl",
    // Served uncompressed at ~200 MB; gzipped on the way in.
    gzip: true,
    note: "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL",
  },
  frequency: {
    file: "ko_full.txt",
    // The full list, not ko_50k. Korean inflection scatters a lemma's mass
    // across dozens of surface forms, so the long tail is where most of a
    // lemma's count actually lives — with the 50k list only a quarter of the
    // dictionary got any rank at all, and the upper bands came out empty.
    url: "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/ko/ko_full.txt",
    note: "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0",
  },
};

const MB = (n) => `${(n / 1024 / 1024).toFixed(1)} MB`;

async function sizeOf(path) {
  try {
    return (await stat(path)).size;
  } catch {
    return null;
  }
}

export async function fetchSource(key) {
  const src = SOURCES[key];
  const dest = VENDOR + src.file;

  const have = await sizeOf(dest);
  if (have) {
    console.log(`  ${src.file} — cached (${MB(have)})`);
    return dest;
  }

  console.log(`  ${src.file} — downloading…`);
  const res = await fetch(src.url);
  if (!res.ok) throw new Error(`${src.url} → HTTP ${res.status}`);
  const stages = [Readable.fromWeb(res.body)];
  if (src.gzip) stages.push(createGzip());
  stages.push(createWriteStream(dest));
  await pipeline(...stages);
  console.log(`  ${src.file} — ${MB(await sizeOf(dest))}  (${src.note})`);
  return dest;
}

export async function ensureVendor() {
  await mkdir(VENDOR, { recursive: true });
  return VENDOR;
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await ensureVendor();
  console.log("Fetching dictionary sources into vendor/ …");
  for (const key of Object.keys(SOURCES)) await fetchSource(key);
  console.log("\nDone. KRDICT, if you want it, is a manual download — see the");
  console.log("comment at the top of tools/dict/fetch.mjs.");
}